def __get_menu_entries(self, kibiter_major):
        """ Get the menu entries from the panel definition """
        menu_entries = []
        for entry in self.panels_menu:
            if entry['source'] not in self.data_sources:
                continue
            parent_menu_item = {
                'name': entry['name'],
                'title': entry['name'],
                'description': "",
                'type': "menu",
                'dashboards': []
            }
            for subentry in entry['menu']:
                try:
                    dash_name = get_dashboard_name(subentry['panel'])
                except FileNotFoundError:
                    logging.error("Can't open dashboard file %s", subentry['panel'])
                    continue
                # The name for the entry is in self.panels_menu
                child_item = {
                    "name": subentry['name'],
                    "title": subentry['name'],
                    "description": "",
                    "type": "entry",
                    "panel_id": dash_name
                }
                parent_menu_item['dashboards'].append(child_item)
            menu_entries.append(parent_menu_item)

        return menu_entries

def
__get_menu_entries
(
self
,
kibiter_major
)
:
menu_entries
=
[
]
for
entry
in
self
.
panels_menu
:
if
entry
[
'source'
]
not
in
self
.
data_sources
:
continue
parent_menu_item
=
{
'name'
:
entry
[
'name'
]
,
'title'
:
entry
[
'name'
]
,
'description'
:
""
,
'type'
:
"menu"
,
'dashboards'
:
[
]
}
for
subentry
in
entry
[
'menu'
]
:
try
:
dash_name
=
get_dashboard_name
(
subentry
[
'panel'
]
)
except
FileNotFoundError
:
logging
.
error
(
"Can't open dashboard file %s"
,
subentry
[
'panel'
]
)
continue
child_item
=
{
"name"
:
subentry
[
'name'
]
,
"title"
:
subentry
[
'name'
]
,
"description"
:
""
,
"type"
:
"entry"
,
"panel_id"
:
dash_name
}
parent_menu_item
[
'dashboards'
]
.
append
(
child_item
)
menu_entries
.
append
(
parent_menu_item
)
return
menu_entries


def __get_dash_menu(self, kibiter_major):
        """Order the dashboard menu"""

        # omenu = OrderedDict()
        omenu = []
        # Start with Overview
        omenu.append(self.menu_panels_common['Overview'])

        # Now the data _getsources
        ds_menu = self.__get_menu_entries(kibiter_major)

        # Remove the kafka and community menus, they will be included at the end
        kafka_menu = None
        community_menu = None

        found_kafka = [pos for pos, menu in enumerate(ds_menu) if menu['name'] == KAFKA_NAME]
        if found_kafka:
            kafka_menu = ds_menu.pop(found_kafka[0])

        found_community = [pos for pos, menu in enumerate(ds_menu) if menu['name'] == COMMUNITY_NAME]
        if found_community:
            community_menu = ds_menu.pop(found_community[0])

        ds_menu.sort(key=operator.itemgetter('name'))
        omenu += ds_menu

        # If kafka and community are present add them before the Data Status and About
        if kafka_menu:
            omenu.append(kafka_menu)

        if community_menu:
            omenu.append(community_menu)

        # At the end Data Status, About
        omenu.append(self.menu_panels_common['Data Status'])
        omenu.append(self.menu_panels_common['About'])

        logger.debug("Menu for panels: %s", json.dumps(ds_menu, indent=4))
        return omenu

def
__get_dash_menu
(
self
,
kibiter_major
)
:
omenu
=
[
]
omenu
.
append
(
self
.
menu_panels_common
[
'Overview'
]
)
ds_menu
=
self
.
__get_menu_entries
(
kibiter_major
)
kafka_menu
=
community_menu
=
found_kafka
=
[
pos
for
pos
,
menu
in
enumerate
(
ds_menu
)
if
menu
[
'name'
]
==
KAFKA_NAME
]
if
found_kafka
:
kafka_menu
=
ds_menu
.
pop
(
found_kafka
[
0
]
)
found_community
=
[
pos
for
pos
,
menu
in
enumerate
(
ds_menu
)
if
menu
[
'name'
]
==
COMMUNITY_NAME
]
if
found_community
:
community_menu
=
ds_menu
.
pop
(
found_community
[
0
]
)
ds_menu
.
sort
(
key
=
operator
.
itemgetter
(
'name'
)
)
omenu
+=
ds_menu
if
kafka_menu
:
omenu
.
append
(
kafka_menu
)
if
community_menu
:
omenu
.
append
(
community_menu
)
omenu
.
append
(
self
.
menu_panels_common
[
'Data Status'
]
)
omenu
.
append
(
self
.
menu_panels_common
[
'About'
]
)
logger
.
debug
(
"Menu for panels: %s"
,
json
.
dumps
(
ds_menu
,
indent
=
4
)
)
return
omenu


def compose_mbox(projects):
    """ Compose projects.json only for mbox, but using the mailing_lists lists

    change: 'https://dev.eclipse.org/mailman/listinfo/emft-dev'
    to: 'emfg-dev /home/bitergia/mboxes/emft-dev.mbox/emft-dev.mbox

    :param projects: projects.json
    :return: projects.json with mbox
    """
    mbox_archives = '/home/bitergia/mboxes'

    mailing_lists_projects = [project for project in projects if 'mailing_lists' in projects[project]]
    for mailing_lists in mailing_lists_projects:
        projects[mailing_lists]['mbox'] = []
        for mailing_list in projects[mailing_lists]['mailing_lists']:
            if 'listinfo' in mailing_list:
                name = mailing_list.split('listinfo/')[1]
            elif 'mailing-list' in mailing_list:
                name = mailing_list.split('mailing-list/')[1]
            else:
                name = mailing_list.split('@')[0]

            list_new = "%s %s/%s.mbox/%s.mbox" % (name, mbox_archives, name, name)
            projects[mailing_lists]['mbox'].append(list_new)

    return projects

def
compose_mbox
(
projects
)
:
mbox_archives
=
'/home/bitergia/mboxes'
mailing_lists_projects
=
[
project
for
project
in
projects
if
'mailing_lists'
in
projects
[
project
]
]
for
mailing_lists
in
mailing_lists_projects
:
projects
[
mailing_lists
]
[
'mbox'
]
=
[
]
for
mailing_list
in
projects
[
mailing_lists
]
[
'mailing_lists'
]
:
if
'listinfo'
in
mailing_list
:
name
=
mailing_list
.
split
(
'listinfo/'
)
[
1
]
elif
'mailing-list'
in
mailing_list
:
name
=
mailing_list
.
split
(
'mailing-list/'
)
[
1
]
else
:
name
=
mailing_list
.
split
(
'@'
)
[
0
]
list_new
=
"%s %s/%s.mbox/%s.mbox"
%
(
name
,
mbox_archives
,
name
,
name
)
projects
[
mailing_lists
]
[
'mbox'
]
.
append
(
list_new
)
return
projects

Compose projects . json only for mbox but using the mailing_lists lists


def compose_gerrit(projects):
    """ Compose projects.json for gerrit, but using the git lists

    change: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'
    to: 'git.eclipse.org_xwt/org.eclipse.xwt

    :param projects: projects.json
    :return: projects.json with gerrit
    """
    git_projects = [project for project in projects if 'git' in projects[project]]
    for project in git_projects:
        repos = [repo for repo in projects[project]['git'] if 'gitroot' in repo]
        if len(repos) > 0:
            projects[project]['gerrit'] = []
        for repo in repos:
            gerrit_project = repo.replace("http://git.eclipse.org/gitroot/", "")
            gerrit_project = gerrit_project.replace(".git", "")
            projects[project]['gerrit'].append("git.eclipse.org_" + gerrit_project)

    return projects

def compose_gerrit ( projects ) : git_projects = [ project for project in projects if 'git' in projects [ project ] ] for project in git_projects : repos = [ repo for repo in projects [ project ] [ 'git' ] if 'gitroot' in repo ] if len ( repos ) > 0 : projects [ project ] [ 'gerrit' ] = [ ] for repo in repos : gerrit_project = repo . replace ( "http://git.eclipse.org/gitroot/" , "" ) gerrit_project = gerrit_project . replace ( ".git" , "" ) projects [ project ] [ 'gerrit' ] . append ( "git.eclipse.org_" + gerrit_project ) return projects

Compose projects . json for gerrit but using the git lists


def compose_git(projects, data):
    """ Compose projects.json for git

    We need to replace '/c/' by '/gitroot/' for instance

    change: 'http://git.eclipse.org/c/xwt/org.eclipse.xwt.git'
    to: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with git
    """
    for p in [project for project in data if len(data[project]['source_repo']) > 0]:
        repos = []
        for url in data[p]['source_repo']:
            if len(url['url'].split()) > 1:  # Error at upstream the project 'tools.corrosion'
                repo = url['url'].split()[1].replace('/c/', '/gitroot/')
            else:
                repo = url['url'].replace('/c/', '/gitroot/')

            if repo not in repos:
                repos.append(repo)

        projects[p]['git'] = repos

    return projects

def compose_git ( projects , data ) : for p in [ project for project in data if len ( data [ project ] [ 'source_repo' ] ) > 0 ] : repos = [ ] for url in data [ p ] [ 'source_repo' ] : if len ( url [ 'url' ] . split ( ) ) > 1 : repo = url [ 'url' ] . split ( ) [ 1 ] . replace ( '/c/' , '/gitroot/' ) else : repo = url [ 'url' ] . replace ( '/c/' , '/gitroot/' ) if repo not in repos : repos . append ( repo ) projects [ p ] [ 'git' ] = repos return projects


def compose_mailing_lists(projects, data):
    """ Compose projects.json for mailing lists

    At upstream has two different key for mailing list: 'mailings_lists' and 'dev_list'
    The key 'mailing_lists' is an array with mailing lists
    The key 'dev_list' is a dict with only one mailing list

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with mailing_lists
    """
    for p in [project for project in data if len(data[project]['mailing_lists']) > 0]:
        if 'mailing_lists' not in projects[p]:
            projects[p]['mailing_lists'] = []

        urls = [url['url'].replace('mailto:', '') for url in data[p]['mailing_lists'] if
                url['url'] not in projects[p]['mailing_lists']]
        projects[p]['mailing_lists'] += urls

    for p in [project for project in data if len(data[project]['dev_list']) > 0]:
        if 'mailing_lists' not in projects[p]:
            projects[p]['mailing_lists'] = []

        mailing_list = data[p]['dev_list']['url'].replace('mailto:', '')
        projects[p]['mailing_lists'].append(mailing_list)

    return projects

def compose_mailing_lists ( projects , data ) : for p in [ project for project in data if len ( data [ project ] [ 'mailing_lists' ] ) > 0 ] : if 'mailing_lists' not in projects [ p ] : projects [ p ] [ 'mailing_lists' ] = [ ] urls = [ url [ 'url' ] . replace ( 'mailto:' , '' ) for url in data [ p ] [ 'mailing_lists' ] if url [ 'url' ] not in projects [ p ] [ 'mailing_lists' ] ] projects [ p ] [ 'mailing_lists' ] += urls for p in [ project for project in data if len ( data [ project ] [ 'dev_list' ] ) > 0 ] : if 'mailing_lists' not in projects [ p ] : projects [ p ] [ 'mailing_lists' ] = [ ] mailing_list = data [ p ] [ 'dev_list' ] [ 'url' ] . replace ( 'mailto:' , '' ) projects [ p ] [ 'mailing_lists' ] . append ( mailing_list ) return projects

Compose projects . json for mailing lists


def compose_github(projects, data):
    """ Compose projects.json for github

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with github
    """
    for p in [project for project in data if len(data[project]['github_repos']) > 0]:
        if 'github' not in projects[p]:
            projects[p]['github'] = []

        urls = [url['url'] for url in data[p]['github_repos'] if
                url['url'] not in projects[p]['github']]
        projects[p]['github'] += urls

    return projects

def compose_github ( projects , data ) : for p in [ project for project in data if len ( data [ project ] [ 'github_repos' ] ) > 0 ] : if 'github' not in projects [ p ] : projects [ p ] [ 'github' ] = [ ] urls = [ url [ 'url' ] for url in data [ p ] [ 'github_repos' ] if url [ 'url' ] not in projects [ p ] [ 'github' ] ] projects [ p ] [ 'github' ] += urls return projects


def compose_bugzilla(projects, data):
    """ Compose projects.json for bugzilla

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with bugzilla
    """
    for p in [project for project in data if len(data[project]['bugzilla']) > 0]:
        if 'bugzilla' not in projects[p]:
            projects[p]['bugzilla'] = []

        urls = [url['query_url'] for url in data[p]['bugzilla'] if
                url['query_url'] not in projects[p]['bugzilla']]
        projects[p]['bugzilla'] += urls

    return projects

def compose_bugzilla ( projects , data ) : for p in [ project for project in data if len ( data [ project ] [ 'bugzilla' ] ) > 0 ] : if 'bugzilla' not in projects [ p ] : projects [ p ] [ 'bugzilla' ] = [ ] urls = [ url [ 'query_url' ] for url in data [ p ] [ 'bugzilla' ] if url [ 'query_url' ] not in projects [ p ] [ 'bugzilla' ] ] projects [ p ] [ 'bugzilla' ] += urls return projects


def compose_title(projects, data):
    """ Compose the projects JSON file only with the projects name

    :param projects: projects.json
    :param data: eclipse JSON with the origin format
    :return: projects.json with titles
    """
    for project in data:
        projects[project] = {
            'meta': {
                'title': data[project]['title']
            }
        }
    return projects

def compose_title ( projects , data ) : for project in data : projects [ project ] = { 'meta' : { 'title' : data [ project ] [ 'title' ] } } return projects


def compose_projects_json(projects, data):
    """ Compose projects.json with all data sources

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with all data sources
    """
    projects = compose_git(projects, data)
    projects = compose_mailing_lists(projects, data)
    projects = compose_bugzilla(projects, data)
    projects = compose_github(projects, data)
    projects = compose_gerrit(projects)
    projects = compose_mbox(projects)

    return projects

def compose_projects_json ( projects , data ) : projects = compose_git ( projects , data ) projects = compose_mailing_lists ( projects , data ) projects = compose_bugzilla ( projects , data ) projects = compose_github ( projects , data ) projects = compose_gerrit ( projects ) projects = compose_mbox ( projects ) return projects

Compose projects . json with all data sources


def __autorefresh_studies(self, cfg):
        """Execute autorefresh for areas of code study if configured"""

        if 'studies' not in self.conf[self.backend_section] or \
                'enrich_areas_of_code:git' not in self.conf[self.backend_section]['studies']:
            logger.debug("Not doing autorefresh for studies, Areas of Code study is not active.")
            return

        aoc_index = self.conf['enrich_areas_of_code:git'].get('out_index', GitEnrich.GIT_AOC_ENRICHED)

        # if `out_index` exists but has no value, use default
        if not aoc_index:
            aoc_index = GitEnrich.GIT_AOC_ENRICHED

        logger.debug("Autorefresh for Areas of Code study index: %s", aoc_index)

        es = Elasticsearch([self.conf['es_enrichment']['url']], timeout=100,
                           verify_certs=self._get_enrich_backend().elastic.requests.verify)

        if not es.indices.exists(index=aoc_index):
            logger.debug("Not doing autorefresh, index doesn't exist for Areas of Code study")
            return

        logger.debug("Doing autorefresh for Areas of Code study")

        # Create a GitEnrich backend tweaked to work with AOC index
        aoc_backend = GitEnrich(self.db_sh, None, cfg['projects']['projects_file'],
                                self.db_user, self.db_password, self.db_host)
        aoc_backend.mapping = None
        aoc_backend.roles = ['author']
        elastic_enrich = get_elastic(self.conf['es_enrichment']['url'],
                                     aoc_index, clean=False, backend=aoc_backend)
        aoc_backend.set_elastic(elastic_enrich)

        self.__autorefresh(aoc_backend, studies=True)


def __studies(self, retention_time):
        """ Execute the studies configured for the current backend """

        cfg = self.config.get_conf()
        if 'studies' not in cfg[self.backend_section] or not \
           cfg[self.backend_section]['studies']:
            logger.debug('No studies for %s' % self.backend_section)
            return

        studies = [study for study in cfg[self.backend_section]['studies'] if study.strip() != ""]
        if not studies:
            logger.debug('No studies for %s' % self.backend_section)
            return

        logger.debug("Executing studies for %s: %s" % (self.backend_section, studies))
        time.sleep(2)  # Wait so enrichment has finished in ES
        enrich_backend = self._get_enrich_backend()
        ocean_backend = self._get_ocean_backend(enrich_backend)

        active_studies = []
        all_studies = enrich_backend.studies
        all_studies_names = [study.__name__ for study in enrich_backend.studies]

        # Time to check that configured studies are valid
        logger.debug("All studies in %s: %s", self.backend_section, all_studies_names)
        logger.debug("Configured studies %s", studies)
        cfg_studies_types = [study.split(":")[0] for study in studies]
        if not set(cfg_studies_types).issubset(set(all_studies_names)):
            logger.error('Wrong studies names for %s: %s', self.backend_section, studies)
            raise RuntimeError('Wrong studies names ', self.backend_section, studies)

        for study in enrich_backend.studies:
            if study.__name__ in cfg_studies_types:
                active_studies.append(study)

        enrich_backend.studies = active_studies
        print("Executing for %s the studies %s" % (self.backend_section,
              [study for study in studies]))

        studies_args = self.__load_studies()

        do_studies(ocean_backend, enrich_backend, studies_args, retention_time=retention_time)
        # Return studies to its original value
        enrich_backend.studies = all_studies

def
__studies
(
self
,
retention_time
)
:
cfg
=
self
.
config
.
get_conf
(
)
if
'studies'
not
in
cfg
[
self
.
backend_section
]
or
not
cfg
[
self
.
backend_section
]
[
'studies'
]
:
logger
.
debug
(
'No studies for %s'
%
self
.
backend_section
)
return
studies
=
[
study
for
study
in
cfg
[
self
.
backend_section
]
[
'studies'
]
if
study
.
strip
(
)
!=
""
]
if
not
studies
:
logger
.
debug
(
'No studies for %s'
%
self
.
backend_section
)
return
logger
.
debug
(
"Executing studies for %s: %s"
%
(
self
.
backend_section
,
studies
)
)
time
.
sleep
(
2
)
enrich_backend
=
self
.
_get_enrich_backend
(
)
ocean_backend
=
self
.
_get_ocean_backend
(
enrich_backend
)
active_studies
=
[
]
all_studies
=
enrich_backend
.
studies
all_studies_names
=
[
study
.
__name__
for
study
in
enrich_backend
.
studies
]
logger
.
debug
(
"All studies in %s: %s"
,
self
.
backend_section
,
all_studies_names
)
logger
.
debug
(
"Configured studies %s"
,
studies
)
cfg_studies_types
=
[
study
.
split
(
":"
)
[
0
]
for
study
in
studies
]
if
not
set
(
cfg_studies_types
)
.
issubset
(
set
(
all_studies_names
)
)
:
logger
.
error
(
'Wrong studies names for %s: %s'
,
self
.
backend_section
,
studies
)
raise
RuntimeError
(
'Wrong studies names '
,
self
.
backend_section
,
studies
)
for
study
in
enrich_backend
.
studies
:
if
study
.
__name__
in
cfg_studies_types
:
active_studies
.
append
(
study
)
enrich_backend
.
studies
=
active_studies
print
(
"Executing for %s the studies %s"
%
(
self
.
backend_section
,
[
study
for
study
in
studies
]
)
)
studies_args
=
self
.
__load_studies
(
)
do_studies
(
ocean_backend
,
enrich_backend
,
studies_args
,
retention_time
=
retention_time
)
enrich_backend
.
studies
=
all_studies


def retain_identities(self, retention_time):
        """Retain the identities in SortingHat based on the `retention_time`
        value declared in the setup.cfg.

        :param retention_time: maximum number of minutes wrt the current date to retain the SortingHat data
        """
        enrich_es = self.conf['es_enrichment']['url']
        sortinghat_db = self.db
        current_data_source = self.get_backend(self.backend_section)
        active_data_sources = self.config.get_active_data_sources()

        if retention_time is None:
            logger.debug("[identities retention] Retention policy disabled, no identities will be deleted.")
            return

        if retention_time <= 0:
            logger.debug("[identities retention] Retention time must be greater than 0.")
            return

        retain_identities(retention_time, enrich_es, sortinghat_db, current_data_source, active_data_sources)

Retain the identities in SortingHat based on the retention_time value declared in the setup . cfg .


def get_repos_by_backend_section(cls, backend_section, raw=True):
        """ return list with the repositories for a backend_section """
        repos = []
        projects = TaskProjects.get_projects()

        for pro in projects:
            if backend_section in projects[pro]:
                # if the projects.json doesn't contain the `unknown` project, add the repos in the bck section
                if cls.GLOBAL_PROJECT not in projects:
                    repos += projects[pro][backend_section]
                else:
                    # if the projects.json contains the `unknown` project
                    # in the case of the collection phase
                    if raw:
                        # if the current project is not `unknown`
                        if pro != cls.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if backend_section not in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                            # if the backend section is in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[cls.GLOBAL_PROJECT][backend_section]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            not_in_unknown = [projects[pro] for pro in projects if pro != cls.GLOBAL_PROJECT][0]
                            if backend_section not in not_in_unknown:
                                repos += projects[cls.GLOBAL_PROJECT][backend_section]
                    # in the case of the enrichment phase
                    else:
                        # if the current project is not `unknown`
                        if pro != cls.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if backend_section not in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                            # if the backend section is in the `unknown` project, add the repos in the bck section
                            elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            not_in_unknown_prj = [projects[prj] for prj in projects if prj != cls.GLOBAL_PROJECT]
                            not_in_unknown_sections = list(set([section for prj in not_in_unknown_prj
                                                                for section in list(prj.keys())]))
                            if backend_section not in not_in_unknown_sections:
                                repos += projects[pro][backend_section]

        logger.debug("List of repos for %s: %s (raw=%s)", backend_section, repos, raw)

        # avoid duplicated repos
        repos = list(set(repos))

        return repos

def
get_repos_by_backend_section
(
cls
,
backend_section
,
raw
=
True
)
:
repos
=
[
]
projects
=
TaskProjects
.
get_projects
(
)
for
pro
in
projects
:
if
backend_section
in
projects
[
pro
]
:
if
cls
.
GLOBAL_PROJECT
not
in
projects
:
repos
+=
projects
[
pro
]
[
backend_section
]
else
:
if
raw
:
if
pro
!=
cls
.
GLOBAL_PROJECT
:
if
backend_section
not
in
projects
[
cls
.
GLOBAL_PROJECT
]
:
repos
+=
projects
[
pro
]
[
backend_section
]
elif
backend_section
in
projects
[
pro
]
and
backend_section
in
projects
[
cls
.
GLOBAL_PROJECT
]
:
repos
+=
projects
[
cls
.
GLOBAL_PROJECT
]
[
backend_section
]
else
:
not_in_unknown
=
[
projects
[
pro
]
for
pro
in
projects
if
pro
!=
cls
.
GLOBAL_PROJECT
]
[
0
]
if
backend_section
not
in
not_in_unknown
:
repos
+=
projects
[
cls
.
GLOBAL_PROJECT
]
[
backend_section
]
else
:
if
pro
!=
cls
.
GLOBAL_PROJECT
:
if
backend_section
not
in
projects
[
cls
.
GLOBAL_PROJECT
]
:
repos
+=
projects
[
pro
]
[
backend_section
]
elif
backend_section
in
projects
[
pro
]
and
backend_section
in
projects
[
cls
.
GLOBAL_PROJECT
]
:
repos
+=
projects
[
pro
]
[
backend_section
]
else
:
not_in_unknown_prj
=
[
projects
[
prj
]
for
prj
in
projects
if
prj
!=
cls
.
GLOBAL_PROJECT
]
not_in_unknown_sections
=
list
(
set
(
[
section
for
prj
in
not_in_unknown_prj
for
section
in
list
(
prj
.
keys
(
)
)
]
)
)
if
backend_section
not
in
not_in_unknown_sections
:
repos
+=
projects
[
pro
]
[
backend_section
]
logger
.
debug
(
"List of repos for %s: %s (raw=%s)"
,
backend_section
,
repos
,
raw
)
repos
=
list
(
set
(
repos
)
)
return
repos


def convert_from_eclipse(self, eclipse_projects):
        """ Convert from eclipse projects format to grimoire projects json format """

        projects = {}

        # We need the global project for downloading the full Bugzilla and Gerrit
        projects['unknown'] = {
            "gerrit": ["git.eclipse.org"],
            "bugzilla": ["https://bugs.eclipse.org/bugs/"]
        }

        projects = compose_title(projects, eclipse_projects)
        projects = compose_projects_json(projects, eclipse_projects)

        return projects

def convert_from_eclipse ( self , eclipse_projects ) : projects = { } projects [ 'unknown' ] = { "gerrit" : [ "git.eclipse.org" ] , "bugzilla" : [ "https://bugs.eclipse.org/bugs/" ] } projects = compose_title ( projects , eclipse_projects ) projects = compose_projects_json ( projects , eclipse_projects ) return projects


def set_param(self, section, param, value):
        """ Change a param in the config """
        if section not in self.conf or param not in self.conf[section]:
            logger.error('Config section %s and param %s not exists', section, param)
        else:
            self.conf[section][param] = value


def _add_to_conf(self, new_conf):
        """Add new configuration to self.conf.

        Adds configuration parameters in new_con to self.conf.
        If they already existed in conf, overwrite them.

        :param new_conf: new configuration, to add
        """

        for section in new_conf:
            if section not in self.conf:
                self.conf[section] = new_conf[section]
            else:
                for param in new_conf[section]:
                    self.conf[section][param] = new_conf[section][param]

def _add_to_conf ( self , new_conf ) : for section in new_conf : if section not in self . conf : self . conf [ section ] = new_conf [ section ] else : for param in new_conf [ section ] : self . conf [ section ] [ param ] = new_conf [ section ] [ param ]


def es_version(self, url):
        """Get Elasticsearch version.

        Get the version of Elasticsearch. This is useful because
        Elasticsearch and Kibiter are paired (same major version for 5, 6).

        :param url: Elasticseearch url hosting Kibiter indices
        :returns:   major version, as string
        """

        try:
            res = self.grimoire_con.get(url)
            res.raise_for_status()
            major = res.json()['version']['number'].split(".")[0]
        except Exception:
            logger.error("Error retrieving Elasticsearch version: " + url)
            raise
        return major


def execute_nonstop_tasks(self, tasks_cls):
        """
            Just a wrapper to the execute_batch_tasks method
        """
        self.execute_batch_tasks(tasks_cls,
                                 self.conf['sortinghat']['sleep_for'],
                                 self.conf['general']['min_update_delay'], False)

def execute_nonstop_tasks ( self , tasks_cls ) : self . execute_batch_tasks ( tasks_cls , self . conf [ 'sortinghat' ] [ 'sleep_for' ] , self . conf [ 'general' ] [ 'min_update_delay' ] , False )


def execute_batch_tasks(self, tasks_cls, big_delay=0, small_delay=0, wait_for_threads=True):
        """
        Start a task manager per backend to complete the tasks.

        :param task_cls: list of tasks classes to be executed
        :param big_delay: seconds before global tasks are executed, should be days usually
        :param small_delay: seconds before backend tasks are executed, should be minutes
        :param wait_for_threads: boolean to set when threads are infinite or
                                should be synchronized in a meeting point
        """

        def _split_tasks(tasks_cls):
            """
            we internally distinguish between tasks executed by backend
            and tasks executed with no specific backend. """
            backend_t = []
            global_t = []
            for t in tasks_cls:
                if t.is_backend_task(t):
                    backend_t.append(t)
                else:
                    global_t.append(t)
            return backend_t, global_t

        backend_tasks, global_tasks = _split_tasks(tasks_cls)
        logger.debug('backend_tasks = %s' % (backend_tasks))
        logger.debug('global_tasks = %s' % (global_tasks))

        threads = []

        # stopper won't be set unless wait_for_threads is True
        stopper = threading.Event()

        # launching threads for tasks by backend
        if len(backend_tasks) > 0:
            repos_backend = self._get_repos_by_backend()
            for backend in repos_backend:
                # Start new Threads and add them to the threads list to complete
                t = TasksManager(backend_tasks, backend, stopper, self.config, small_delay)
                threads.append(t)
                t.start()

        # launch thread for global tasks
        if len(global_tasks) > 0:
            # FIXME timer is applied to all global_tasks, does it make sense?
            # All tasks are executed in the same thread sequentially
            gt = TasksManager(global_tasks, "Global tasks", stopper, self.config, big_delay)
            threads.append(gt)
            gt.start()
            if big_delay > 0:
                when = datetime.now() + timedelta(seconds=big_delay)
                when_str = when.strftime('%a, %d %b %Y %H:%M:%S %Z')
                logger.info("%s will be executed on %s" % (global_tasks, when_str))

        if wait_for_threads:
            time.sleep(1)  # Give enough time create and run all threads
            stopper.set()  # All threads must stop in the next iteration

        # Wait for all threads to complete
        for t in threads:
            t.join()

        # Checking for exceptions in threads to log them
        self.__check_queue_for_errors()

        logger.debug("[thread:main] All threads (and their tasks) are finished")

Start a task manager per backend to complete the tasks .


def __execute_initial_load(self):
        """
        Tasks that should be done just one time
        """

        if self.conf['phases']['panels']:
            tasks_cls = [TaskPanels, TaskPanelsMenu]
            self.execute_tasks(tasks_cls)
        if self.conf['phases']['identities']:
            tasks_cls = [TaskInitSortingHat]
            self.execute_tasks(tasks_cls)

        logger.info("Loading projects")
        tasks_cls = [TaskProjects]
        self.execute_tasks(tasks_cls)
        logger.info("Done")

        return

def
__execute_initial_load
(
self
)
:
if
self
.
conf
[
'phases'
]
[
'panels'
]
:
tasks_cls
=
[
TaskPanels
,
TaskPanelsMenu
]
self
.
execute_tasks
(
tasks_cls
)
if
self
.
conf
[
'phases'
]
[
'identities'
]
:
tasks_cls
=
[
TaskInitSortingHat
]
self
.
execute_tasks
(
tasks_cls
)
logger
.
info
(
"Loading projects"
)
tasks_cls
=
[
TaskProjects
]
self
.
execute_tasks
(
tasks_cls
)
logger
.
info
(
"Done"
)
return


def validate_config(self):
        '''
        Validates the provided config to make sure all the required fields are
        there.
        '''
        # first ensure that all the required fields are there
        for key, key_config in self.params_map.items():
            if key_config['required']:
                if key not in self.config:
                    raise ValueError("Invalid Configuration! Required parameter '%s' was not provided to Sultan.")

        # second ensure that the fields that were pased were actually fields that
        # can be used
        for key in self.config.keys():
            if key not in self.params_map:
                raise ValueError("Invalid Configuration! The parameter '%s' provided is not used by Sultan!" % key)

Validates the provided config to make sure all the required fields are there .


def stdout(self):
        """
        Converts stdout string to a list.
        """
        if self._streaming:
            stdout = []
            while not self.__stdout.empty():
                try:
                    line = self.__stdout.get_nowait()
                    stdout.append(line)
                except:
                    pass
        else:
            stdout =  self.__stdout
        return stdout

def stdout ( self ) : if self . _streaming : stdout = [ ] while not self . __stdout . empty ( ) : try : line = self . __stdout . get_nowait ( ) stdout . append ( line ) except : pass else : stdout = self . __stdout return stdout


def stderr(self):
        """
        Converts stderr string to a list.
        """
        if self._streaming:
            stderr = []
            while not self.__stderr.empty():
                try:
                    line = self.__stderr.get_nowait()
                    stderr.append(line)
                except:
                    pass
        else:
            stderr = self.__stderr
        return stderr

def stderr ( self ) : if self . _streaming : stderr = [ ] while not self . __stderr . empty ( ) : try : line = self . __stderr . get_nowait ( ) stderr . append ( line ) except : pass else : stderr = self . __stderr return stderr


def format(self, record):
        """Customize the message format based on the log level."""
        if isinstance(self.fmt, dict):
            self._fmt = self.fmt[record.levelname]
            if sys.version_info > (3, 2):
                # Update self._style because we've changed self._fmt
                # (code based on stdlib's logging.Formatter.__init__())
                if self.style not in logging._STYLES:
                    raise ValueError('Style must be one of: %s' % ','.join(
                        list(logging._STYLES.keys())))
                self._style = logging._STYLES[self.style][0](self._fmt)

        if sys.version_info > (2, 7):
            message = super(LevelFormatter, self).format(record)
        else:
            message = ColoredFormatter.format(self, record)

        return message

def
format
(
self
,
record
)
:
if
isinstance
(
self
.
fmt
,
dict
)
:
self
.
_fmt
=
self
.
fmt
[
record
.
levelname
]
if
sys
.
version_info
>
(
3
,
2
)
:
if
self
.
style
not
in
logging
.
_STYLES
:
raise
ValueError
(
'Style must be one of: %s'
%
','
.
join
(
list
(
logging
.
_STYLES
.
keys
(
)
)
)
)
self
.
_style
=
logging
.
_STYLES
[
self
.
style
]
[
0
]
(
self
.
_fmt
)
if
sys
.
version_info
>
(
2
,
7
)
:
message
=
super
(
LevelFormatter
,
self
)
.
format
(
record
)
else
:
message
=
ColoredFormatter
.
format
(
self
,
record
)
return
message

Customize the message format based on the log level .


def replace_print(fileobj=sys.stderr):
  """Sys.out replacer, by default with stderr.

  Use it like this:
  with replace_print_with(fileobj):
    print "hello"  # writes to the file
  print "done"  # prints to stdout

  Args:
    fileobj: a file object to replace stdout.

  Yields:
    The printer.
  """
  printer = _Printer(fileobj)

  previous_stdout = sys.stdout
  sys.stdout = printer
  try:
    yield printer
  finally:
    sys.stdout = previous_stdout

def replace_print ( fileobj = sys . stderr ) : printer = _Printer ( fileobj ) previous_stdout = sys . stdout sys . stdout = printer try : yield printer finally : sys . stdout = previous_stdout

Sys . out replacer by default with stderr .


def compact_interval_string(value_list):
  """Compact a list of integers into a comma-separated string of intervals.

  Args:
    value_list: A list of sortable integers such as a list of numbers

  Returns:
    A compact string representation, such as "1-5,8,12-15"
  """

  if not value_list:
    return ''

  value_list.sort()

  # Start by simply building up a list of separate contiguous intervals
  interval_list = []
  curr = []
  for val in value_list:
    if curr and (val > curr[-1] + 1):
      interval_list.append((curr[0], curr[-1]))
      curr = [val]
    else:
      curr.append(val)

  if curr:
    interval_list.append((curr[0], curr[-1]))

  # For each interval collapse it down to "first, last" or just "first" if
  # if first == last.
  return ','.join([
      '{}-{}'.format(pair[0], pair[1]) if pair[0] != pair[1] else str(pair[0])
      for pair in interval_list
  ])

def compact_interval_string ( value_list ) : if not value_list : return '' value_list . sort ( ) interval_list = [ ] curr = [ ] for val in value_list : if curr and ( val > curr [ - 1 ] + 1 ) : interval_list . append ( ( curr [ 0 ] , curr [ - 1 ] ) ) curr = [ val ] else : curr . append ( val ) if curr : interval_list . append ( ( curr [ 0 ] , curr [ - 1 ] ) ) return ',' . join ( [ '{}-{}' . format ( pair [ 0 ] , pair [ 1 ] ) if pair [ 0 ] != pair [ 1 ] else str ( pair [ 0 ] ) for pair in interval_list ] )

Compact a list of integers into a comma - separated string of intervals .


def _get_storage_service(credentials):
  """Get a storage client using the provided credentials or defaults."""
  if credentials is None:
    credentials = oauth2client.client.GoogleCredentials.get_application_default(
    )
  return discovery.build('storage', 'v1', credentials=credentials)

def _get_storage_service ( credentials ) : if credentials is None : credentials = oauth2client . client . GoogleCredentials . get_application_default ( ) return discovery . build ( 'storage' , 'v1' , credentials = credentials )

Get a storage client using the provided credentials or defaults .


def _retry_storage_check(exception):
  """Return True if we should retry, False otherwise."""
  now = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
  print_error(
      '%s: Exception %s: %s' % (now, type(exception).__name__, str(exception)))
  return isinstance(exception, oauth2client.client.AccessTokenRefreshError)

def
_retry_storage_check
(
exception
)
:
now
=
datetime
.
now
(
)
.
strftime
(
'%Y-%m-%d %H:%M:%S.%f'
)
print_error
(
'%s: Exception %s: %s'
%
(
now
,
type
(
exception
)
.
__name__
,
str
(
exception
)
)
)
return
isinstance
(
exception
,
oauth2client
.
client
.
AccessTokenRefreshError
)

Return True if we should retry False otherwise .


def _load_file_from_gcs(gcs_file_path, credentials=None):
  """Load context from a text file in gcs.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    The content of the text file as a string.
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = gcs_service.objects().get_media(
      bucket=bucket_name, object=object_name)

  file_handle = io.BytesIO()
  downloader = MediaIoBaseDownload(file_handle, request, chunksize=1024 * 1024)
  done = False
  while not done:
    _, done = _downloader_next_chunk(downloader)
  filevalue = file_handle.getvalue()
  if not isinstance(filevalue, six.string_types):
    filevalue = filevalue.decode()
  return six.StringIO(filevalue)

def _load_file_from_gcs ( gcs_file_path , credentials = None ) : gcs_service = _get_storage_service ( credentials ) bucket_name , object_name = gcs_file_path [ len ( 'gs://' ) : ] . split ( '/' , 1 ) request = gcs_service . objects ( ) . get_media ( bucket = bucket_name , object = object_name ) file_handle = io . BytesIO ( ) downloader = MediaIoBaseDownload ( file_handle , request , chunksize = 1024 * 1024 ) done = False while not done : _ , done = _downloader_next_chunk ( downloader ) filevalue = file_handle . getvalue ( ) if not isinstance ( filevalue , six . string_types ) : filevalue = filevalue . decode ( ) return six . StringIO ( filevalue )


def load_file(file_path, credentials=None):
  """Load a file from either local or gcs.

  Args:
    file_path: The target file path, which should have the prefix 'gs://' if
               to be loaded from gcs.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    A python File object if loading file from local or a StringIO object if
    loading from gcs.
  """
  if file_path.startswith('gs://'):
    return _load_file_from_gcs(file_path, credentials)
  else:
    return open(file_path, 'r')

def load_file ( file_path , credentials = None ) : if file_path . startswith ( 'gs://' ) : return _load_file_from_gcs ( file_path , credentials ) else : return open ( file_path , 'r' )


def _file_exists_in_gcs(gcs_file_path, credentials=None):
  """Check whether the file exists, in GCS.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the file's there.
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = gcs_service.objects().get(
      bucket=bucket_name, object=object_name, projection='noAcl')
  try:
    request.execute()
    return True
  except errors.HttpError:
    return False

def _file_exists_in_gcs ( gcs_file_path , credentials = None ) : gcs_service = _get_storage_service ( credentials ) bucket_name , object_name = gcs_file_path [ len ( 'gs://' ) : ] . split ( '/' , 1 ) request = gcs_service . objects ( ) . get ( bucket = bucket_name , object = object_name , projection = 'noAcl' ) try : request . execute ( ) return True except errors . HttpError : return False


def file_exists(file_path, credentials=None):
  """Check whether the file exists, on local disk or GCS.

  Args:
    file_path: The target file path; should have the 'gs://' prefix if in gcs.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the file's there.
  """
  if file_path.startswith('gs://'):
    return _file_exists_in_gcs(file_path, credentials)
  else:
    return os.path.isfile(file_path)

def file_exists ( file_path , credentials = None ) : if file_path . startswith ( 'gs://' ) : return _file_exists_in_gcs ( file_path , credentials ) else : return os . path . isfile ( file_path )

Check whether the file exists on local disk or GCS .


def _prefix_exists_in_gcs(gcs_prefix, credentials=None):
  """Check whether there is a GCS object whose name starts with the prefix.

  Since GCS doesn't actually have folders, this is how we check instead.

  Args:
    gcs_prefix: The path; should start with 'gs://'.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the prefix matches at least one object in GCS.

  Raises:
    errors.HttpError: if it can't talk to the server
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, prefix = gcs_prefix[len('gs://'):].split('/', 1)
  # documentation in
  # https://cloud.google.com/storage/docs/json_api/v1/objects/list
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix, maxResults=1)
  response = request.execute()
  return response.get('items', None)

def _prefix_exists_in_gcs ( gcs_prefix , credentials = None ) : gcs_service = _get_storage_service ( credentials ) bucket_name , prefix = gcs_prefix [ len ( 'gs://' ) : ] . split ( '/' , 1 ) request = gcs_service . objects ( ) . list ( bucket = bucket_name , prefix = prefix , maxResults = 1 ) response = request . execute ( ) return response . get ( 'items' , None )

Check whether there is a GCS object whose name starts with the prefix .


def simple_pattern_exists_in_gcs(file_pattern, credentials=None):
  """True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
  """
  if '*' not in file_pattern:
    return _file_exists_in_gcs(file_pattern, credentials)
  if not file_pattern.startswith('gs://'):
    raise ValueError('file name must start with gs://')
  gcs_service = _get_storage_service(credentials)
  bucket_name, prefix = file_pattern[len('gs://'):].split('/', 1)
  if '*' in bucket_name:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in prefix
  prefix_no_wildcard = prefix[:prefix.index('*')]
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix_no_wildcard)
  response = request.execute()
  if 'items' not in response:
    return False
  items_list = [i['name'] for i in response['items']]
  return any(fnmatch.fnmatch(i, prefix) for i in items_list)

True iff an object exists matching the input GCS pattern .


def outputs_are_present(outputs):
  """True if each output contains at least one file or no output specified."""
  # outputs are OutputFileParam (see param_util.py)

  # If outputs contain a pattern, then there is no way for `dsub` to verify
  # that *all* output is present. The best that `dsub` can do is to verify
  # that *some* output was created for each such parameter.
  for o in outputs:
    if not o.value:
      continue
    if o.recursive:
      if not folder_exists(o.value):
        return False
    else:
      if not simple_pattern_exists_in_gcs(o.value):
        return False
  return True

def outputs_are_present ( outputs ) : for o in outputs : if not o . value : continue if o . recursive : if not folder_exists ( o . value ) : return False else : if not simple_pattern_exists_in_gcs ( o . value ) : return False return True

True if each output contains at least one file or no output specified .


def _build_pipeline_input_file_param(cls, var_name, docker_path):
    """Return a dict object representing a pipeline input argument."""

    # If the filename contains a wildcard, then the target Docker path must
    # be a directory in order to ensure consistency whether the source pattern
    # contains 1 or multiple files.
    #
    # In that case, we set the docker_path to explicitly have a trailing slash
    # (for the Pipelines API "gsutil cp" handling, and then override the
    # associated var_name environment variable in the generated Docker command.

    path, filename = os.path.split(docker_path)
    if '*' in filename:
      return cls._build_pipeline_file_param(var_name, path + '/')
    else:
      return cls._build_pipeline_file_param(var_name, docker_path)

def _build_pipeline_input_file_param ( cls , var_name , docker_path ) : path , filename = os . path . split ( docker_path ) if '*' in filename : return cls . _build_pipeline_file_param ( var_name , path + '/' ) else : return cls . _build_pipeline_file_param ( var_name , docker_path )

Return a dict object representing a pipeline input argument .


def _build_pipeline_docker_command(cls, script_name, inputs, outputs, envs):
    """Return a multi-line string of the full pipeline docker command."""

    # We upload the user script as an environment argument
    # and write it to SCRIPT_DIR (preserving its local file name).
    #
    # The docker_command:
    # * writes the script body to a file
    # * installs gcloud if there are recursive copies to do
    # * sets environment variables for inputs with wildcards
    # * sets environment variables for recursive input directories
    # * recursively copies input directories
    # * creates output directories
    # * sets environment variables for recursive output directories
    # * sets the DATA_ROOT environment variable to /mnt/data
    # * sets the working directory to ${DATA_ROOT}
    # * executes the user script
    # * recursively copies output directories
    recursive_input_dirs = [
        var for var in inputs if var.recursive and var.value
    ]
    recursive_output_dirs = [
        var for var in outputs if var.recursive and var.value
    ]

    install_cloud_sdk = ''
    if recursive_input_dirs or recursive_output_dirs:
      install_cloud_sdk = INSTALL_CLOUD_SDK

    export_input_dirs = ''
    copy_input_dirs = ''
    if recursive_input_dirs:
      export_input_dirs = providers_util.build_recursive_localize_env(
          providers_util.DATA_MOUNT_POINT, inputs)
      copy_input_dirs = providers_util.build_recursive_localize_command(
          providers_util.DATA_MOUNT_POINT, inputs, job_model.P_GCS)

    export_output_dirs = ''
    copy_output_dirs = ''
    if recursive_output_dirs:
      export_output_dirs = providers_util.build_recursive_gcs_delocalize_env(
          providers_util.DATA_MOUNT_POINT, outputs)
      copy_output_dirs = providers_util.build_recursive_delocalize_command(
          providers_util.DATA_MOUNT_POINT, outputs, job_model.P_GCS)

    docker_paths = [
        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
        for var in outputs
        if var.value
    ]

    mkdirs = '\n'.join([
        'mkdir -p {0}/{1}'.format(providers_util.DATA_MOUNT_POINT, path)
        for path in docker_paths
    ])

    inputs_with_wildcards = [
        var for var in inputs if not var.recursive and var.docker_path and
        '*' in os.path.basename(var.docker_path)
    ]
    export_inputs_with_wildcards = '\n'.join([
        'export {0}="{1}/{2}"'.format(var.name, providers_util.DATA_MOUNT_POINT,
                                      var.docker_path)
        for var in inputs_with_wildcards
    ])

    export_empty_envs = '\n'.join([
        'export {0}=""'.format(var.name)
        for var in envs | inputs | outputs
        if not var.value
    ])

    return DOCKER_COMMAND.format(
        mk_runtime_dirs=MK_RUNTIME_DIRS_COMMAND,
        script_path='%s/%s' % (providers_util.SCRIPT_DIR, script_name),
        install_cloud_sdk=install_cloud_sdk,
        export_inputs_with_wildcards=export_inputs_with_wildcards,
        export_input_dirs=export_input_dirs,
        copy_input_dirs=copy_input_dirs,
        mk_output_dirs=mkdirs,
        export_output_dirs=export_output_dirs,
        export_empty_envs=export_empty_envs,
        tmpdir=providers_util.TMP_DIR,
        working_dir=providers_util.WORKING_DIR,
        copy_output_dirs=copy_output_dirs)

def
_build_pipeline_docker_command
(
cls
,
script_name
,
inputs
,
outputs
,
envs
)
:
recursive_input_dirs
=
[
var
for
var
in
inputs
if
var
.
recursive
and
var
.
value
]
recursive_output_dirs
=
[
var
for
var
in
outputs
if
var
.
recursive
and
var
.
value
]
install_cloud_sdk
=
''
if
recursive_input_dirs
or
recursive_output_dirs
:
install_cloud_sdk
=
INSTALL_CLOUD_SDK
export_input_dirs
=
''
copy_input_dirs
=
''
if
recursive_input_dirs
:
export_input_dirs
=
providers_util
.
build_recursive_localize_env
(
providers_util
.
DATA_MOUNT_POINT
,
inputs
)
copy_input_dirs
=
providers_util
.
build_recursive_localize_command
(
providers_util
.
DATA_MOUNT_POINT
,
inputs
,
job_model
.
P_GCS
)
export_output_dirs
=
''
copy_output_dirs
=
''
if
recursive_output_dirs
:
export_output_dirs
=
providers_util
.
build_recursive_gcs_delocalize_env
(
providers_util
.
DATA_MOUNT_POINT
,
outputs
)
copy_output_dirs
=
providers_util
.
build_recursive_delocalize_command
(
providers_util
.
DATA_MOUNT_POINT
,
outputs
,
job_model
.
P_GCS
)
docker_paths
=
[
var
.
docker_path
if
var
.
recursive
else
os
.
path
.
dirname
(
var
.
docker_path
)
for
var
in
outputs
if
var
.
value
]
mkdirs
=
'\n'
.
join
(
[
'mkdir -p {0}/{1}'
.
format
(
providers_util
.
DATA_MOUNT_POINT
,
path
)
for
path
in
docker_paths
]
)
inputs_with_wildcards
=
[
var
for
var
in
inputs
if
not
var
.
recursive
and
var
.
docker_path
and
'*'
in
os
.
path
.
basename
(
var
.
docker_path
)
]
export_inputs_with_wildcards
=
'\n'
.
join
(
[
'export {0}="{1}/{2}"'
.
format
(
var
.
name
,
providers_util
.
DATA_MOUNT_POINT
,
var
.
docker_path
)
for
var
in
inputs_with_wildcards
]
)
export_empty_envs
=
'\n'
.
join
(
[
'export {0}=""'
.
format
(
var
.
name
)
for
var
in
envs
|
inputs
|
outputs
if
not
var
.
value
]
)
return
DOCKER_COMMAND
.
format
(
mk_runtime_dirs
=
MK_RUNTIME_DIRS_COMMAND
,
script_path
=
'%s/%s'
%
(
providers_util
.
SCRIPT_DIR
,
script_name
)
,
install_cloud_sdk
=
install_cloud_sdk
,
export_inputs_with_wildcards
=
export_inputs_with_wildcards
,
export_input_dirs
=
export_input_dirs
,
copy_input_dirs
=
copy_input_dirs
,
mk_output_dirs
=
mkdirs
,
export_output_dirs
=
export_output_dirs
,
export_empty_envs
=
export_empty_envs
,
tmpdir
=
providers_util
.
TMP_DIR
,
working_dir
=
providers_util
.
WORKING_DIR
,
copy_output_dirs
=
copy_output_dirs
)

Return a multi - line string of the full pipeline docker command .


def build_pipeline(cls, project, zones, min_cores, min_ram, disk_size,
                     boot_disk_size, preemptible, accelerator_type,
                     accelerator_count, image, script_name, envs, inputs,
                     outputs, pipeline_name):
    """Builds a pipeline configuration for execution.

    Args:
      project: string name of project.
      zones: list of zone names for jobs to be run at.
      min_cores: int number of CPU cores required per job.
      min_ram: int GB of RAM required per job.
      disk_size: int GB of disk to attach under /mnt/data.
      boot_disk_size: int GB of disk for boot.
      preemptible: use a preemptible VM for the job
      accelerator_type: string GCE defined accelerator type.
      accelerator_count: int number of accelerators of the specified type to
        attach.
      image: string Docker image name in which to run.
      script_name: file name of the script to run.
      envs: list of EnvParam objects specifying environment variables to set
        within each job.
      inputs: list of FileParam objects specifying input variables to set
        within each job.
      outputs: list of FileParam objects specifying output variables to set
        within each job.
      pipeline_name: string name of pipeline.

    Returns:
      A nested dictionary with one entry under the key ephemeralPipeline
      containing the pipeline configuration.
    """
    if min_cores is None:
      min_cores = job_model.DEFAULT_MIN_CORES
    if min_ram is None:
      min_ram = job_model.DEFAULT_MIN_RAM
    if disk_size is None:
      disk_size = job_model.DEFAULT_DISK_SIZE
    if boot_disk_size is None:
      boot_disk_size = job_model.DEFAULT_BOOT_DISK_SIZE
    if preemptible is None:
      preemptible = job_model.DEFAULT_PREEMPTIBLE

    # Format the docker command
    docker_command = cls._build_pipeline_docker_command(script_name, inputs,
                                                        outputs, envs)

    # Pipelines inputParameters can be both simple name/value pairs which get
    # set as environment variables, as well as input file paths which the
    # Pipelines controller will automatically localize to the Pipeline VM.

    # In the ephemeralPipeline object, the inputParameters are only defined;
    # the values are passed in the pipelineArgs.

    # Pipelines outputParameters are only output file paths, which the
    # Pipelines controller can automatically de-localize after the docker
    # command completes.

    # The Pipelines API does not support recursive copy of file parameters,
    # so it is implemented within the dsub-generated pipeline.
    # Any inputs or outputs marked as "recursive" are completely omitted here;
    # their environment variables will be set in the docker command, and
    # recursive copy code will be generated there as well.

    # The Pipelines API does not accept empty environment variables. Set them to
    # empty in DOCKER_COMMAND instead.
    input_envs = [{
        'name': SCRIPT_VARNAME
    }] + [{
        'name': env.name
    } for env in envs if env.value]

    input_files = [
        cls._build_pipeline_input_file_param(var.name, var.docker_path)
        for var in inputs
        if not var.recursive and var.value
    ]

    # Outputs are an array of file parameters
    output_files = [
        cls._build_pipeline_file_param(var.name, var.docker_path)
        for var in outputs
        if not var.recursive and var.value
    ]

    # The ephemeralPipeline provides the template for the pipeline.
    # pyformat: disable
    return {
        'ephemeralPipeline': {
            'projectId': project,
            'name': pipeline_name,

            # Define the resources needed for this pipeline.
            'resources': {
                'minimumCpuCores': min_cores,
                'minimumRamGb': min_ram,
                'bootDiskSizeGb': boot_disk_size,
                'preemptible': preemptible,
                'zones': google_base.get_zones(zones),
                'acceleratorType': accelerator_type,
                'acceleratorCount': accelerator_count,

                # Create a data disk that is attached to the VM and destroyed
                # when the pipeline terminates.
                'disks': [{
                    'name': 'datadisk',
                    'autoDelete': True,
                    'sizeGb': disk_size,
                    'mountPoint': providers_util.DATA_MOUNT_POINT,
                }],
            },

            'inputParameters': input_envs + input_files,
            'outputParameters': output_files,

            'docker': {
                'imageName': image,
                'cmd': docker_command,
            }
        }
    }

def build_pipeline ( cls , project , zones , min_cores , min_ram , disk_size , boot_disk_size , preemptible , accelerator_type , accelerator_count , image , script_name , envs , inputs , outputs , pipeline_name ) : if min_cores is None : min_cores = job_model . DEFAULT_MIN_CORES if min_ram is None : min_ram = job_model . DEFAULT_MIN_RAM if disk_size is None : disk_size = job_model . DEFAULT_DISK_SIZE if boot_disk_size is None : boot_disk_size = job_model . DEFAULT_BOOT_DISK_SIZE if preemptible is None : preemptible = job_model . DEFAULT_PREEMPTIBLE docker_command = cls . _build_pipeline_docker_command ( script_name , inputs , outputs , envs ) input_envs = [ { 'name' : SCRIPT_VARNAME } ] + [ { 'name' : env . name } for env in envs if env . value ] input_files = [ cls . _build_pipeline_input_file_param ( var . name , var . docker_path ) for var in inputs if not var . recursive and var . value ] output_files = [ cls . _build_pipeline_file_param ( var . name , var . docker_path ) for var in outputs if not var . recursive and var . value ] return { 'ephemeralPipeline' : { 'projectId' : project , 'name' : pipeline_name , 'resources' : { 'minimumCpuCores' : min_cores , 'minimumRamGb' : min_ram , 'bootDiskSizeGb' : boot_disk_size , 'preemptible' : preemptible , 'zones' : google_base . get_zones ( zones ) , 'acceleratorType' : accelerator_type , 'acceleratorCount' : accelerator_count , 'disks' : [ { 'name' : 'datadisk' , 'autoDelete' : True , 'sizeGb' : disk_size , 'mountPoint' : providers_util . DATA_MOUNT_POINT , } ] , } , 'inputParameters' : input_envs + input_files , 'outputParameters' : output_files , 'docker' : { 'imageName' : image , 'cmd' : docker_command , } } }

Builds a pipeline configuration for execution .


def build_pipeline_args(cls, project, script, job_params, task_params,
                          reserved_labels, preemptible, logging_uri, scopes,
                          keep_alive):
    """Builds pipeline args for execution.

    Args:
      project: string name of project.
      script: Body of the script to execute.
      job_params: dictionary of values for labels, envs, inputs, and outputs
          for this job.
      task_params: dictionary of values for labels, envs, inputs, and outputs
          for this task.
      reserved_labels: dictionary of reserved labels (e.g. task-id,
          task-attempt)
      preemptible: use a preemptible VM for the job
      logging_uri: path for job logging output.
      scopes: list of scope.
      keep_alive: Seconds to keep VM alive on failure

    Returns:
      A nested dictionary with one entry under the key pipelineArgs containing
      the pipeline arguments.
    """
    # For the Pipelines API, envs and file inputs are all "inputs".
    inputs = {}
    inputs.update({SCRIPT_VARNAME: script})
    inputs.update({
        var.name: var.value
        for var in job_params['envs'] | task_params['envs']
        if var.value
    })
    inputs.update({
        var.name: var.uri
        for var in job_params['inputs'] | task_params['inputs']
        if not var.recursive and var.value
    })

    # Remove wildcard references for non-recursive output. When the pipelines
    # controller generates a delocalize call, it must point to a bare directory
    # for patterns. The output param OUTFILE=gs://bucket/path/*.bam should
    # delocalize with a call similar to:
    #   gsutil cp /mnt/data/output/gs/bucket/path/*.bam gs://bucket/path/
    outputs = {}
    for var in job_params['outputs'] | task_params['outputs']:
      if var.recursive or not var.value:
        continue
      if '*' in var.uri.basename:
        outputs[var.name] = var.uri.path
      else:
        outputs[var.name] = var.uri

    labels = {}
    labels.update({
        label.name: label.value if label.value else ''
        for label in (reserved_labels | job_params['labels']
                      | task_params['labels'])
    })

    # pyformat: disable
    args = {
        'pipelineArgs': {
            'projectId': project,
            'resources': {
                'preemptible': preemptible,
            },
            'inputs': inputs,
            'outputs': outputs,
            'labels': labels,
            'serviceAccount': {
                'email': 'default',
                'scopes': scopes,
            },
            # Pass the user-specified GCS destination for pipeline logging.
            'logging': {
                'gcsPath': logging_uri
            },
        }
    }
    # pyformat: enable

    if keep_alive:
      args['pipelineArgs'][
          'keep_vm_alive_on_failure_duration'] = '%ss' % keep_alive

    return args

def build_pipeline_args ( cls , project , script , job_params , task_params , reserved_labels , preemptible , logging_uri , scopes , keep_alive ) : inputs = { } inputs . update ( { SCRIPT_VARNAME : script } ) inputs . update ( { var . name : var . value for var in job_params [ 'envs' ] | task_params [ 'envs' ] if var . value } ) inputs . update ( { var . name : var . uri for var in job_params [ 'inputs' ] | task_params [ 'inputs' ] if not var . recursive and var . value } ) outputs = { } for var in job_params [ 'outputs' ] | task_params [ 'outputs' ] : if var . recursive or not var . value : continue if '*' in var . uri . basename : outputs [ var . name ] = var . uri . path else : outputs [ var . name ] = var . uri labels = { } labels . update ( { label . name : label . value if label . value else '' for label in ( reserved_labels | job_params [ 'labels' ] | task_params [ 'labels' ] ) } ) args = { 'pipelineArgs' : { 'projectId' : project , 'resources' : { 'preemptible' : preemptible , } , 'inputs' : inputs , 'outputs' : outputs , 'labels' : labels , 'serviceAccount' : { 'email' : 'default' , 'scopes' : scopes , } , 'logging' : { 'gcsPath' : logging_uri } , } } if keep_alive : args [ 'pipelineArgs' ] [ 'keep_vm_alive_on_failure_duration' ] = '%ss' % keep_alive return args


def _datetime_to_utc_int(date):
    """Convert the integer UTC time value into a local datetime."""
    if date is None:
      return None

    # Convert localized datetime to a UTC integer
    epoch = dsub_util.replace_timezone(datetime.utcfromtimestamp(0), pytz.utc)
    return (date - epoch).total_seconds()

def _datetime_to_utc_int ( date ) : if date is None : return None epoch = dsub_util . replace_timezone ( datetime . utcfromtimestamp ( 0 ) , pytz . utc ) return ( date - epoch ) . total_seconds ( )

Convert the integer UTC time value into a local datetime .


def _build_pipeline_request(self, task_view):
    """Returns a Pipeline objects for the job."""
    job_metadata = task_view.job_metadata
    job_params = task_view.job_params
    job_resources = task_view.job_resources
    task_metadata = task_view.task_descriptors[0].task_metadata
    task_params = task_view.task_descriptors[0].task_params
    task_resources = task_view.task_descriptors[0].task_resources

    script = task_view.job_metadata['script']

    reserved_labels = google_base.build_pipeline_labels(
        job_metadata, task_metadata, task_id_pattern='task-%d')

    # Build the ephemeralPipeline for this job.
    # The ephemeralPipeline definition changes for each job because file
    # parameters localCopy.path changes based on the remote_uri.
    pipeline = _Pipelines.build_pipeline(
        project=self._project,
        zones=job_resources.zones,
        min_cores=job_resources.min_cores,
        min_ram=job_resources.min_ram,
        disk_size=job_resources.disk_size,
        boot_disk_size=job_resources.boot_disk_size,
        preemptible=job_resources.preemptible,
        accelerator_type=job_resources.accelerator_type,
        accelerator_count=job_resources.accelerator_count,
        image=job_resources.image,
        script_name=script.name,
        envs=job_params['envs'] | task_params['envs'],
        inputs=job_params['inputs'] | task_params['inputs'],
        outputs=job_params['outputs'] | task_params['outputs'],
        pipeline_name=job_metadata['pipeline-name'])

    # Build the pipelineArgs for this job.
    logging_uri = task_resources.logging_path.uri
    scopes = job_resources.scopes or google_base.DEFAULT_SCOPES
    pipeline.update(
        _Pipelines.build_pipeline_args(self._project, script.value, job_params,
                                       task_params, reserved_labels,
                                       job_resources.preemptible, logging_uri,
                                       scopes, job_resources.keep_alive))

    return pipeline

Returns a Pipeline objects for the job .


def delete_jobs(self,
                  user_ids,
                  job_ids,
                  task_ids,
                  labels,
                  create_time_min=None,
                  create_time_max=None):
    """Kills the operations associated with the specified job or job.task.

    Args:
      user_ids: List of user ids who "own" the job(s) to cancel.
      job_ids: List of job_ids to cancel.
      task_ids: List of task-ids to cancel.
      labels: List of LabelParam, each must match the job(s) to be canceled.
      create_time_min: a timezone-aware datetime value for the earliest create
                       time of a task, inclusive.
      create_time_max: a timezone-aware datetime value for the most recent
                       create time of a task, inclusive.

    Returns:
      A list of tasks canceled and a list of error messages.
    """
    # Look up the job(s)
    tasks = list(
        self.lookup_job_tasks(
            {'RUNNING'},
            user_ids=user_ids,
            job_ids=job_ids,
            task_ids=task_ids,
            labels=labels,
            create_time_min=create_time_min,
            create_time_max=create_time_max))

    print('Found %d tasks to delete.' % len(tasks))

    return google_base.cancel(self._service.new_batch_http_request,
                              self._service.operations().cancel, tasks)

def
delete_jobs
(
self
,
user_ids
,
job_ids
,
task_ids
,
labels
,
create_time_min
=
,
create_time_max
=
)
:
tasks
=
list
(
self
.
lookup_job_tasks
(
{
'RUNNING'
}
,
user_ids
=
user_ids
,
job_ids
=
job_ids
,
task_ids
=
task_ids
,
labels
=
labels
,
create_time_min
=
create_time_min
,
create_time_max
=
create_time_max
)
)
print
(
'Found %d tasks to delete.'
%
len
(
tasks
)
)
return
google_base
.
cancel
(
self
.
_service
.
new_batch_http_request
,
self
.
_service
.
operations
(
)
.
cancel
,
tasks
)

Kills the operations associated with the specified job or job . task .


def _operation_status_message(self):
    """Returns the most relevant status string and last updated date string.

    This string is meant for display only.

    Returns:
      A printable status string and date string.
    """
    metadata = self._op['metadata']
    if not self._op['done']:
      if 'events' in metadata and metadata['events']:
        # Get the last event
        last_event = metadata['events'][-1]

        msg = last_event['description']
        ds = last_event['startTime']
      else:
        msg = 'Pending'
        ds = metadata['createTime']
    else:
      ds = metadata['endTime']

      if 'error' in self._op:
        msg = self._op['error']['message']
      else:
        msg = 'Success'

    return (msg, google_base.parse_rfc3339_utc_string(ds))

def _operation_status_message ( self ) : metadata = self . _op [ 'metadata' ] if not self . _op [ 'done' ] : if 'events' in metadata and metadata [ 'events' ] : last_event = metadata [ 'events' ] [ - 1 ] msg = last_event [ 'description' ] ds = last_event [ 'startTime' ] else : msg = 'Pending' ds = metadata [ 'createTime' ] else : ds = metadata [ 'endTime' ] if 'error' in self . _op : msg = self . _op [ 'error' ] [ 'message' ] else : msg = 'Success' return ( msg , google_base . parse_rfc3339_utc_string ( ds ) )

Returns the most relevant status string and last updated date string .


def _get_operation_input_field_values(self, metadata, file_input):
    """Returns a dictionary of envs or file inputs for an operation.

    Args:
      metadata: operation metadata field
      file_input: True to return a dict of file inputs, False to return envs.

    Returns:
      A dictionary of input field name value pairs
    """

    # To determine input parameter type, we iterate through the
    # pipeline inputParameters.
    # The values come from the pipelineArgs inputs.
    input_args = metadata['request']['ephemeralPipeline']['inputParameters']
    vals_dict = metadata['request']['pipelineArgs']['inputs']

    # Get the names for files or envs
    names = [
        arg['name'] for arg in input_args if ('localCopy' in arg) == file_input
    ]

    # Build the return dict
    return {name: vals_dict[name] for name in names if name in vals_dict}

def _get_operation_input_field_values ( self , metadata , file_input ) : input_args = metadata [ 'request' ] [ 'ephemeralPipeline' ] [ 'inputParameters' ] vals_dict = metadata [ 'request' ] [ 'pipelineArgs' ] [ 'inputs' ] names = [ arg [ 'name' ] for arg in input_args if ( 'localCopy' in arg ) == file_input ] return { name : vals_dict [ name ] for name in names if name in vals_dict }

Returns a dictionary of envs or file inputs for an operation .


def _format_task_name(job_id, task_id, task_attempt):
  """Create a task name from a job-id, task-id, and task-attempt.

  Task names are used internally by dsub as well as by the docker task runner.
  The name is formatted as "<job-id>.<task-id>[.task-attempt]". Task names
  follow formatting conventions allowing them to be safely used as a docker
  name.

  Args:
    job_id: (str) the job ID.
    task_id: (str) the task ID.
    task_attempt: (int) the task attempt.

  Returns:
    a task name string.
  """
  docker_name = '%s.%s' % (job_id, 'task' if task_id is None else task_id)

  if task_attempt is not None:
    docker_name += '.' + str(task_attempt)

  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  # So 1) prefix it with "dsub-" and 2) change all invalid characters to "-".
  return 'dsub-{}'.format(_convert_suffix_to_docker_chars(docker_name))

def _format_task_name ( job_id , task_id , task_attempt ) : docker_name = '%s.%s' % ( job_id , 'task' if task_id is None else task_id ) if task_attempt is not None : docker_name += '.' + str ( task_attempt ) return 'dsub-{}' . format ( _convert_suffix_to_docker_chars ( docker_name ) )

Create a task name from a job - id task - id and task - attempt .


def _convert_suffix_to_docker_chars(suffix):
  """Rewrite string so that all characters are valid in a docker name suffix."""
  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  accepted_characters = string.ascii_letters + string.digits + '_.-'

  def label_char_transform(char):
    if char in accepted_characters:
      return char
    return '-'

  return ''.join(label_char_transform(c) for c in suffix)

def _convert_suffix_to_docker_chars ( suffix ) : accepted_characters = string . ascii_letters + string . digits + '_.-' def label_char_transform ( char ) : if char in accepted_characters : return char return '-' return '' . join ( label_char_transform ( c ) for c in suffix )

Rewrite string so that all characters are valid in a docker name suffix .


def _task_sort_function(task):
  """Return a tuple for sorting 'most recent first'."""
  return (task.get_field('create-time'), int(task.get_field('task-id', 0)),
          int(task.get_field('task-attempt', 0)))

def _task_sort_function ( task ) : return ( task . get_field ( 'create-time' ) , int ( task . get_field ( 'task-id' , 0 ) ) , int ( task . get_field ( 'task-attempt' , 0 ) ) )

Return a tuple for sorting most recent first .


def _datetime_in_range(self, dt, dt_min=None, dt_max=None):
    """Determine if the provided time is within the range, inclusive."""
    # The pipelines API stores operation create-time with second granularity.
    # We mimic this behavior in the local provider by truncating to seconds.
    dt = dt.replace(microsecond=0)
    if dt_min:
      dt_min = dt_min.replace(microsecond=0)
    else:
      dt_min = dsub_util.replace_timezone(datetime.datetime.min, pytz.utc)
    if dt_max:
      dt_max = dt_max.replace(microsecond=0)
    else:
      dt_max = dsub_util.replace_timezone(datetime.datetime.max, pytz.utc)

    return dt_min <= dt <= dt_max

def _datetime_in_range ( self , dt , dt_min = None , dt_max = None ) : dt = dt . replace ( microsecond = 0 ) if dt_min : dt_min = dt_min . replace ( microsecond = 0 ) else : dt_min = dsub_util . replace_timezone ( datetime . datetime . min , pytz . utc ) if dt_max : dt_max = dt_max . replace ( microsecond = 0 ) else : dt_max = dsub_util . replace_timezone ( datetime . datetime . max , pytz . utc ) return dt_min <= dt <= dt_max

Determine if the provided time is within the range inclusive .


def _get_task_from_task_dir(self, job_id, user_id, task_id, task_attempt):
    """Return a Task object with this task's info."""

    # We need to be very careful about how we read and interpret the contents
    # of the task directory. The directory could be changing because a new
    # task is being created. The directory could be changing because a task
    # is ending.
    #
    # If the meta.yaml does not exist, the task does not yet exist.
    # If the meta.yaml exists, it means the task is scheduled. It does not mean
    # it is yet running.
    # If the task.pid file exists, it means that the runner.sh was started.

    task_dir = self._task_directory(job_id, task_id, task_attempt)

    job_descriptor = self._read_task_metadata(task_dir)
    if not job_descriptor:
      return None

    # If we read up an old task, the user-id will not be in the job_descriptor.
    if not job_descriptor.job_metadata.get('user-id'):
      job_descriptor.job_metadata['user-id'] = user_id

    # Get the pid of the runner
    pid = -1
    try:
      with open(os.path.join(task_dir, 'task.pid'), 'r') as f:
        pid = int(f.readline().strip())
    except (IOError, OSError):
      pass

    # Get the script contents
    script = None
    script_name = job_descriptor.job_metadata.get('script-name')
    if script_name:
      script = self._read_script(task_dir, script_name)

    # Read the files written by the runner.sh.
    # For new tasks, these may not have been written yet.
    end_time = self._get_end_time_from_task_dir(task_dir)
    last_update = self._get_last_update_time_from_task_dir(task_dir)
    events = self._get_events_from_task_dir(task_dir)
    status = self._get_status_from_task_dir(task_dir)
    log_detail = self._get_log_detail_from_task_dir(task_dir)

    # If the status file is not yet written, then mark the task as pending
    if not status:
      status = 'RUNNING'
      log_detail = ['Pending']

    return LocalTask(
        task_status=status,
        events=events,
        log_detail=log_detail,
        job_descriptor=job_descriptor,
        end_time=end_time,
        last_update=last_update,
        pid=pid,
        script=script)

def _get_task_from_task_dir ( self , job_id , user_id , task_id , task_attempt ) : task_dir = self . _task_directory ( job_id , task_id , task_attempt ) job_descriptor = self . _read_task_metadata ( task_dir ) if not job_descriptor : return None if not job_descriptor . job_metadata . get ( 'user-id' ) : job_descriptor . job_metadata [ 'user-id' ] = user_id pid = - 1 try : with open ( os . path . join ( task_dir , 'task.pid' ) , 'r' ) as f : pid = int ( f . readline ( ) . strip ( ) ) except ( IOError , OSError ) : pass script = None script_name = job_descriptor . job_metadata . get ( 'script-name' ) if script_name : script = self . _read_script ( task_dir , script_name ) end_time = self . _get_end_time_from_task_dir ( task_dir ) last_update = self . _get_last_update_time_from_task_dir ( task_dir ) events = self . _get_events_from_task_dir ( task_dir ) status = self . _get_status_from_task_dir ( task_dir ) log_detail = self . _get_log_detail_from_task_dir ( task_dir ) if not status : status = 'RUNNING' log_detail = [ 'Pending' ] return LocalTask ( task_status = status , events = events , log_detail = log_detail , job_descriptor = job_descriptor , end_time = end_time , last_update = last_update , pid = pid , script = script )

Return a Task object with this task s info .


def _delocalize_logging_command(self, logging_path, user_project):
    """Returns a command to delocalize logs.

    Args:
      logging_path: location of log files.
      user_project: name of the project to be billed for the request.

    Returns:
      eg. 'gs://bucket/path/myfile' or 'gs://bucket/script-foobar-12'
    """

    # Get the logging prefix (everything up to ".log")
    logging_prefix = os.path.splitext(logging_path.uri)[0]

    # Set the provider-specific mkdir and file copy commands
    if logging_path.file_provider == job_model.P_LOCAL:
      mkdir_cmd = 'mkdir -p "%s"\n' % os.path.dirname(logging_prefix)
      cp_cmd = 'cp'
    elif logging_path.file_provider == job_model.P_GCS:
      mkdir_cmd = ''
      if user_project:
        cp_cmd = 'gsutil -u {} -mq cp'.format(user_project)
      else:
        cp_cmd = 'gsutil -mq cp'
    else:
      assert False

    # Construct the copy command
    copy_logs_cmd = textwrap.dedent("""\
      local cp_cmd="{cp_cmd}"
      local prefix="{prefix}"
    """).format(
        cp_cmd=cp_cmd, prefix=logging_prefix)

    # Build up the command
    body = textwrap.dedent("""\
      {mkdir_cmd}
      {copy_logs_cmd}
    """).format(
        mkdir_cmd=mkdir_cmd, copy_logs_cmd=copy_logs_cmd)

    return body

def
_delocalize_logging_command
(
self
,
logging_path
,
user_project
)
:
logging_prefix
=
os
.
path
.
splitext
(
logging_path
.
uri
)
[
0
]
if
logging_path
.
file_provider
==
job_model
.
P_LOCAL
:
mkdir_cmd
=
'mkdir -p "%s"\n'
%
os
.
path
.
dirname
(
logging_prefix
)
cp_cmd
=
'cp'
elif
logging_path
.
file_provider
==
job_model
.
P_GCS
:
mkdir_cmd
=
''
if
user_project
:
cp_cmd
=
'gsutil -u {} -mq cp'
.
format
(
user_project
)
else
:
cp_cmd
=
'gsutil -mq cp'
else
:
assert
False
copy_logs_cmd
=
textwrap
.
dedent
(
)
.
format
(
cp_cmd
=
cp_cmd
,
prefix
=
logging_prefix
)
body
=
textwrap
.
dedent
(
)
.
format
(
mkdir_cmd
=
mkdir_cmd
,
copy_logs_cmd
=
copy_logs_cmd
)
return
body


def _task_directory(self, job_id, task_id, task_attempt):
    """The local dir for staging files for that particular task."""
    dir_name = 'task' if task_id is None else str(task_id)
    if task_attempt:
      dir_name = '%s.%s' % (dir_name, task_attempt)
    return self._provider_root() + '/' + job_id + '/' + dir_name

def _task_directory ( self , job_id , task_id , task_attempt ) : dir_name = 'task' if task_id is None else str ( task_id ) if task_attempt : dir_name = '%s.%s' % ( dir_name , task_attempt ) return self . _provider_root ( ) + '/' + job_id + '/' + dir_name

The local dir for staging files for that particular task .


def _make_environment(self, inputs, outputs, mounts):
    """Return a dictionary of environment variables for the container."""
    env = {}
    env.update(providers_util.get_file_environment_variables(inputs))
    env.update(providers_util.get_file_environment_variables(outputs))
    env.update(providers_util.get_file_environment_variables(mounts))
    return env

def _make_environment ( self , inputs , outputs , mounts ) : env = { } env . update ( providers_util . get_file_environment_variables ( inputs ) ) env . update ( providers_util . get_file_environment_variables ( outputs ) ) env . update ( providers_util . get_file_environment_variables ( mounts ) ) return env

Return a dictionary of environment variables for the container .


def _localize_inputs_recursive_command(self, task_dir, inputs):
    """Returns a command that will stage recursive inputs."""
    data_dir = os.path.join(task_dir, _DATA_SUBDIR)
    provider_commands = [
        providers_util.build_recursive_localize_command(data_dir, inputs,
                                                        file_provider)
        for file_provider in _SUPPORTED_INPUT_PROVIDERS
    ]
    return '\n'.join(provider_commands)

def _localize_inputs_recursive_command ( self , task_dir , inputs ) : data_dir = os . path . join ( task_dir , _DATA_SUBDIR ) provider_commands = [ providers_util . build_recursive_localize_command ( data_dir , inputs , file_provider ) for file_provider in _SUPPORTED_INPUT_PROVIDERS ] return '\n' . join ( provider_commands )

Returns a command that will stage recursive inputs .


def _get_input_target_path(self, local_file_path):
    """Returns a directory or file path to be the target for "gsutil cp".

    If the filename contains a wildcard, then the target path must
    be a directory in order to ensure consistency whether the source pattern
    contains one or multiple files.

    Args:
      local_file_path: A full path terminating in a file or a file wildcard.

    Returns:
      The path to use as the "gsutil cp" target.
    """

    path, filename = os.path.split(local_file_path)
    if '*' in filename:
      return path + '/'
    else:
      return local_file_path

def _get_input_target_path ( self , local_file_path ) : path , filename = os . path . split ( local_file_path ) if '*' in filename : return path + '/' else : return local_file_path

Returns a directory or file path to be the target for gsutil cp .


def _localize_inputs_command(self, task_dir, inputs, user_project):
    """Returns a command that will stage inputs."""
    commands = []
    for i in inputs:
      if i.recursive or not i.value:
        continue

      source_file_path = i.uri
      local_file_path = task_dir + '/' + _DATA_SUBDIR + '/' + i.docker_path
      dest_file_path = self._get_input_target_path(local_file_path)

      commands.append('mkdir -p "%s"' % os.path.dirname(local_file_path))

      if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
        # The semantics that we expect here are implemented consistently in
        # "gsutil cp", and are a bit different than "cp" when it comes to
        # wildcard handling, so use it for both local and GCS:
        #
        # - `cp path/* dest/` will error if "path" has subdirectories.
        # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
        #   to come from shell).
        if user_project:
          command = 'gsutil -u %s -mq cp "%s" "%s"' % (
              user_project, source_file_path, dest_file_path)
        else:
          command = 'gsutil -mq cp "%s" "%s"' % (source_file_path,
                                                 dest_file_path)
        commands.append(command)

    return '\n'.join(commands)

def
_localize_inputs_command
(
self
,
task_dir
,
inputs
,
user_project
)
:
commands
=
[
]
for
i
in
inputs
:
if
i
.
recursive
or
not
i
.
value
:
continue
source_file_path
=
i
.
uri
local_file_path
=
task_dir
+
'/'
+
_DATA_SUBDIR
+
'/'
+
i
.
docker_path
dest_file_path
=
self
.
_get_input_target_path
(
local_file_path
)
commands
.
append
(
'mkdir -p "%s"'
%
os
.
path
.
dirname
(
local_file_path
)
)
if
i
.
file_provider
in
[
job_model
.
P_LOCAL
,
job_model
.
P_GCS
]
:
if
user_project
:
command
=
'gsutil -u %s -mq cp "%s" "%s"'
%
(
user_project
,
source_file_path
,
dest_file_path
)
else
:
command
=
'gsutil -mq cp "%s" "%s"'
%
(
source_file_path
,
dest_file_path
)
commands
.
append
(
command
)
return
'\n'
.
join
(
commands
)

Returns a command that will stage inputs .


def _delocalize_outputs_commands(self, task_dir, outputs, user_project):
    """Copy outputs from local disk to GCS."""
    commands = []
    for o in outputs:
      if o.recursive or not o.value:
        continue

      # The destination path is o.uri.path, which is the target directory
      # (rather than o.uri, which includes the filename or wildcard).
      dest_path = o.uri.path
      local_path = task_dir + '/' + _DATA_SUBDIR + '/' + o.docker_path

      if o.file_provider == job_model.P_LOCAL:
        commands.append('mkdir -p "%s"' % dest_path)

      # Use gsutil even for local files (explained in _localize_inputs_command).
      if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
        if user_project:
          command = 'gsutil -u %s -mq cp "%s" "%s"' % (user_project, local_path,
                                                       dest_path)
        else:
          command = 'gsutil -mq cp "%s" "%s"' % (local_path, dest_path)
        commands.append(command)

    return '\n'.join(commands)

def
_delocalize_outputs_commands
(
self
,
task_dir
,
outputs
,
user_project
)
:
commands
=
[
]
for
o
in
outputs
:
if
o
.
recursive
or
not
o
.
value
:
continue
dest_path
=
o
.
uri
.
path
local_path
=
task_dir
+
'/'
+
_DATA_SUBDIR
+
'/'
+
o
.
docker_path
if
o
.
file_provider
==
job_model
.
P_LOCAL
:
commands
.
append
(
'mkdir -p "%s"'
%
dest_path
)
if
o
.
file_provider
in
[
job_model
.
P_LOCAL
,
job_model
.
P_GCS
]
:
if
user_project
:
command
=
'gsutil -u %s -mq cp "%s" "%s"'
%
(
user_project
,
local_path
,
dest_path
)
else
:
command
=
'gsutil -mq cp "%s" "%s"'
%
(
local_path
,
dest_path
)
commands
.
append
(
command
)
return
'\n'
.
join
(
commands
)


def get_dsub_version():
  """Get the dsub version out of the _dsub_version.py source file.

  Setup.py should not import dsub version from dsub directly since ambiguity in
  import order could lead to an old version of dsub setting the version number.
  Parsing the file directly is simpler than using import tools (whose interface
  varies between python 2.7, 3.4, and 3.5).

  Returns:
    string of dsub version.

  Raises:
    ValueError: if the version is not found.
  """
  filename = os.path.join(os.path.dirname(__file__), 'dsub/_dsub_version.py')
  with open(filename, 'r') as versionfile:
    for line in versionfile:
      if line.startswith('DSUB_VERSION ='):
        # Get the version then strip whitespace and quote characters.
        version = line.partition('=')[2]
        return version.strip().strip('\'"')
  raise ValueError('Could not find version.')

def
get_dsub_version
(
)
:
filename
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
)
,
'dsub/_dsub_version.py'
)
with
open
(
filename
,
'r'
)
as
versionfile
:
for
line
in
versionfile
:
if
line
.
startswith
(
'DSUB_VERSION ='
)
:
version
=
line
.
partition
(
'='
)
[
2
]
return
version
.
strip
(
)
.
strip
(
'\'"'
)
raise
ValueError
(
'Could not find version.'
)

Get the dsub version out of the _dsub_version . py source file .


def get_filtered_normalized_events(self):
    """Filter the granular v2 events down to events of interest.

    Filter through the large number of granular events returned by the
    pipelines API, and extract only those that are interesting to a user. This
    is implemented by filtering out events which are known to be uninteresting
    (i.e. the default actions run for every job) and by explicitly matching
    specific events which are interesting and mapping those to v1 style naming.

    Events which are not whitelisted or blacklisted will still be output,
    meaning any events which are added in the future won't be masked.
    We don't want to suppress display of events that we don't recognize.
    They may be important.

    Returns:
      A list of maps containing the normalized, filtered events.
    """
    # Need the user-image to look for the right "pulling image" event
    user_image = google_v2_operations.get_action_image(self._op,
                                                       _ACTION_USER_COMMAND)

    # Only create an "ok" event for operations with SUCCESS status.
    need_ok = google_v2_operations.is_success(self._op)

    # Events are keyed by name for easier deletion.
    events = {}

    # Events are assumed to be ordered by timestamp (newest to oldest).
    for event in google_v2_operations.get_events(self._op):
      if self._filter(event):
        continue

      mapped, match = self._map(event)
      name = mapped['name']

      if name == 'ok':
        # If we want the "ok" event, we grab the first (most recent).
        if not need_ok or 'ok' in events:
          continue

      if name == 'pulling-image':
        if match.group(1) != user_image:
          continue

      events[name] = mapped

    return sorted(events.values(), key=operator.itemgetter('start-time'))

def get_filtered_normalized_events ( self ) : user_image = google_v2_operations . get_action_image ( self . _op , _ACTION_USER_COMMAND ) need_ok = google_v2_operations . is_success ( self . _op ) events = { } for event in google_v2_operations . get_events ( self . _op ) : if self . _filter ( event ) : continue mapped , match = self . _map ( event ) name = mapped [ 'name' ] if name == 'ok' : if not need_ok or 'ok' in events : continue if name == 'pulling-image' : if match . group ( 1 ) != user_image : continue events [ name ] = mapped return sorted ( events . values ( ) , key = operator . itemgetter ( 'start-time' ) )

Filter the granular v2 events down to events of interest .


def _map(self, event):
    """Extract elements from an operation event and map to a named event."""
    description = event.get('description', '')
    start_time = google_base.parse_rfc3339_utc_string(
        event.get('timestamp', ''))

    for name, regex in _EVENT_REGEX_MAP.items():
      match = regex.match(description)
      if match:
        return {'name': name, 'start-time': start_time}, match

    return {'name': description, 'start-time': start_time}, None

def _map ( self , event ) : description = event . get ( 'description' , '' ) start_time = google_base . parse_rfc3339_utc_string ( event . get ( 'timestamp' , '' ) ) for name , regex in _EVENT_REGEX_MAP . items ( ) : match = regex . match ( description ) if match : return { 'name' : name , 'start-time' : start_time } , match return { 'name' : description , 'start-time' : start_time } , None

Extract elements from an operation event and map to a named event .


def _get_logging_env(self, logging_uri, user_project):
    """Returns the environment for actions that copy logging files."""
    if not logging_uri.endswith('.log'):
      raise ValueError('Logging URI must end in ".log": {}'.format(logging_uri))

    logging_prefix = logging_uri[:-len('.log')]
    return {
        'LOGGING_PATH': '{}.log'.format(logging_prefix),
        'STDOUT_PATH': '{}-stdout.log'.format(logging_prefix),
        'STDERR_PATH': '{}-stderr.log'.format(logging_prefix),
        'USER_PROJECT': user_project,
    }

def
_get_logging_env
(
self
,
logging_uri
,
user_project
)
:
if
not
logging_uri
.
endswith
(
'.log'
)
:
raise
ValueError
(
'Logging URI must end in ".log": {}'
.
format
(
logging_uri
)
)
logging_prefix
=
logging_uri
[
:
len
(
'.log'
)
]
return
{
'LOGGING_PATH'
:
'{}.log'
.
format
(
logging_prefix
)
,
'STDOUT_PATH'
:
'{}-stdout.log'
.
format
(
logging_prefix
)
,
'STDERR_PATH'
:
'{}-stderr.log'
.
format
(
logging_prefix
)
,
'USER_PROJECT'
:
user_project
,
}

Returns the environment for actions that copy logging files .


def _get_prepare_env(self, script, job_descriptor, inputs, outputs, mounts):
    """Return a dict with variables for the 'prepare' action."""

    # Add the _SCRIPT_REPR with the repr(script) contents
    # Add the _META_YAML_REPR with the repr(meta) contents

    # Add variables for directories that need to be created, for example:
    # DIR_COUNT: 2
    # DIR_0: /mnt/data/input/gs/bucket/path1/
    # DIR_1: /mnt/data/output/gs/bucket/path2

    # List the directories in sorted order so that they are created in that
    # order. This is primarily to ensure that permissions are set as we create
    # each directory.
    # For example:
    #   mkdir -m 777 -p /root/first/second
    #   mkdir -m 777 -p /root/first
    # *may* not actually set 777 on /root/first

    docker_paths = sorted([
        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
        for var in inputs | outputs | mounts
        if var.value
    ])

    env = {
        _SCRIPT_VARNAME: repr(script.value),
        _META_YAML_VARNAME: repr(job_descriptor.to_yaml()),
        'DIR_COUNT': str(len(docker_paths))
    }

    for idx, path in enumerate(docker_paths):
      env['DIR_{}'.format(idx)] = os.path.join(providers_util.DATA_MOUNT_POINT,
                                               path)

    return env

def _get_prepare_env ( self , script , job_descriptor , inputs , outputs , mounts ) : docker_paths = sorted ( [ var . docker_path if var . recursive else os . path . dirname ( var . docker_path ) for var in inputs | outputs | mounts if var . value ] ) env = { _SCRIPT_VARNAME : repr ( script . value ) , _META_YAML_VARNAME : repr ( job_descriptor . to_yaml ( ) ) , 'DIR_COUNT' : str ( len ( docker_paths ) ) } for idx , path in enumerate ( docker_paths ) : env [ 'DIR_{}' . format ( idx ) ] = os . path . join ( providers_util . DATA_MOUNT_POINT , path ) return env

Return a dict with variables for the prepare action .


def _get_localization_env(self, inputs, user_project):
    """Return a dict with variables for the 'localization' action."""

    # Add variables for paths that need to be localized, for example:
    # INPUT_COUNT: 1
    # INPUT_0: MY_INPUT_FILE
    # INPUT_RECURSIVE_0: 0
    # INPUT_SRC_0: gs://mybucket/mypath/myfile
    # INPUT_DST_0: /mnt/data/inputs/mybucket/mypath/myfile

    non_empty_inputs = [var for var in inputs if var.value]
    env = {'INPUT_COUNT': str(len(non_empty_inputs))}

    for idx, var in enumerate(non_empty_inputs):
      env['INPUT_{}'.format(idx)] = var.name
      env['INPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
      env['INPUT_SRC_{}'.format(idx)] = var.value

      # For wildcard paths, the destination must be a directory
      dst = os.path.join(providers_util.DATA_MOUNT_POINT, var.docker_path)
      path, filename = os.path.split(dst)
      if '*' in filename:
        dst = '{}/'.format(path)
      env['INPUT_DST_{}'.format(idx)] = dst

    env['USER_PROJECT'] = user_project

    return env

def _get_localization_env ( self , inputs , user_project ) : non_empty_inputs = [ var for var in inputs if var . value ] env = { 'INPUT_COUNT' : str ( len ( non_empty_inputs ) ) } for idx , var in enumerate ( non_empty_inputs ) : env [ 'INPUT_{}' . format ( idx ) ] = var . name env [ 'INPUT_RECURSIVE_{}' . format ( idx ) ] = str ( int ( var . recursive ) ) env [ 'INPUT_SRC_{}' . format ( idx ) ] = var . value dst = os . path . join ( providers_util . DATA_MOUNT_POINT , var . docker_path ) path , filename = os . path . split ( dst ) if '*' in filename : dst = '{}/' . format ( path ) env [ 'INPUT_DST_{}' . format ( idx ) ] = dst env [ 'USER_PROJECT' ] = user_project return env

Return a dict with variables for the localization action .


def _get_delocalization_env(self, outputs, user_project):
    """Return a dict with variables for the 'delocalization' action."""

    # Add variables for paths that need to be delocalized, for example:
    # OUTPUT_COUNT: 1
    # OUTPUT_0: MY_OUTPUT_FILE
    # OUTPUT_RECURSIVE_0: 0
    # OUTPUT_SRC_0: gs://mybucket/mypath/myfile
    # OUTPUT_DST_0: /mnt/data/outputs/mybucket/mypath/myfile

    non_empty_outputs = [var for var in outputs if var.value]
    env = {'OUTPUT_COUNT': str(len(non_empty_outputs))}

    for idx, var in enumerate(non_empty_outputs):
      env['OUTPUT_{}'.format(idx)] = var.name
      env['OUTPUT_RECURSIVE_{}'.format(idx)] = str(int(var.recursive))
      env['OUTPUT_SRC_{}'.format(idx)] = os.path.join(
          providers_util.DATA_MOUNT_POINT, var.docker_path)

      # For wildcard paths, the destination must be a directory
      if '*' in var.uri.basename:
        dst = var.uri.path
      else:
        dst = var.uri
      env['OUTPUT_DST_{}'.format(idx)] = dst

    env['USER_PROJECT'] = user_project

    return env

def _get_delocalization_env ( self , outputs , user_project ) : non_empty_outputs = [ var for var in outputs if var . value ] env = { 'OUTPUT_COUNT' : str ( len ( non_empty_outputs ) ) } for idx , var in enumerate ( non_empty_outputs ) : env [ 'OUTPUT_{}' . format ( idx ) ] = var . name env [ 'OUTPUT_RECURSIVE_{}' . format ( idx ) ] = str ( int ( var . recursive ) ) env [ 'OUTPUT_SRC_{}' . format ( idx ) ] = os . path . join ( providers_util . DATA_MOUNT_POINT , var . docker_path ) if '*' in var . uri . basename : dst = var . uri . path else : dst = var . uri env [ 'OUTPUT_DST_{}' . format ( idx ) ] = dst env [ 'USER_PROJECT' ] = user_project return env

Return a dict with variables for the delocalization action .