def _build_user_environment(self, envs, inputs, outputs, mounts):
    """Returns a dictionary of for the user container environment."""
    envs = {env.name: env.value for env in envs}
    envs.update(providers_util.get_file_environment_variables(inputs))
    envs.update(providers_util.get_file_environment_variables(outputs))
    envs.update(providers_util.get_file_environment_variables(mounts))
    return envs

def _build_user_environment ( self , envs , inputs , outputs , mounts ) : envs = { env . name : env . value for env in envs } envs . update ( providers_util . get_file_environment_variables ( inputs ) ) envs . update ( providers_util . get_file_environment_variables ( outputs ) ) envs . update ( providers_util . get_file_environment_variables ( mounts ) ) return envs

Returns a dictionary of for the user container environment .


def _get_mount_actions(self, mounts, mnt_datadisk):
    """Returns a list of two actions per gcs bucket to mount."""
    actions_to_add = []
    for mount in mounts:
      bucket = mount.value[len('gs://'):]
      mount_path = mount.docker_path
      actions_to_add.extend([
          google_v2_pipelines.build_action(
              name='mount-{}'.format(bucket),
              flags=['ENABLE_FUSE', 'RUN_IN_BACKGROUND'],
              image_uri=_GCSFUSE_IMAGE,
              mounts=[mnt_datadisk],
              commands=[
                  '--implicit-dirs', '--foreground', '-o ro', bucket,
                  os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)
              ]),
          google_v2_pipelines.build_action(
              name='mount-wait-{}'.format(bucket),
              flags=['ENABLE_FUSE'],
              image_uri=_GCSFUSE_IMAGE,
              mounts=[mnt_datadisk],
              commands=[
                  'wait',
                  os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)
              ])
      ])
    return actions_to_add

def
_get_mount_actions
(
self
,
mounts
,
mnt_datadisk
)
:
actions_to_add
=
[
]
for
mount
in
mounts
:
bucket
=
mount
.
value
[
len
(
'gs://'
)
:
]
mount_path
=
mount
.
docker_path
actions_to_add
.
extend
(
[
google_v2_pipelines
.
build_action
(
name
=
'mount-{}'
.
format
(
bucket
)
,
flags
=
[
'ENABLE_FUSE'
,
'RUN_IN_BACKGROUND'
]
,
image_uri
=
_GCSFUSE_IMAGE
,
mounts
=
[
mnt_datadisk
]
,
commands
=
[
'--implicit-dirs'
,
'--foreground'
,
'-o ro'
,
bucket
,
os
.
path
.
join
(
providers_util
.
DATA_MOUNT_POINT
,
mount_path
)
]
)
,
google_v2_pipelines
.
build_action
(
name
=
'mount-wait-{}'
.
format
(
bucket
)
,
flags
=
[
'ENABLE_FUSE'
]
,
image_uri
=
_GCSFUSE_IMAGE
,
mounts
=
[
mnt_datadisk
]
,
commands
=
[
'wait'
,
os
.
path
.
join
(
providers_util
.
DATA_MOUNT_POINT
,
mount_path
)
]
)
]
)
return
actions_to_add

Returns a list of two actions per gcs bucket to mount .


def _operation_status(self):
    """Returns the status of this operation.

    Raises:
      ValueError: if the operation status cannot be determined.

    Returns:
      A printable status string (RUNNING, SUCCESS, CANCELED or FAILURE).
    """
    if not google_v2_operations.is_done(self._op):
      return 'RUNNING'
    if google_v2_operations.is_success(self._op):
      return 'SUCCESS'
    if google_v2_operations.is_canceled(self._op):
      return 'CANCELED'
    if google_v2_operations.is_failed(self._op):
      return 'FAILURE'

    raise ValueError('Status for operation {} could not be determined'.format(
        self._op['name']))


def _operation_status_message(self):
    """Returns the most relevant status string and failed action.

    This string is meant for display only.

    Returns:
      A printable status string and name of failed action (if any).
    """
    msg = None
    action = None
    if not google_v2_operations.is_done(self._op):
      last_event = google_v2_operations.get_last_event(self._op)
      if last_event:
        msg = last_event['description']
        action_id = last_event.get('details', {}).get('actionId')
        if action_id:
          action = google_v2_operations.get_action_by_id(self._op, action_id)
      else:
        msg = 'Pending'
    else:
      failed_events = google_v2_operations.get_failed_events(self._op)
      if failed_events:
        failed_event = failed_events[-1]
        msg = failed_event.get('details', {}).get('stderr')
        action_id = failed_event.get('details', {}).get('actionId')
        if action_id:
          action = google_v2_operations.get_action_by_id(self._op, action_id)
      if not msg:
        error = google_v2_operations.get_error(self._op)
        if error:
          msg = error['message']
        else:
          msg = 'Success'

    return msg, action

def _operation_status_message ( self ) : msg = None action = None if not google_v2_operations . is_done ( self . _op ) : last_event = google_v2_operations . get_last_event ( self . _op ) if last_event : msg = last_event [ 'description' ] action_id = last_event . get ( 'details' , { } ) . get ( 'actionId' ) if action_id : action = google_v2_operations . get_action_by_id ( self . _op , action_id ) else : msg = 'Pending' else : failed_events = google_v2_operations . get_failed_events ( self . _op ) if failed_events : failed_event = failed_events [ - 1 ] msg = failed_event . get ( 'details' , { } ) . get ( 'stderr' ) action_id = failed_event . get ( 'details' , { } ) . get ( 'actionId' ) if action_id : action = google_v2_operations . get_action_by_id ( self . _op , action_id ) if not msg : error = google_v2_operations . get_error ( self . _op ) if error : msg = error [ 'message' ] else : msg = 'Success' return msg , action

Returns the most relevant status string and failed action .


def _validate_ram(ram_in_mb):
    """Rounds ram up to the nearest multiple of _MEMORY_MULTIPLE."""
    return int(GoogleV2CustomMachine._MEMORY_MULTIPLE * math.ceil(
        ram_in_mb / GoogleV2CustomMachine._MEMORY_MULTIPLE))

def _validate_ram ( ram_in_mb ) : return int ( GoogleV2CustomMachine . _MEMORY_MULTIPLE * math . ceil ( ram_in_mb / GoogleV2CustomMachine . _MEMORY_MULTIPLE ) )

Rounds ram up to the nearest multiple of _MEMORY_MULTIPLE .


def build_machine_type(cls, min_cores, min_ram):
    """Returns a custom machine type string."""
    min_cores = min_cores or job_model.DEFAULT_MIN_CORES
    min_ram = min_ram or job_model.DEFAULT_MIN_RAM

    # First, min_ram is given in GB. Convert to MB.
    min_ram *= GoogleV2CustomMachine._MB_PER_GB

    # Only machine types with 1 vCPU or an even number of vCPUs can be created.
    cores = cls._validate_cores(min_cores)
    # The total memory of the instance must be a multiple of 256 MB.
    ram = cls._validate_ram(min_ram)

    # Memory must be between 0.9 GB per vCPU, up to 6.5 GB per vCPU.
    memory_to_cpu_ratio = ram / cores

    if memory_to_cpu_ratio < GoogleV2CustomMachine._MIN_MEMORY_PER_CPU:
      # If we're under the ratio, top up the memory.
      adjusted_ram = GoogleV2CustomMachine._MIN_MEMORY_PER_CPU * cores
      ram = cls._validate_ram(adjusted_ram)

    elif memory_to_cpu_ratio > GoogleV2CustomMachine._MAX_MEMORY_PER_CPU:
      # If we're over the ratio, top up the CPU.
      adjusted_cores = math.ceil(
          ram / GoogleV2CustomMachine._MAX_MEMORY_PER_CPU)
      cores = cls._validate_cores(adjusted_cores)

    else:
      # Ratio is within the restrictions - no adjustments needed.
      pass

    return 'custom-{}-{}'.format(int(cores), int(ram))

def build_machine_type ( cls , min_cores , min_ram ) : min_cores = min_cores or job_model . DEFAULT_MIN_CORES min_ram = min_ram or job_model . DEFAULT_MIN_RAM min_ram *= GoogleV2CustomMachine . _MB_PER_GB cores = cls . _validate_cores ( min_cores ) ram = cls . _validate_ram ( min_ram ) memory_to_cpu_ratio = ram / cores if memory_to_cpu_ratio < GoogleV2CustomMachine . _MIN_MEMORY_PER_CPU : adjusted_ram = GoogleV2CustomMachine . _MIN_MEMORY_PER_CPU * cores ram = cls . _validate_ram ( adjusted_ram ) elif memory_to_cpu_ratio > GoogleV2CustomMachine . _MAX_MEMORY_PER_CPU : adjusted_cores = math . ceil ( ram / GoogleV2CustomMachine . _MAX_MEMORY_PER_CPU ) cores = cls . _validate_cores ( adjusted_cores ) else : pass return 'custom-{}-{}' . format ( int ( cores ) , int ( ram ) )


def build_machine(network=None,
                  machine_type=None,
                  preemptible=None,
                  service_account=None,
                  boot_disk_size_gb=None,
                  disks=None,
                  accelerators=None,
                  labels=None,
                  cpu_platform=None,
                  nvidia_driver_version=None):
  """Build a VirtualMachine object for a Pipeline request.

  Args:
    network (dict): Network details for the pipeline to run in.
    machine_type (str): GCE Machine Type string for the pipeline.
    preemptible (bool): Use a preemptible VM for the job.
    service_account (dict): Service account configuration for the VM.
    boot_disk_size_gb (int): Boot disk size in GB.
    disks (list[dict]): List of disks to mount.
    accelerators (list[dict]): List of accelerators to attach to the VM.
    labels (dict[string, string]): Labels for the VM.
    cpu_platform (str): The CPU platform to request.
    nvidia_driver_version (str): The NVIDIA driver version to use when attaching
      an NVIDIA GPU accelerator.

  Returns:
    An object representing a VirtualMachine.
  """
  return {
      'network': network,
      'machineType': machine_type,
      'preemptible': preemptible,
      'serviceAccount': service_account,
      'bootDiskSizeGb': boot_disk_size_gb,
      'disks': disks,
      'accelerators': accelerators,
      'labels': labels,
      'cpuPlatform': cpu_platform,
      'nvidiaDriverVersion': nvidia_driver_version,
  }

def build_machine ( network = None , machine_type = None , preemptible = None , service_account = None , boot_disk_size_gb = None , disks = None , accelerators = None , labels = None , cpu_platform = None , nvidia_driver_version = None ) : return { 'network' : network , 'machineType' : machine_type , 'preemptible' : preemptible , 'serviceAccount' : service_account , 'bootDiskSizeGb' : boot_disk_size_gb , 'disks' : disks , 'accelerators' : accelerators , 'labels' : labels , 'cpuPlatform' : cpu_platform , 'nvidiaDriverVersion' : nvidia_driver_version , }

Build a VirtualMachine object for a Pipeline request .


def build_action(name=None,
                 image_uri=None,
                 commands=None,
                 entrypoint=None,
                 environment=None,
                 pid_namespace=None,
                 flags=None,
                 port_mappings=None,
                 mounts=None,
                 labels=None):
  """Build an Action object for a Pipeline request.

  Args:
    name (str): An optional name for the container.
    image_uri (str): The URI to pull the container image from.
    commands (List[str]): commands and arguments to run inside the container.
    entrypoint (str): overrides the ENTRYPOINT specified in the container.
    environment (dict[str,str]): The environment to pass into the container.
    pid_namespace (str): The PID namespace to run the action inside.
    flags (str): Flags that control the execution of this action.
    port_mappings (dict[int, int]): A map of container to host port mappings for
      this container.
    mounts (List): A list of mounts to make available to the action.
    labels (dict[str]): Labels to associate with the action.

  Returns:
    An object representing an Action resource.
  """

  return {
      'name': name,
      'imageUri': image_uri,
      'commands': commands,
      'entrypoint': entrypoint,
      'environment': environment,
      'pidNamespace': pid_namespace,
      'flags': flags,
      'portMappings': port_mappings,
      'mounts': mounts,
      'labels': labels,
  }

def build_action ( name = None , image_uri = None , commands = None , entrypoint = None , environment = None , pid_namespace = None , flags = None , port_mappings = None , mounts = None , labels = None ) : return { 'name' : name , 'imageUri' : image_uri , 'commands' : commands , 'entrypoint' : entrypoint , 'environment' : environment , 'pidNamespace' : pid_namespace , 'flags' : flags , 'portMappings' : port_mappings , 'mounts' : mounts , 'labels' : labels , }

Build an Action object for a Pipeline request .


def lookup_job_tasks(self,
                       statuses,
                       user_ids=None,
                       job_ids=None,
                       job_names=None,
                       task_ids=None,
                       task_attempts=None,
                       labels=None,
                       create_time_min=None,
                       create_time_max=None,
                       max_tasks=0):
    """Return a list of operations. See base.py for additional detail."""
    statuses = None if statuses == {'*'} else statuses
    user_ids = None if user_ids == {'*'} else user_ids
    job_ids = None if job_ids == {'*'} else job_ids
    job_names = None if job_names == {'*'} else job_names
    task_ids = None if task_ids == {'*'} else task_ids
    task_attempts = None if task_attempts == {'*'} else task_attempts

    if labels or create_time_min or create_time_max:
      raise NotImplementedError(
          'Lookup by labels and create_time not yet supported by stub.')

    operations = [
        x for x in self._operations
        if ((not statuses or x.get_field('status', (None, None))[0] in statuses
            ) and (not user_ids or x.get_field('user', None) in user_ids) and
            (not job_ids or x.get_field('job-id', None) in job_ids) and
            (not job_names or x.get_field('job-name', None) in job_names) and
            (not task_ids or x.get_field('task-id', None) in task_ids) and
            (not task_attempts or
             x.get_field('task-attempt', None) in task_attempts))
    ]
    if max_tasks > 0:
      operations = operations[:max_tasks]
    return operations

Return a list of operations . See base . py for additional detail .


def get_provider(args, resources):
  """Returns a provider for job submission requests."""

  provider = getattr(args, 'provider', 'google')

  if provider == 'google':
    return google.GoogleJobProvider(
        getattr(args, 'verbose', False),
        getattr(args, 'dry_run', False), args.project)
  elif provider == 'google-v2':
    return google_v2.GoogleV2JobProvider(
        getattr(args, 'verbose', False), getattr(args, 'dry_run', False),
        args.project)
  elif provider == 'local':
    return local.LocalJobProvider(resources)
  elif provider == 'test-fails':
    return test_fails.FailsJobProvider()
  else:
    raise ValueError('Unknown provider: ' + provider)

def
get_provider
(
args
,
resources
)
:
provider
=
getattr
(
args
,
'provider'
,
'google'
)
if
provider
==
'google'
:
return
google
.
GoogleJobProvider
(
getattr
(
args
,
'verbose'
,
False
)
,
getattr
(
args
,
'dry_run'
,
False
)
,
args
.
project
)
elif
provider
==
'google-v2'
:
return
google_v2
.
GoogleV2JobProvider
(
getattr
(
args
,
'verbose'
,
False
)
,
getattr
(
args
,
'dry_run'
,
False
)
,
args
.
project
)
elif
provider
==
'local'
:
return
local
.
LocalJobProvider
(
resources
)
elif
provider
==
'test-fails'
:
return
test_fails
.
FailsJobProvider
(
)
else
:
raise
ValueError
(
'Unknown provider: '
+
provider
)

Returns a provider for job submission requests .


def parse_args(parser, provider_required_args, argv):
  """Add provider required arguments epilog message, parse, and validate."""

  # Add the provider required arguments epilog message
  epilog = 'Provider-required arguments:\n'
  for provider in provider_required_args:
    epilog += '  %s: %s\n' % (provider, provider_required_args[provider])
  parser.epilog = epilog

  # Parse arguments
  args = parser.parse_args(argv)

  # For the selected provider, check the required arguments
  for arg in provider_required_args[args.provider]:
    if not args.__getattribute__(arg):
      parser.error('argument --%s is required' % arg)

  return args

def
parse_args
(
parser
,
provider_required_args
,
argv
)
:
epilog
=
'Provider-required arguments:\n'
for
provider
in
provider_required_args
:
epilog
+=
'  %s: %s\n'
%
(
provider
,
provider_required_args
[
provider
]
)
parser
.
epilog
=
epilog
args
=
parser
.
parse_args
(
argv
)
for
arg
in
provider_required_args
[
args
.
provider
]
:
if
not
args
.
__getattribute__
(
arg
)
:
parser
.
error
(
'argument --%s is required'
%
arg
)
return
args

Add provider required arguments epilog message parse and validate .


def get_dstat_provider_args(provider, project):
  """A string with the arguments to point dstat to the same provider+project."""
  provider_name = get_provider_name(provider)

  args = []
  if provider_name == 'google':
    args.append('--project %s' % project)
  elif provider_name == 'google-v2':
    args.append('--project %s' % project)
  elif provider_name == 'local':
    pass
  elif provider_name == 'test-fails':
    pass
  else:
    # New providers should add their dstat required arguments here.
    assert False, 'Provider %s needs get_dstat_provider_args support' % provider

  args.insert(0, '--provider %s' % provider_name)
  return ' '.join(args)

A string with the arguments to point dstat to the same provider + project .


def _format_task_uri(fmt, job_metadata, task_metadata):
  """Returns a URI with placeholders replaced by metadata values."""

  values = {
      'job-id': None,
      'task-id': 'task',
      'job-name': None,
      'user-id': None,
      'task-attempt': None
  }
  for key in values:
    values[key] = task_metadata.get(key) or job_metadata.get(key) or values[key]

  return fmt.format(**values)

def _format_task_uri ( fmt , job_metadata , task_metadata ) : values = { 'job-id' : None , 'task-id' : 'task' , 'job-name' : None , 'user-id' : None , 'task-attempt' : None } for key in values : values [ key ] = task_metadata . get ( key ) or job_metadata . get ( key ) or values [ key ] return fmt . format ( ** values )

Returns a URI with placeholders replaced by metadata values .


def format_logging_uri(uri, job_metadata, task_metadata):
  """Inserts task metadata into the logging URI.

  The core behavior is inspired by the Google Pipelines API:
    (1) If a the uri ends in ".log", then that is the logging path.
    (2) Otherwise, the uri is treated as "directory" for logs and a filename
        needs to be automatically generated.

  For (1), if the job is a --tasks job, then the {task-id} is inserted
  before ".log".

  For (2), the file name generated is {job-id}, or for --tasks jobs, it is
  {job-id}.{task-id}.

  In both cases .{task-attempt} is inserted before .log for --retries jobs.

  In addition, full task metadata substitution is supported. The URI
  may include substitution strings such as
  "{job-id}", "{task-id}", "{job-name}", "{user-id}", and "{task-attempt}".

  Args:
    uri: User-specified logging URI which may contain substitution fields.
    job_metadata: job-global metadata.
    task_metadata: tasks-specific metadata.

  Returns:
    The logging_uri formatted as described above.
  """

  # If the user specifies any formatting (with curly braces), then use that
  # as the format string unchanged.
  fmt = str(uri)
  if '{' not in fmt:
    if uri.endswith('.log'):
      # URI includes a filename. Trim the extension and just use the prefix.
      fmt = os.path.splitext(uri)[0]
    else:
      # URI is a path to a directory. The job-id becomes the filename prefix.
      fmt = os.path.join(uri, '{job-id}')

    # If this is a task job, add the task-id.
    if task_metadata.get('task-id') is not None:
      fmt += '.{task-id}'

    # If this is a retryable task, add the task-attempt.
    if task_metadata.get('task-attempt') is not None:
      fmt += '.{task-attempt}'

    fmt += '.log'

  return _format_task_uri(fmt, job_metadata, task_metadata)

def format_logging_uri ( uri , job_metadata , task_metadata ) : fmt = str ( uri ) if '{' not in fmt : if uri . endswith ( '.log' ) : fmt = os . path . splitext ( uri ) [ 0 ] else : fmt = os . path . join ( uri , '{job-id}' ) if task_metadata . get ( 'task-id' ) is not None : fmt += '.{task-id}' if task_metadata . get ( 'task-attempt' ) is not None : fmt += '.{task-attempt}' fmt += '.log' return _format_task_uri ( fmt , job_metadata , task_metadata )

Inserts task metadata into the logging URI .


def _google_v2_parse_arguments(args):
  """Validated google-v2 arguments."""
  if (args.zones and args.regions) or (not args.zones and not args.regions):
    raise ValueError('Exactly one of --regions and --zones must be specified')

  if args.machine_type and (args.min_cores or args.min_ram):
    raise ValueError(
        '--machine-type not supported together with --min-cores or --min-ram.')


def _get_job_resources(args):
  """Extract job-global resources requirements from input args.

  Args:
    args: parsed command-line arguments

  Returns:
    Resources object containing the requested resources for the job
  """
  logging = param_util.build_logging_param(
      args.logging) if args.logging else None
  timeout = param_util.timeout_in_seconds(args.timeout)
  log_interval = param_util.log_interval_in_seconds(args.log_interval)

  return job_model.Resources(
      min_cores=args.min_cores,
      min_ram=args.min_ram,
      machine_type=args.machine_type,
      disk_size=args.disk_size,
      disk_type=args.disk_type,
      boot_disk_size=args.boot_disk_size,
      preemptible=args.preemptible,
      image=args.image,
      regions=args.regions,
      zones=args.zones,
      logging=logging,
      logging_path=None,
      service_account=args.service_account,
      scopes=args.scopes,
      keep_alive=args.keep_alive,
      cpu_platform=args.cpu_platform,
      network=args.network,
      subnetwork=args.subnetwork,
      use_private_address=args.use_private_address,
      accelerator_type=args.accelerator_type,
      accelerator_count=args.accelerator_count,
      nvidia_driver_version=args.nvidia_driver_version,
      timeout=timeout,
      log_interval=log_interval,
      ssh=args.ssh)

def _get_job_resources ( args ) : logging = param_util . build_logging_param ( args . logging ) if args . logging else None timeout = param_util . timeout_in_seconds ( args . timeout ) log_interval = param_util . log_interval_in_seconds ( args . log_interval ) return job_model . Resources ( min_cores = args . min_cores , min_ram = args . min_ram , machine_type = args . machine_type , disk_size = args . disk_size , disk_type = args . disk_type , boot_disk_size = args . boot_disk_size , preemptible = args . preemptible , image = args . image , regions = args . regions , zones = args . zones , logging = logging , logging_path = None , service_account = args . service_account , scopes = args . scopes , keep_alive = args . keep_alive , cpu_platform = args . cpu_platform , network = args . network , subnetwork = args . subnetwork , use_private_address = args . use_private_address , accelerator_type = args . accelerator_type , accelerator_count = args . accelerator_count , nvidia_driver_version = args . nvidia_driver_version , timeout = timeout , log_interval = log_interval , ssh = args . ssh )

Extract job - global resources requirements from input args .


def _get_job_metadata(provider, user_id, job_name, script, task_ids,
                      user_project, unique_job_id):
  """Allow provider to extract job-specific metadata from command-line args.

  Args:
    provider: job service provider
    user_id: user submitting the job
    job_name: name for the job
    script: the script to run
    task_ids: a set of the task-ids for all tasks in the job
    user_project: name of the project to be billed for the request
    unique_job_id: generate a unique job id

  Returns:
    A dictionary of job-specific metadata (such as job id, name, etc.)
  """
  create_time = dsub_util.replace_timezone(datetime.datetime.now(), tzlocal())
  user_id = user_id or dsub_util.get_os_user()
  job_metadata = provider.prepare_job_metadata(script.name, job_name, user_id,
                                               create_time)
  if unique_job_id:
    job_metadata['job-id'] = uuid.uuid4().hex

  job_metadata['create-time'] = create_time
  job_metadata['script'] = script
  job_metadata['user-project'] = user_project
  if task_ids:
    job_metadata['task-ids'] = dsub_util.compact_interval_string(list(task_ids))

  return job_metadata

def _get_job_metadata ( provider , user_id , job_name , script , task_ids , user_project , unique_job_id ) : create_time = dsub_util . replace_timezone ( datetime . datetime . now ( ) , tzlocal ( ) ) user_id = user_id or dsub_util . get_os_user ( ) job_metadata = provider . prepare_job_metadata ( script . name , job_name , user_id , create_time ) if unique_job_id : job_metadata [ 'job-id' ] = uuid . uuid4 ( ) . hex job_metadata [ 'create-time' ] = create_time job_metadata [ 'script' ] = script job_metadata [ 'user-project' ] = user_project if task_ids : job_metadata [ 'task-ids' ] = dsub_util . compact_interval_string ( list ( task_ids ) ) return job_metadata

Allow provider to extract job - specific metadata from command - line args .


def _resolve_task_logging(job_metadata, job_resources, task_descriptors):
  """Resolve the logging path from job and task properties.

  Args:
    job_metadata: Job metadata, such as job-id, job-name, and user-id.
    job_resources: Resources specified such as ram, cpu, and logging path.
    task_descriptors: Task metadata, parameters, and resources.

  Resolve the logging path, which may have substitution parameters such as
  job-id, task-id, user-id, and job-name.
  """
  if not job_resources.logging:
    return

  for task_descriptor in task_descriptors:
    logging_uri = provider_base.format_logging_uri(
        job_resources.logging.uri, job_metadata, task_descriptor.task_metadata)
    logging_path = job_model.LoggingParam(logging_uri,
                                          job_resources.logging.file_provider)

    if task_descriptor.task_resources:
      task_descriptor.task_resources = task_descriptor.task_resources._replace(
          logging_path=logging_path)
    else:
      task_descriptor.task_resources = job_model.Resources(
          logging_path=logging_path)

Resolve the logging path from job and task properties .


def _wait_after(provider, job_ids, poll_interval, stop_on_failure):
  """Print status info as we wait for those jobs.

  Blocks until either all of the listed jobs succeed,
  or one of them fails.

  Args:
    provider: job service provider
    job_ids: a set of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations
    stop_on_failure: whether to stop waiting if one of the tasks fails.

  Returns:
    Empty list if there was no error,
    a list of error messages from the failed tasks otherwise.
  """

  # Each time through the loop, the job_set is re-set to the jobs remaining to
  # check. Jobs are removed from the list when they complete.
  #
  # We exit the loop when:
  # * No jobs remain are running, OR
  # * stop_on_failure is TRUE AND at least one job returned an error

  # remove NO_JOB
  job_ids_to_check = {j for j in job_ids if j != dsub_util.NO_JOB}
  error_messages = []
  while job_ids_to_check and (not error_messages or not stop_on_failure):
    print('Waiting for: %s.' % (', '.join(job_ids_to_check)))

    # Poll until any remaining jobs have completed
    jobs_left = _wait_for_any_job(provider, job_ids_to_check, poll_interval)

    # Calculate which jobs just completed
    jobs_completed = job_ids_to_check.difference(jobs_left)

    # Get all tasks for the newly completed jobs
    tasks_completed = provider.lookup_job_tasks({'*'}, job_ids=jobs_completed)

    # We don't want to overwhelm the user with output when there are many
    # tasks per job. So we get a single "dominant" task for each of the
    # completed jobs (one that is representative of the job's fate).
    dominant_job_tasks = _dominant_task_for_jobs(tasks_completed)
    if len(dominant_job_tasks) != len(jobs_completed):
      # print info about the jobs we couldn't find
      # (should only occur for "--after" where the job ID is a typo).
      jobs_found = dsub_util.tasks_to_job_ids(dominant_job_tasks)
      jobs_not_found = jobs_completed.difference(jobs_found)
      for j in jobs_not_found:
        error = '%s: not found' % j
        print_error('  %s' % error)
        error_messages += [error]

    # Print the dominant task for the completed jobs
    for t in dominant_job_tasks:
      job_id = t.get_field('job-id')
      status = t.get_field('task-status')
      print('  %s: %s' % (str(job_id), str(status)))
      if status in ['FAILURE', 'CANCELED']:
        error_messages += [provider.get_tasks_completion_messages([t])]

    job_ids_to_check = jobs_left

  return error_messages

def
_wait_after
(
provider
,
job_ids
,
poll_interval
,
stop_on_failure
)
:
job_ids_to_check
=
{
j
for
j
in
job_ids
if
j
!=
dsub_util
.
NO_JOB
}
error_messages
=
[
]
while
job_ids_to_check
and
(
not
error_messages
or
not
stop_on_failure
)
:
print
(
'Waiting for: %s.'
%
(
', '
.
join
(
job_ids_to_check
)
)
)
jobs_left
=
_wait_for_any_job
(
provider
,
job_ids_to_check
,
poll_interval
)
jobs_completed
=
job_ids_to_check
.
difference
(
jobs_left
)
tasks_completed
=
provider
.
lookup_job_tasks
(
{
'*'
}
,
job_ids
=
jobs_completed
)
dominant_job_tasks
=
_dominant_task_for_jobs
(
tasks_completed
)
if
len
(
dominant_job_tasks
)
!=
len
(
jobs_completed
)
:
jobs_found
=
dsub_util
.
tasks_to_job_ids
(
dominant_job_tasks
)
jobs_not_found
=
jobs_completed
.
difference
(
jobs_found
)
for
j
in
jobs_not_found
:
error
=
'%s: not found'
%
j
print_error
(
'  %s'
%
error
)
error_messages
+=
[
error
]
for
t
in
dominant_job_tasks
:
job_id
=
t
.
get_field
(
'job-id'
)
status
=
t
.
get_field
(
'task-status'
)
print
(
'  %s: %s'
%
(
str
(
job_id
)
,
str
(
status
)
)
)
if
status
in
[
'FAILURE'
,
'CANCELED'
]
:
error_messages
+=
[
provider
.
get_tasks_completion_messages
(
[
t
]
)
]
job_ids_to_check
=
jobs_left
return
error_messages

Print status info as we wait for those jobs .


def _wait_and_retry(provider, job_id, poll_interval, retries, job_descriptor):
  """Wait for job and retry any tasks that fail.

  Stops retrying an individual task when: it succeeds, is canceled, or has been
  retried "retries" times.

  This function exits when there are no tasks running and there are no tasks
  eligible to be retried.

  Args:
    provider: job service provider
    job_id: a single job ID (string) to wait for
    poll_interval: integer seconds to wait between iterations
    retries: number of retries
    job_descriptor: job descriptor used to originally submit job

  Returns:
    Empty list if there was no error,
    a list containing an error message from a failed task otherwise.
  """

  while True:
    tasks = provider.lookup_job_tasks({'*'}, job_ids=[job_id])

    running_tasks = set()
    completed_tasks = set()
    canceled_tasks = set()
    fully_failed_tasks = set()
    task_fail_count = dict()

    # This is an arbitrary task that is either fully failed or canceled (with
    # preference for the former).
    message_task = None

    task_dict = dict()
    for t in tasks:
      task_id = job_model.numeric_task_id(t.get_field('task-id'))
      task_dict[task_id] = t

      status = t.get_field('task-status')
      if status == 'FAILURE':
        # Could compute this from task-attempt as well.
        task_fail_count[task_id] = task_fail_count.get(task_id, 0) + 1
        if task_fail_count[task_id] > retries:
          fully_failed_tasks.add(task_id)
          message_task = t
      elif status == 'CANCELED':
        canceled_tasks.add(task_id)
        if not message_task:
          message_task = t
      elif status == 'SUCCESS':
        completed_tasks.add(task_id)
      elif status == 'RUNNING':
        running_tasks.add(task_id)

    retry_tasks = (
        set(task_fail_count).difference(fully_failed_tasks)
        .difference(running_tasks).difference(completed_tasks)
        .difference(canceled_tasks))

    # job completed.
    if not retry_tasks and not running_tasks:
      # If there are any fully failed tasks, return the completion message of an
      # arbitrary one.
      # If not, but there are canceled tasks, return the completion message of
      # an arbitrary one.
      if message_task:
        return [provider.get_tasks_completion_messages([message_task])]

      # Otherwise successful completion.
      return []

    for task_id in retry_tasks:
      identifier = '{}.{}'.format(job_id, task_id) if task_id else job_id
      print('  {} (attempt {}) failed. Retrying.'.format(
          identifier, task_fail_count[task_id]))
      msg = task_dict[task_id].get_field('status-message')
      print('  Failure message: {}'.format(msg))

      _retry_task(provider, job_descriptor, task_id,
                  task_fail_count[task_id] + 1)

    SLEEP_FUNCTION(poll_interval)

def
_wait_and_retry
(
provider
,
job_id
,
poll_interval
,
retries
,
job_descriptor
)
:
while
True
:
tasks
=
provider
.
lookup_job_tasks
(
{
'*'
}
,
job_ids
=
[
job_id
]
)
running_tasks
=
set
(
)
completed_tasks
=
set
(
)
canceled_tasks
=
set
(
)
fully_failed_tasks
=
set
(
)
task_fail_count
=
dict
(
)
message_task
=
task_dict
=
dict
(
)
for
t
in
tasks
:
task_id
=
job_model
.
numeric_task_id
(
t
.
get_field
(
'task-id'
)
)
task_dict
[
task_id
]
=
t
status
=
t
.
get_field
(
'task-status'
)
if
status
==
'FAILURE'
:
task_fail_count
[
task_id
]
=
task_fail_count
.
get
(
task_id
,
0
)
+
1
if
task_fail_count
[
task_id
]
>
retries
:
fully_failed_tasks
.
add
(
task_id
)
message_task
=
t
elif
status
==
'CANCELED'
:
canceled_tasks
.
add
(
task_id
)
if
not
message_task
:
message_task
=
t
elif
status
==
'SUCCESS'
:
completed_tasks
.
add
(
task_id
)
elif
status
==
'RUNNING'
:
running_tasks
.
add
(
task_id
)
retry_tasks
=
(
set
(
task_fail_count
)
.
difference
(
fully_failed_tasks
)
.
difference
(
running_tasks
)
.
difference
(
completed_tasks
)
.
difference
(
canceled_tasks
)
)
if
not
retry_tasks
and
not
running_tasks
:
if
message_task
:
return
[
provider
.
get_tasks_completion_messages
(
[
message_task
]
)
]
return
[
]
for
task_id
in
retry_tasks
:
identifier
=
'{}.{}'
.
format
(
job_id
,
task_id
)
if
task_id
else
job_id
print
(
'  {} (attempt {}) failed. Retrying.'
.
format
(
identifier
,
task_fail_count
[
task_id
]
)
)
msg
=
task_dict
[
task_id
]
.
get_field
(
'status-message'
)
print
(
'  Failure message: {}'
.
format
(
msg
)
)
_retry_task
(
provider
,
job_descriptor
,
task_id
,
task_fail_count
[
task_id
]
+
1
)
SLEEP_FUNCTION
(
poll_interval
)

Wait for job and retry any tasks that fail .


def _dominant_task_for_jobs(tasks):
  """A list with, for each job, its dominant task.

  The dominant task is the one that exemplifies its job's
  status. It is either:
  - the first (FAILURE or CANCELED) task, or if none
  - the first RUNNING task, or if none
  - the first SUCCESS task.

  Args:
    tasks: a list of tasks to consider

  Returns:
    A list with, for each job, its dominant task.
  """

  per_job = _group_tasks_by_jobid(tasks)

  ret = []
  for job_id in per_job.keys():
    tasks_in_salience_order = sorted(per_job[job_id], key=_importance_of_task)
    ret.append(tasks_in_salience_order[0])
  return ret

def _dominant_task_for_jobs ( tasks ) : per_job = _group_tasks_by_jobid ( tasks ) ret = [ ] for job_id in per_job . keys ( ) : tasks_in_salience_order = sorted ( per_job [ job_id ] , key = _importance_of_task ) ret . append ( tasks_in_salience_order [ 0 ] ) return ret

A list with for each job its dominant task .


def _group_tasks_by_jobid(tasks):
  """A defaultdict with, for each job, a list of its tasks."""
  ret = collections.defaultdict(list)
  for t in tasks:
    ret[t.get_field('job-id')].append(t)
  return ret

def _group_tasks_by_jobid ( tasks ) : ret = collections . defaultdict ( list ) for t in tasks : ret [ t . get_field ( 'job-id' ) ] . append ( t ) return ret

A defaultdict with for each job a list of its tasks .


def _wait_for_any_job(provider, job_ids, poll_interval):
  """Waits until any of the listed jobs is not running.

  In particular, if any of the jobs sees one of its tasks fail,
  we count the whole job as failing (but do not terminate the remaining
  tasks ourselves).

  Args:
    provider: job service provider
    job_ids: a list of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations

  Returns:
    A set of the jobIDs with still at least one running task.
  """
  if not job_ids:
    return
  while True:
    tasks = provider.lookup_job_tasks({'*'}, job_ids=job_ids)
    running_jobs = set()
    failed_jobs = set()
    for t in tasks:
      status = t.get_field('task-status')
      job_id = t.get_field('job-id')
      if status in ['FAILURE', 'CANCELED']:
        failed_jobs.add(job_id)
      if status == 'RUNNING':
        running_jobs.add(job_id)
    remaining_jobs = running_jobs.difference(failed_jobs)
    if failed_jobs or len(remaining_jobs) != len(job_ids):
      return remaining_jobs
    SLEEP_FUNCTION(poll_interval)

def _wait_for_any_job ( provider , job_ids , poll_interval ) : if not job_ids : return while True : tasks = provider . lookup_job_tasks ( { '*' } , job_ids = job_ids ) running_jobs = set ( ) failed_jobs = set ( ) for t in tasks : status = t . get_field ( 'task-status' ) job_id = t . get_field ( 'job-id' ) if status in [ 'FAILURE' , 'CANCELED' ] : failed_jobs . add ( job_id ) if status == 'RUNNING' : running_jobs . add ( job_id ) remaining_jobs = running_jobs . difference ( failed_jobs ) if failed_jobs or len ( remaining_jobs ) != len ( job_ids ) : return remaining_jobs SLEEP_FUNCTION ( poll_interval )

Waits until any of the listed jobs is not running .


def _validate_job_and_task_arguments(job_params, task_descriptors):
  """Validates that job and task argument names do not overlap."""

  if not task_descriptors:
    return

  task_params = task_descriptors[0].task_params

  # The use case for specifying a label or env/input/output parameter on
  # the command-line and also including it in the --tasks file is not obvious.
  # Should the command-line override the --tasks file? Why?
  # Until this use is articulated, generate an error on overlapping names.

  # Check labels
  from_jobs = {label.name for label in job_params['labels']}
  from_tasks = {label.name for label in task_params['labels']}

  intersect = from_jobs & from_tasks
  if intersect:
    raise ValueError(
        'Names for labels on the command-line and in the --tasks file must not '
        'be repeated: {}'.format(','.join(intersect)))

  # Check envs, inputs, and outputs, all of which must not overlap each other
  from_jobs = {
      item.name
      for item in job_params['envs'] | job_params['inputs']
      | job_params['outputs']
  }
  from_tasks = {
      item.name
      for item in task_params['envs'] | task_params['inputs']
      | task_params['outputs']
  }

  intersect = from_jobs & from_tasks
  if intersect:
    raise ValueError(
        'Names for envs, inputs, and outputs on the command-line and in the '
        '--tasks file must not be repeated: {}'.format(','.join(intersect)))

Validates that job and task argument names do not overlap .


def _name_for_command(command):
  r"""Craft a simple command name from the command.

  The best command strings for this are going to be those where a simple
  command was given; we will use the command to derive the name.

  We won't always be able to figure something out and the caller should just
  specify a "--name" on the command-line.

  For example, commands like "export VAR=val\necho ${VAR}", this function would
  return "export".

  If the command starts space or a comment, then we'll skip to the first code
  we can find.

  If we find nothing, just return "command".

  >>> _name_for_command('samtools index "${BAM}"')
  'samtools'
  >>> _name_for_command('/usr/bin/sort "${INFILE}" > "${OUTFILE}"')
  'sort'
  >>> _name_for_command('# This should be ignored')
  'command'
  >>> _name_for_command('\\\n\\\n# Bad continuations, but ignore.\necho hello.')
  'echo'

  Arguments:
    command: the user-provided command
  Returns:
    a proposed name for the task.
  """

  lines = command.splitlines()
  for line in lines:
    line = line.strip()
    if line and not line.startswith('#') and line != '\\':
      return os.path.basename(re.split(r'\s', line)[0])

  return 'command'

def _name_for_command ( command ) : r lines = command . splitlines ( ) for line in lines : line = line . strip ( ) if line and not line . startswith ( '#' ) and line != '\\' : return os . path . basename ( re . split ( r'\s' , line ) [ 0 ] ) return 'command'

r Craft a simple command name from the command .


def _local_uri_rewriter(raw_uri):
  """Rewrite local file URIs as required by the rewrite_uris method.

  Local file paths, unlike GCS paths, may have their raw URI simplified by
  os.path.normpath which collapses extraneous indirect characters.

  >>> _local_uri_rewriter('/tmp/a_path/../B_PATH/file.txt')
  ('/tmp/B_PATH/file.txt', 'file/tmp/B_PATH/file.txt')
  >>> _local_uri_rewriter('/myhome/./mydir/')
  ('/myhome/mydir/', 'file/myhome/mydir/')

  The local path rewriter will also work to preserve relative paths even
  when creating the docker path. This prevents leaking of information on the
  invoker's system to the remote system. Doing this requires a number of path
  substitutions denoted with the _<rewrite>_ convention.

  >>> _local_uri_rewriter('./../upper_dir/')[1]
  'file/_dotdot_/upper_dir/'
  >>> _local_uri_rewriter('~/localdata/*.bam')[1]
  'file/_home_/localdata/*.bam'

  Args:
    raw_uri: (str) the raw file or directory path.

  Returns:
    normalized: a simplified and/or expanded version of the uri.
    docker_path: the uri rewritten in the format required for mounting inside
                 a docker worker.

  """
  # The path is split into components so that the filename is not rewritten.
  raw_path, filename = os.path.split(raw_uri)
  # Generate the local path that can be resolved by filesystem operations,
  # this removes special shell characters, condenses indirects and replaces
  # any unnecessary prefix.
  prefix_replacements = [('file:///', '/'), ('~/', os.getenv('HOME')), ('./',
                                                                        ''),
                         ('file:/', '/')]
  normed_path = raw_path
  for prefix, replacement in prefix_replacements:
    if normed_path.startswith(prefix):
      normed_path = os.path.join(replacement, normed_path[len(prefix):])
  # Because abspath strips the trailing '/' from bare directory references
  # other than root, this ensures that all directory references end with '/'.
  normed_uri = directory_fmt(os.path.abspath(normed_path))
  normed_uri = os.path.join(normed_uri, filename)

  # Generate the path used inside the docker image;
  #  1) Get rid of extra indirects: /this/./that -> /this/that
  #  2) Rewrite required indirects as synthetic characters.
  #  3) Strip relative or absolute path leading character.
  #  4) Add 'file/' prefix.
  docker_rewrites = [(r'/\.\.', '/_dotdot_'), (r'^\.\.', '_dotdot_'),
                     (r'^~/', '_home_/'), (r'^file:/', '')]
  docker_path = os.path.normpath(raw_path)
  for pattern, replacement in docker_rewrites:
    docker_path = re.sub(pattern, replacement, docker_path)
  docker_path = docker_path.lstrip('./')  # Strips any of '.' './' '/'.
  docker_path = directory_fmt('file/' + docker_path) + filename
  return normed_uri, docker_path

def _local_uri_rewriter ( raw_uri ) : raw_path , filename = os . path . split ( raw_uri ) prefix_replacements = [ ( 'file:///' , '/' ) , ( '~/' , os . getenv ( 'HOME' ) ) , ( './' , '' ) , ( 'file:/' , '/' ) ] normed_path = raw_path for prefix , replacement in prefix_replacements : if normed_path . startswith ( prefix ) : normed_path = os . path . join ( replacement , normed_path [ len ( prefix ) : ] ) normed_uri = directory_fmt ( os . path . abspath ( normed_path ) ) normed_uri = os . path . join ( normed_uri , filename ) docker_rewrites = [ ( r'/\.\.' , '/_dotdot_' ) , ( r'^\.\.' , '_dotdot_' ) , ( r'^~/' , '_home_/' ) , ( r'^file:/' , '' ) ] docker_path = os . path . normpath ( raw_path ) for pattern , replacement in docker_rewrites : docker_path = re . sub ( pattern , replacement , docker_path ) docker_path = docker_path . lstrip ( './' ) docker_path = directory_fmt ( 'file/' + docker_path ) + filename return normed_uri , docker_path

Rewrite local file URIs as required by the rewrite_uris method .


def _get_filtered_mounts(mounts, mount_param_type):
  """Helper function to return an appropriate set of mount parameters."""
  return set([mount for mount in mounts if isinstance(mount, mount_param_type)])

def _get_filtered_mounts ( mounts , mount_param_type ) : return set ( [ mount for mount in mounts if isinstance ( mount , mount_param_type ) ] )

Helper function to return an appropriate set of mount parameters .


def build_logging_param(logging_uri, util_class=OutputFileParamUtil):
  """Convenience function simplifies construction of the logging uri."""
  if not logging_uri:
    return job_model.LoggingParam(None, None)
  recursive = not logging_uri.endswith('.log')
  oututil = util_class('')
  _, uri, provider = oututil.parse_uri(logging_uri, recursive)
  if '*' in uri.basename:
    raise ValueError('Wildcards not allowed in logging URI: %s' % uri)
  return job_model.LoggingParam(uri, provider)

Convenience function simplifies construction of the logging uri .


def split_pair(pair_string, separator, nullable_idx=1):
  """Split a string into a pair, which can have one empty value.

  Args:
    pair_string: The string to be split.
    separator: The separator to be used for splitting.
    nullable_idx: The location to be set to null if the separator is not in the
                  input string. Should be either 0 or 1.

  Returns:
    A list containing the pair.

  Raises:
    IndexError: If nullable_idx is not 0 or 1.
  """

  pair = pair_string.split(separator, 1)
  if len(pair) == 1:
    if nullable_idx == 0:
      return [None, pair[0]]
    elif nullable_idx == 1:
      return [pair[0], None]
    else:
      raise IndexError('nullable_idx should be either 0 or 1.')
  else:
    return pair

def
split_pair
(
pair_string
,
separator
,
nullable_idx
=
1
)
:
pair
=
pair_string
.
split
(
separator
,
1
)
if
len
(
pair
)
==
1
:
if
nullable_idx
==
0
:
return
[
,
pair
[
0
]
]
elif
nullable_idx
==
1
:
return
[
pair
[
0
]
,
]
else
:
raise
IndexError
(
'nullable_idx should be either 0 or 1.'
)
else
:
return
pair

Split a string into a pair which can have one empty value .


def parse_tasks_file_header(header, input_file_param_util,
                            output_file_param_util):
  """Parse the header from the tasks file into env, input, output definitions.

  Elements are formatted similar to their equivalent command-line arguments,
  but with associated values coming from the data rows.

  Environment variables columns are headered as "--env <name>"
  Inputs columns are headered as "--input <name>" with the name optional.
  Outputs columns are headered as "--output <name>" with the name optional.

  For historical reasons, bareword column headers (such as "JOB_ID") are
  equivalent to "--env var_name".

  Args:
    header: Array of header fields
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.

  Returns:
    job_params: A list of EnvParams and FileParams for the environment
    variables, LabelParams, input file parameters, and output file parameters.

  Raises:
    ValueError: If a header contains a ":" and the prefix is not supported.
  """
  job_params = []

  for col in header:

    # Reserve the "-" and "--" namespace.
    # If the column has no leading "-", treat it as an environment variable
    col_type = '--env'
    col_value = col
    if col.startswith('-'):
      col_type, col_value = split_pair(col, ' ', 1)

    if col_type == '--env':
      job_params.append(job_model.EnvParam(col_value))

    elif col_type == '--label':
      job_params.append(job_model.LabelParam(col_value))

    elif col_type == '--input' or col_type == '--input-recursive':
      name = input_file_param_util.get_variable_name(col_value)
      job_params.append(
          job_model.InputFileParam(
              name, recursive=(col_type.endswith('recursive'))))

    elif col_type == '--output' or col_type == '--output-recursive':
      name = output_file_param_util.get_variable_name(col_value)
      job_params.append(
          job_model.OutputFileParam(
              name, recursive=(col_type.endswith('recursive'))))

    else:
      raise ValueError('Unrecognized column header: %s' % col)

  return job_params

def
parse_tasks_file_header
(
header
,
input_file_param_util
,
output_file_param_util
)
:
job_params
=
[
]
for
col
in
header
:
col_type
=
'--env'
col_value
=
col
if
col
.
startswith
(
'-'
)
:
col_type
,
col_value
=
split_pair
(
col
,
' '
,
1
)
if
col_type
==
'--env'
:
job_params
.
append
(
job_model
.
EnvParam
(
col_value
)
)
elif
col_type
==
'--label'
:
job_params
.
append
(
job_model
.
LabelParam
(
col_value
)
)
elif
col_type
==
'--input'
or
col_type
==
'--input-recursive'
:
name
=
input_file_param_util
.
get_variable_name
(
col_value
)
job_params
.
append
(
job_model
.
InputFileParam
(
name
,
recursive
=
(
col_type
.
endswith
(
'recursive'
)
)
)
)
elif
col_type
==
'--output'
or
col_type
==
'--output-recursive'
:
name
=
output_file_param_util
.
get_variable_name
(
col_value
)
job_params
.
append
(
job_model
.
OutputFileParam
(
name
,
recursive
=
(
col_type
.
endswith
(
'recursive'
)
)
)
)
else
:
raise
ValueError
(
'Unrecognized column header: %s'
%
col
)
return
job_params

Parse the header from the tasks file into env input output definitions .


def tasks_file_to_task_descriptors(tasks, retries, input_file_param_util,
                                   output_file_param_util):
  """Parses task parameters from a TSV.

  Args:
    tasks: Dict containing the path to a TSV file and task numbers to run
    variables, input, and output parameters as column headings. Subsequent
    lines specify parameter values, one row per job.
    retries: Number of retries allowed.
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.

  Returns:
    task_descriptors: an array of records, each containing the task-id,
    task-attempt, 'envs', 'inputs', 'outputs', 'labels' that defines the set of
    parameters for each task of the job.

  Raises:
    ValueError: If no job records were provided
  """
  task_descriptors = []

  path = tasks['path']
  task_min = tasks.get('min')
  task_max = tasks.get('max')

  # Load the file and set up a Reader that tokenizes the fields
  param_file = dsub_util.load_file(path)
  reader = csv.reader(param_file, delimiter='\t')

  # Read the first line and extract the parameters
  header = six.advance_iterator(reader)
  job_params = parse_tasks_file_header(header, input_file_param_util,
                                       output_file_param_util)

  # Build a list of records from the parsed input file
  for row in reader:
    # Tasks are numbered starting at 1 and since the first line of the TSV
    # file is a header, the first task appears on line 2.
    task_id = reader.line_num - 1
    if task_min and task_id < task_min:
      continue
    if task_max and task_id > task_max:
      continue

    if len(row) != len(job_params):
      dsub_util.print_error('Unexpected number of fields %s vs %s: line %s' %
                            (len(row), len(job_params), reader.line_num))

    # Each row can contain "envs", "inputs", "outputs"
    envs = set()
    inputs = set()
    outputs = set()
    labels = set()

    for i in range(0, len(job_params)):
      param = job_params[i]
      name = param.name
      if isinstance(param, job_model.EnvParam):
        envs.add(job_model.EnvParam(name, row[i]))

      elif isinstance(param, job_model.LabelParam):
        labels.add(job_model.LabelParam(name, row[i]))

      elif isinstance(param, job_model.InputFileParam):
        inputs.add(
            input_file_param_util.make_param(name, row[i], param.recursive))

      elif isinstance(param, job_model.OutputFileParam):
        outputs.add(
            output_file_param_util.make_param(name, row[i], param.recursive))

    task_descriptors.append(
        job_model.TaskDescriptor({
            'task-id': task_id,
            'task-attempt': 1 if retries else None
        }, {
            'labels': labels,
            'envs': envs,
            'inputs': inputs,
            'outputs': outputs
        }, job_model.Resources()))

  # Ensure that there are jobs to execute (and not just a header)
  if not task_descriptors:
    raise ValueError('No tasks added from %s' % path)

  return task_descriptors


def parse_pair_args(labels, argclass):
  """Parse flags of key=value pairs and return a list of argclass.

  For pair variables, we need to:
     * split the input into name=value pairs (value optional)
     * Create the EnvParam object

  Args:
    labels: list of 'key' or 'key=value' strings.
    argclass: Container class for args, must instantiate with argclass(k, v).

  Returns:
    list of argclass objects.
  """
  label_data = set()
  for arg in labels:
    name, value = split_pair(arg, '=', nullable_idx=1)
    label_data.add(argclass(name, value))
  return label_data

def parse_pair_args ( labels , argclass ) : label_data = set ( ) for arg in labels : name , value = split_pair ( arg , '=' , nullable_idx = 1 ) label_data . add ( argclass ( name , value ) ) return label_data

Parse flags of key = value pairs and return a list of argclass .


def args_to_job_params(envs, labels, inputs, inputs_recursive, outputs,
                       outputs_recursive, mounts, input_file_param_util,
                       output_file_param_util, mount_param_util):
  """Parse env, input, and output parameters into a job parameters and data.

  Passing arguments on the command-line allows for launching a single job.
  The env, input, and output arguments encode both the definition of the
  job as well as the single job's values.

  Env arguments are simple name=value pairs.
  Input and output file arguments can contain name=value pairs or just values.
  Either of the following is valid:

    uri
    myfile=uri

  Args:
    envs: list of environment variable job parameters
    labels: list of labels to attach to the tasks
    inputs: list of file input parameters
    inputs_recursive: list of recursive directory input parameters
    outputs: list of file output parameters
    outputs_recursive: list of recursive directory output parameters
    mounts: list of gcs buckets to mount
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.
    mount_param_util: Utility for producing MountParam objects.

  Returns:
    job_params: a dictionary of 'envs', 'inputs', and 'outputs' that defines the
    set of parameters and data for a job.
  """
  # Parse environmental variables and labels.
  env_data = parse_pair_args(envs, job_model.EnvParam)
  label_data = parse_pair_args(labels, job_model.LabelParam)

  # For input files, we need to:
  #   * split the input into name=uri pairs (name optional)
  #   * get the environmental variable name, or automatically set if null.
  #   * create the input file param
  input_data = set()
  for (recursive, args) in ((False, inputs), (True, inputs_recursive)):
    for arg in args:
      name, value = split_pair(arg, '=', nullable_idx=0)
      name = input_file_param_util.get_variable_name(name)
      input_data.add(input_file_param_util.make_param(name, value, recursive))

  # For output files, we need to:
  #   * split the input into name=uri pairs (name optional)
  #   * get the environmental variable name, or automatically set if null.
  #   * create the output file param
  output_data = set()
  for (recursive, args) in ((False, outputs), (True, outputs_recursive)):
    for arg in args:
      name, value = split_pair(arg, '=', 0)
      name = output_file_param_util.get_variable_name(name)
      output_data.add(output_file_param_util.make_param(name, value, recursive))

  mount_data = set()
  for arg in mounts:
    # Mounts can look like `--mount VAR=PATH` or `--mount VAR=PATH {num}`,
    # where num is the size of the disk in Gb. We assume a space is the
    # separator between path and disk size.
    if ' ' in arg:
      key_value_pair, disk_size = arg.split(' ')
      name, value = split_pair(key_value_pair, '=', 1)
      mount_data.add(mount_param_util.make_param(name, value, disk_size))
    else:
      name, value = split_pair(arg, '=', 1)
      mount_data.add(mount_param_util.make_param(name, value, disk_size=None))
  return {
      'envs': env_data,
      'inputs': input_data,
      'outputs': output_data,
      'labels': label_data,
      'mounts': mount_data,
  }

def
args_to_job_params
(
envs
,
labels
,
inputs
,
inputs_recursive
,
outputs
,
outputs_recursive
,
mounts
,
input_file_param_util
,
output_file_param_util
,
mount_param_util
)
:
env_data
=
parse_pair_args
(
envs
,
job_model
.
EnvParam
)
label_data
=
parse_pair_args
(
labels
,
job_model
.
LabelParam
)
input_data
=
set
(
)
for
(
recursive
,
args
)
in
(
(
False
,
inputs
)
,
(
True
,
inputs_recursive
)
)
:
for
arg
in
args
:
name
,
value
=
split_pair
(
arg
,
'='
,
nullable_idx
=
0
)
name
=
input_file_param_util
.
get_variable_name
(
name
)
input_data
.
add
(
input_file_param_util
.
make_param
(
name
,
value
,
recursive
)
)
output_data
=
set
(
)
for
(
recursive
,
args
)
in
(
(
False
,
outputs
)
,
(
True
,
outputs_recursive
)
)
:
for
arg
in
args
:
name
,
value
=
split_pair
(
arg
,
'='
,
0
)
name
=
output_file_param_util
.
get_variable_name
(
name
)
output_data
.
add
(
output_file_param_util
.
make_param
(
name
,
value
,
recursive
)
)
mount_data
=
set
(
)
for
arg
in
mounts
:
if
' '
in
arg
:
key_value_pair
,
disk_size
=
arg
.
split
(
' '
)
name
,
value
=
split_pair
(
key_value_pair
,
'='
,
1
)
mount_data
.
add
(
mount_param_util
.
make_param
(
name
,
value
,
disk_size
)
)
else
:
name
,
value
=
split_pair
(
arg
,
'='
,
1
)
mount_data
.
add
(
mount_param_util
.
make_param
(
name
,
value
,
disk_size
=
)
)
return
{
'envs'
:
env_data
,
'inputs'
:
input_data
,
'outputs'
:
output_data
,
'labels'
:
label_data
,
'mounts'
:
mount_data
,
}

Parse env input and output parameters into a job parameters and data .


def validate_submit_args_or_fail(job_descriptor, provider_name, input_providers,
                                 output_providers, logging_providers):
  """Validate that arguments passed to submit_job have valid file providers.

  This utility function takes resources and task data args from `submit_job`
  in the base provider. This function will fail with a value error if any of the
  parameters are not valid. See the following example;

  >>> job_resources = type('', (object,),
  ...    {"logging": job_model.LoggingParam('gs://logtemp', job_model.P_GCS)})()
  >>> job_params={'inputs': set(), 'outputs': set(), 'mounts': set()}
  >>> task_descriptors = [
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': {
  ...           job_model.FileParam('IN', uri='gs://in/*',
  ...                               file_provider=job_model.P_GCS)},
  ...       'outputs': set()}, None),
  ...     job_model.TaskDescriptor(None, {
  ...       'inputs': set(),
  ...       'outputs': {
  ...           job_model.FileParam('OUT', uri='gs://out/*',
  ...                               file_provider=job_model.P_GCS)}}, None)]
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_GCS],
  ...                              logging_providers=[job_model.P_GCS])
  ...
  >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
  ...                              job_resources, task_descriptors),
  ...                              provider_name='MYPROVIDER',
  ...                              input_providers=[job_model.P_GCS],
  ...                              output_providers=[job_model.P_LOCAL],
  ...                              logging_providers=[job_model.P_GCS])
  Traceback (most recent call last):
       ...
  ValueError: Unsupported output path (gs://out/*) for provider 'MYPROVIDER'.

  Args:
    job_descriptor: instance of job_model.JobDescriptor.
    provider_name: (str) the name of the execution provider.
    input_providers: (string collection) whitelist of file providers for input.
    output_providers: (string collection) whitelist of providers for output.
    logging_providers: (string collection) whitelist of providers for logging.

  Raises:
    ValueError: if any file providers do not match the whitelists.
  """
  job_resources = job_descriptor.job_resources
  job_params = job_descriptor.job_params
  task_descriptors = job_descriptor.task_descriptors

  # Validate logging file provider.
  _validate_providers([job_resources.logging], 'logging', logging_providers,
                      provider_name)

  # Validate job input and output file providers
  _validate_providers(job_params['inputs'], 'input', input_providers,
                      provider_name)
  _validate_providers(job_params['outputs'], 'output', output_providers,
                      provider_name)

  # Validate input and output file providers.
  for task_descriptor in task_descriptors:
    _validate_providers(task_descriptor.task_params['inputs'], 'input',
                        input_providers, provider_name)
    _validate_providers(task_descriptor.task_params['outputs'], 'output',
                        output_providers, provider_name)

Validate that arguments passed to submit_job have valid file providers .


def _interval_to_seconds(interval, valid_units='smhdw'):
  """Convert the timeout duration to seconds.

  The value must be of the form "<integer><unit>" where supported
  units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

  Args:
    interval: A "<integer><unit>" string.
    valid_units: A list of supported units.

  Returns:
    A string of the form "<integer>s" or None if timeout is empty.
  """
  if not interval:
    return None

  try:
    last_char = interval[-1]

    if last_char == 's' and 's' in valid_units:
      return str(float(interval[:-1])) + 's'
    elif last_char == 'm' and 'm' in valid_units:
      return str(float(interval[:-1]) * 60) + 's'
    elif last_char == 'h' and 'h' in valid_units:
      return str(float(interval[:-1]) * 60 * 60) + 's'
    elif last_char == 'd' and 'd' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24) + 's'
    elif last_char == 'w' and 'w' in valid_units:
      return str(float(interval[:-1]) * 60 * 60 * 24 * 7) + 's'
    else:
      raise ValueError(
          'Unsupported units in interval string %s: %s' % (interval, last_char))

  except (ValueError, OverflowError) as e:
    raise ValueError('Unable to parse interval string %s: %s' % (interval, e))

Convert the timeout duration to seconds .


def get_variable_name(self, name):
    """Produce a default variable name if none is specified."""
    if not name:
      name = '%s%s' % (self._auto_prefix, self._auto_index)
      self._auto_index += 1
    return name

def get_variable_name ( self , name ) : if not name : name = '%s%s' % ( self . _auto_prefix , self . _auto_index ) self . _auto_index += 1 return name

Produce a default variable name if none is specified .


def rewrite_uris(self, raw_uri, file_provider):
    """Accept a raw uri and return rewritten versions.

    This function returns a normalized URI and a docker path. The normalized
    URI may have minor alterations meant to disambiguate and prepare for use
    by shell utilities that may require a specific format.

    The docker rewriter makes substantial modifications to the raw URI when
    constructing a docker path, but modifications must follow these rules:
      1) System specific characters are not allowed (ex. indirect paths).
      2) The path, if it is a directory, must end in a forward slash.
      3) The path will begin with the value set in self._relative_path.
      4) The path will have an additional prefix (after self._relative_path) set
         by the file provider-specific rewriter.

    Rewrite output for the docker path:
      >>> out_util = FileParamUtil('AUTO_', 'output')
      >>> out_util.rewrite_uris('gs://mybucket/myfile.txt', job_model.P_GCS)[1]
      'output/gs/mybucket/myfile.txt'
      >>> out_util.rewrite_uris('./data/myfolder/', job_model.P_LOCAL)[1]
      'output/file/data/myfolder/'

    When normalizing the URI for cloud buckets, no rewrites are done. For local
    files, the user directory will be expanded and relative paths will be
    converted to absolute:
      >>> in_util = FileParamUtil('AUTO_', 'input')
      >>> in_util.rewrite_uris('gs://mybucket/gcs_dir/', job_model.P_GCS)[0]
      'gs://mybucket/gcs_dir/'
      >>> in_util.rewrite_uris('/data/./dir_a/../myfile.txt',
      ...   job_model.P_LOCAL)[0]
      '/data/myfile.txt'
      >>> in_util.rewrite_uris('file:///tmp/data/*.bam', job_model.P_LOCAL)[0]
      '/tmp/data/*.bam'

    Args:
      raw_uri: (str) the path component of the raw URI.
      file_provider: a valid provider (contained in job_model.FILE_PROVIDERS).

    Returns:
      normalized: a cleaned version of the uri provided by command line.
      docker_path: the uri rewritten in the format required for mounting inside
                   a docker worker.

    Raises:
      ValueError: if file_provider is not valid.
    """
    if file_provider == job_model.P_GCS:
      normalized, docker_path = _gcs_uri_rewriter(raw_uri)
    elif file_provider == job_model.P_LOCAL:
      normalized, docker_path = _local_uri_rewriter(raw_uri)
    else:
      raise ValueError('File provider not supported: %r' % file_provider)
    return normalized, os.path.join(self._relative_path, docker_path)

def
rewrite_uris
(
self
,
raw_uri
,
file_provider
)
:
if
file_provider
==
job_model
.
P_GCS
:
normalized
,
docker_path
=
_gcs_uri_rewriter
(
raw_uri
)
elif
file_provider
==
job_model
.
P_LOCAL
:
normalized
,
docker_path
=
_local_uri_rewriter
(
raw_uri
)
else
:
raise
ValueError
(
'File provider not supported: %r'
%
file_provider
)
return
normalized
,
os
.
path
.
join
(
self
.
_relative_path
,
docker_path
)

Accept a raw uri and return rewritten versions .


def parse_file_provider(uri):
    """Find the file provider for a URI."""
    providers = {'gs': job_model.P_GCS, 'file': job_model.P_LOCAL}
    # URI scheme detector uses a range up to 30 since none of the IANA
    # registered schemes are longer than this.
    provider_found = re.match(r'^([A-Za-z][A-Za-z0-9+.-]{0,29})://', uri)
    if provider_found:
      prefix = provider_found.group(1).lower()
    else:
      # If no provider is specified in the URI, assume that the local
      # filesystem is being used. Availability and validity of the local
      # file/directory will be checked later.
      prefix = 'file'
    if prefix in providers:
      return providers[prefix]
    else:
      raise ValueError('File prefix not supported: %s://' % prefix)

def
parse_file_provider
(
uri
)
:
providers
=
{
'gs'
:
job_model
.
P_GCS
,
'file'
:
job_model
.
P_LOCAL
}
provider_found
=
re
.
match
(
r'^([A-Za-z][A-Za-z0-9+.-]{0,29})://'
,
uri
)
if
provider_found
:
prefix
=
provider_found
.
group
(
1
)
.
lower
(
)
else
:
prefix
=
'file'
if
prefix
in
providers
:
return
providers
[
prefix
]
else
:
raise
ValueError
(
'File prefix not supported: %s://'
%
prefix
)


def _validate_paths_or_fail(uri, recursive):
    """Do basic validation of the uri, return the path and filename."""
    path, filename = os.path.split(uri)

    # dsub could support character ranges ([0-9]) with some more work, but for
    # now we assume that basic asterisk wildcards are sufficient. Reject any URI
    # that includes square brackets or question marks, since we know that
    # if they actually worked, it would be accidental.
    if '[' in uri or ']' in uri:
      raise ValueError(
          'Square bracket (character ranges) are not supported: %s' % uri)
    if '?' in uri:
      raise ValueError('Question mark wildcards are not supported: %s' % uri)

    # Only support file URIs and *filename* wildcards
    # Wildcards at the directory level or "**" syntax would require better
    # support from the Pipelines API *or* doing expansion here and
    # (potentially) producing a series of FileParams, instead of one.
    if '*' in path:
      raise ValueError(
          'Path wildcard (*) are only supported for files: %s' % uri)
    if '**' in filename:
      raise ValueError('Recursive wildcards ("**") not supported: %s' % uri)
    if filename in ('..', '.'):
      raise ValueError('Path characters ".." and "." not supported '
                       'for file names: %s' % uri)

    # Do not allow non-recursive IO to reference directories.
    if not recursive and not filename:
      raise ValueError('Input or output values that are not recursive must '
                       'reference a filename or wildcard: %s' % uri)

Do basic validation of the uri return the path and filename .


def parse_uri(self, raw_uri, recursive):
    """Return a valid docker_path, uri, and file provider from a flag value."""
    # Assume recursive URIs are directory paths.
    if recursive:
      raw_uri = directory_fmt(raw_uri)
    # Get the file provider, validate the raw URI, and rewrite the path
    # component of the URI for docker and remote.
    file_provider = self.parse_file_provider(raw_uri)
    self._validate_paths_or_fail(raw_uri, recursive)
    uri, docker_uri = self.rewrite_uris(raw_uri, file_provider)
    uri_parts = job_model.UriParts(
        directory_fmt(os.path.dirname(uri)), os.path.basename(uri))
    return docker_uri, uri_parts, file_provider

def parse_uri ( self , raw_uri , recursive ) : if recursive : raw_uri = directory_fmt ( raw_uri ) file_provider = self . parse_file_provider ( raw_uri ) self . _validate_paths_or_fail ( raw_uri , recursive ) uri , docker_uri = self . rewrite_uris ( raw_uri , file_provider ) uri_parts = job_model . UriParts ( directory_fmt ( os . path . dirname ( uri ) ) , os . path . basename ( uri ) ) return docker_uri , uri_parts , file_provider

Return a valid docker_path uri and file provider from a flag value .


def _parse_image_uri(self, raw_uri):
    """Return a valid docker_path from a Google Persistent Disk url."""
    # The string replace is so we don't have colons and double slashes in the
    # mount path. The idea is the resulting mount path would look like:
    # /mnt/data/mount/http/www.googleapis.com/compute/v1/projects/...
    docker_uri = os.path.join(self._relative_path,
                              raw_uri.replace('https://', 'https/', 1))
    return docker_uri

def _parse_image_uri ( self , raw_uri ) : docker_uri = os . path . join ( self . _relative_path , raw_uri . replace ( 'https://' , 'https/' , 1 ) ) return docker_uri

Return a valid docker_path from a Google Persistent Disk url .


def _parse_local_mount_uri(self, raw_uri):
    """Return a valid docker_path for a local file path."""
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _local_uri_rewriter(raw_uri)
    local_path = docker_path[len('file'):]
    docker_uri = os.path.join(self._relative_path, docker_path)
    return local_path, docker_uri

def _parse_local_mount_uri ( self , raw_uri ) : raw_uri = directory_fmt ( raw_uri ) _ , docker_path = _local_uri_rewriter ( raw_uri ) local_path = docker_path [ len ( 'file' ) : ] docker_uri = os . path . join ( self . _relative_path , docker_path ) return local_path , docker_uri

Return a valid docker_path for a local file path .


def _parse_gcs_uri(self, raw_uri):
    """Return a valid docker_path for a GCS bucket."""
    # Assume URI is a directory path.
    raw_uri = directory_fmt(raw_uri)
    _, docker_path = _gcs_uri_rewriter(raw_uri)
    docker_uri = os.path.join(self._relative_path, docker_path)
    return docker_uri

Return a valid docker_path for a GCS bucket .


def make_param(self, name, raw_uri, disk_size):
    """Return a MountParam given a GCS bucket, disk image or local path."""
    if raw_uri.startswith('https://www.googleapis.com/compute'):
      # Full Image URI should look something like:
      # https://www.googleapis.com/compute/v1/projects/<project>/global/images/
      # But don't validate further, should the form of a valid image URI
      # change (v1->v2, for example)
      docker_path = self._parse_image_uri(raw_uri)
      return job_model.PersistentDiskMountParam(
          name, raw_uri, docker_path, disk_size, disk_type=None)
    elif raw_uri.startswith('file://'):
      local_path, docker_path = self._parse_local_mount_uri(raw_uri)
      return job_model.LocalMountParam(name, raw_uri, docker_path, local_path)
    elif raw_uri.startswith('gs://'):
      docker_path = self._parse_gcs_uri(raw_uri)
      return job_model.GCSMountParam(name, raw_uri, docker_path)
    else:
      raise ValueError(
          'Mount parameter {} must begin with valid prefix.'.format(raw_uri))

Return a MountParam given a GCS bucket disk image or local path .


def validate_param_name(name, param_type):
  """Validate that the name follows posix conventions for env variables."""
  # http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_235
  #
  # 3.235 Name
  # In the shell command language, a word consisting solely of underscores,
  # digits, and alphabetics from the portable character set.
  if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
    raise ValueError('Invalid %s: %s' % (param_type, name))

def
validate_param_name
(
name
,
param_type
)
:
if
not
re
.
match
(
r'^[a-zA-Z_][a-zA-Z0-9_]*$'
,
name
)
:
raise
ValueError
(
'Invalid %s: %s'
%
(
param_type
,
name
)
)

Validate that the name follows posix conventions for env variables .


def validate_bucket_name(bucket):
  """Validate that the name is a valid GCS bucket."""
  if not bucket.startswith('gs://'):
    raise ValueError(
        'Invalid bucket path "%s". Must start with "gs://".' % bucket)
  bucket_name = bucket[len('gs://'):]
  if not re.search(r'^\w[\w_\.-]{1,61}\w$', bucket_name):
    raise ValueError('Invalid bucket name: %s' % bucket)

Validate that the name is a valid GCS bucket .


def convert_to_label_chars(s):
  """Turn the specified name and value into a valid Google label."""

  # We want the results to be user-friendly, not just functional.
  # So we can't base-64 encode it.
  #   * If upper-case: lower-case it
  #   * If the char is not a standard letter or digit. make it a dash

  # March 2019 note: underscores are now allowed in labels.
  # However, removing the conversion of underscores to dashes here would
  # create inconsistencies between old jobs and new jobs.
  # With existing code, $USER "jane_doe" has a user-id label of "jane-doe".
  # If we remove the conversion, the user-id label for new jobs is "jane_doe".
  # This makes looking up old jobs more complicated.

  accepted_characters = string.ascii_lowercase + string.digits + '-'

  def label_char_transform(char):
    if char in accepted_characters:
      return char
    if char in string.ascii_uppercase:
      return char.lower()
    return '-'

  return ''.join(label_char_transform(c) for c in s)

def convert_to_label_chars ( s ) : accepted_characters = string . ascii_lowercase + string . digits + '-' def label_char_transform ( char ) : if char in accepted_characters : return char if char in string . ascii_uppercase : return char . lower ( ) return '-' return '' . join ( label_char_transform ( c ) for c in s )

Turn the specified name and value into a valid Google label .


def ensure_task_params_are_complete(task_descriptors):
  """For each task, ensure that each task param entry is not None."""
  for task_desc in task_descriptors:
    for param in [
        'labels', 'envs', 'inputs', 'outputs', 'input-recursives',
        'output-recursives'
    ]:
      if not task_desc.task_params.get(param):
        task_desc.task_params[param] = set()

def ensure_task_params_are_complete ( task_descriptors ) : for task_desc in task_descriptors : for param in [ 'labels' , 'envs' , 'inputs' , 'outputs' , 'input-recursives' , 'output-recursives' ] : if not task_desc . task_params . get ( param ) : task_desc . task_params [ param ] = set ( )

For each task ensure that each task param entry is not None .


def _remove_empty_items(d, required):
  """Return a new dict with any empty items removed.

  Note that this is not a deep check. If d contains a dictionary which
  itself contains empty items, those are never checked.

  This method exists to make to_serializable() functions cleaner.
  We could revisit this some day, but for now, the serialized objects are
  stripped of empty values to keep the output YAML more compact.

  Args:
    d: a dictionary
    required: list of required keys (for example, TaskDescriptors always emit
      the "task-id", even if None)

  Returns:
    A dictionary with empty items removed.
  """

  new_dict = {}
  for k, v in d.items():
    if k in required:
      new_dict[k] = v
    elif isinstance(v, int) or v:
      # "if v" would suppress emitting int(0)
      new_dict[k] = v

  return new_dict

def _remove_empty_items ( d , required ) : new_dict = { } for k , v in d . items ( ) : if k in required : new_dict [ k ] = v elif isinstance ( v , int ) or v : new_dict [ k ] = v return new_dict

Return a new dict with any empty items removed .


def task_view_generator(job_descriptor):
  """Generator that yields a task-specific view of the job.

  This generator exists to make it easy for callers to iterate over the tasks
  in a JobDescriptor. Each pass yields a new JobDescriptor with a single task.

  Args:
    job_descriptor: A JobDescriptor with 1 or more tasks.

  Yields:
    A JobDescriptor with a single task.
  """
  for task_descriptor in job_descriptor.task_descriptors:
    jd = JobDescriptor(job_descriptor.job_metadata, job_descriptor.job_params,
                       job_descriptor.job_resources, [task_descriptor])
    yield jd

Generator that yields a task - specific view of the job .


def numeric_task_id(task_id):
  """Converts a task-id to the numeric task-id.

  Args:
    task_id: task-id in either task-n or n format

  Returns:
    n
  """

  # This function exists to support the legacy "task-id" format in the "google"
  # provider. Google labels originally could not be numeric. When the google
  # provider is completely replaced by the google-v2 provider, this function can
  # go away.

  if task_id is not None:
    if task_id.startswith('task-'):
      return int(task_id[len('task-'):])
    else:
      return int(task_id)

def numeric_task_id ( task_id ) : if task_id is not None : if task_id . startswith ( 'task-' ) : return int ( task_id [ len ( 'task-' ) : ] ) else : return int ( task_id )

Converts a task - id to the numeric task - id .


def _validate_label(cls, name, value):
    """Raise ValueError if the label is invalid."""
    # Rules for labels are described in:
    #  https://cloud.google.com/compute/docs/labeling-resources#restrictions

    # * Keys and values cannot be longer than 63 characters each.
    # * Keys and values can only contain lowercase letters, numeric characters,
    #   underscores, and dashes.
    # * International characters are allowed.
    # * Label keys must start with a lowercase letter and international
    #   characters are allowed.
    # * Label keys cannot be empty.
    cls._check_label_name(name)
    cls._check_label_value(value)

    # Ensure that reserved labels are not being used.
    if not cls._allow_reserved_keys and name in RESERVED_LABELS:
      raise ValueError('Label flag (%s=...) must not use reserved keys: %r' %
                       (name, list(RESERVED_LABELS)))

Raise ValueError if the label is invalid .


def _from_yaml_v0(cls, job):
    """Populate a JobDescriptor from the local provider's original meta.yaml.

    The local job provider had the first incarnation of a YAML file for each
    task. That idea was extended here in the JobDescriptor and the local
    provider adopted the JobDescriptor.to_yaml() call to write its meta.yaml.

    The JobDescriptor.from_yaml() detects if it receives a local provider's
    "v0" meta.yaml and calls this function.

    Args:
      job: an object produced from decoding meta.yaml.

    Returns:
      A JobDescriptor populated as best we can from the old meta.yaml.
    """

    # The v0 meta.yaml only contained:
    #   create-time, job-id, job-name, logging, task-id
    #   labels, envs, inputs, outputs
    # It did NOT contain user-id.
    # dsub-version might be there as a label.

    job_metadata = {}
    for key in ['job-id', 'job-name', 'create-time']:
      job_metadata[key] = job.get(key)

    # Make sure that create-time string is turned into a datetime
    job_metadata['create-time'] = dsub_util.replace_timezone(
        datetime.datetime.strptime(job['create-time'], '%Y-%m-%d %H:%M:%S.%f'),
        tzlocal())

    # The v0 meta.yaml contained a "logging" field which was the task-specific
    # logging path. It did not include the actual "--logging" value the user
    # specified.
    job_resources = Resources()

    # The v0 meta.yaml represented a single task.
    # It did not distinguish whether params were job params or task params.
    # We will treat them as either all job params or all task params, based on
    # whether the task-id is empty or an integer value.
    #
    # We also cannot distinguish whether inputs/outputs were recursive or not.
    # Just treat them all as non-recursive.
    params = {}

    # The dsub-version may be in the meta.yaml as a label. If so remove it
    # and set it as a top-level job metadata value.
    labels = job.get('labels', {})
    if 'dsub-version' in labels:
      job_metadata['dsub-version'] = labels['dsub-version']
      del labels['dsub-version']
    params['labels'] = cls._label_params_from_dict(labels)

    params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
    params['inputs'] = cls._input_file_params_from_dict(
        job.get('inputs', {}), False)
    params['outputs'] = cls._output_file_params_from_dict(
        job.get('outputs', {}), False)

    if job.get('task-id') is None:
      job_params = params
      task_metadata = {'task-id': None}
      task_params = {}
    else:
      job_params = {}
      task_metadata = {'task-id': str(job.get('task-id'))}
      task_params = params

    task_resources = Resources(logging_path=job.get('logging'))

    task_descriptors = [
        TaskDescriptor.get_complete_descriptor(task_metadata, task_params,
                                               task_resources)
    ]

    return JobDescriptor.get_complete_descriptor(
        job_metadata, job_params, job_resources, task_descriptors)

def
_from_yaml_v0
(
cls
,
job
)
:
job_metadata
=
{
}
for
key
in
[
'job-id'
,
'job-name'
,
'create-time'
]
:
job_metadata
[
key
]
=
job
.
get
(
key
)
job_metadata
[
'create-time'
]
=
dsub_util
.
replace_timezone
(
datetime
.
datetime
.
strptime
(
job
[
'create-time'
]
,
'%Y-%m-%d %H:%M:%S.%f'
)
,
tzlocal
(
)
)
job_resources
=
Resources
(
)
params
=
{
}
labels
=
job
.
get
(
'labels'
,
{
}
)
if
'dsub-version'
in
labels
:
job_metadata
[
'dsub-version'
]
=
labels
[
'dsub-version'
]
del
labels
[
'dsub-version'
]
params
[
'labels'
]
=
cls
.
_label_params_from_dict
(
labels
)
params
[
'envs'
]
=
cls
.
_env_params_from_dict
(
job
.
get
(
'envs'
,
{
}
)
)
params
[
'inputs'
]
=
cls
.
_input_file_params_from_dict
(
job
.
get
(
'inputs'
,
{
}
)
,
False
)
params
[
'outputs'
]
=
cls
.
_output_file_params_from_dict
(
job
.
get
(
'outputs'
,
{
}
)
,
False
)
if
job
.
get
(
'task-id'
)
is
:
job_params
=
params
task_metadata
=
{
'task-id'
:
}
task_params
=
{
}
else
:
job_params
=
{
}
task_metadata
=
{
'task-id'
:
str
(
job
.
get
(
'task-id'
)
)
}
task_params
=
params
task_resources
=
Resources
(
logging_path
=
job
.
get
(
'logging'
)
)
task_descriptors
=
[
TaskDescriptor
.
get_complete_descriptor
(
task_metadata
,
task_params
,
task_resources
)
]
return
JobDescriptor
.
get_complete_descriptor
(
job_metadata
,
job_params
,
job_resources
,
task_descriptors
)

Populate a JobDescriptor from the local provider s original meta . yaml .


def from_yaml(cls, yaml_string):
    """Populate and return a JobDescriptor from a YAML string."""
    try:
      job = yaml.full_load(yaml_string)
    except AttributeError:
      # For installations that cannot update their PyYAML version
      job = yaml.load(yaml_string)

    # If the YAML does not contain a top-level dsub version, then assume that
    # the string is coming from the local provider, reading an old version of
    # its meta.yaml.
    dsub_version = job.get('dsub-version')
    if not dsub_version:
      return cls._from_yaml_v0(job)

    job_metadata = {}
    for key in [
        'job-id', 'job-name', 'task-ids', 'user-id', 'dsub-version',
        'user-project', 'script-name'
    ]:
      if job.get(key) is not None:
        job_metadata[key] = job.get(key)

    # Make sure that create-time string is turned into a datetime
    job_metadata['create-time'] = dsub_util.replace_timezone(
        job.get('create-time'), pytz.utc)

    job_resources = Resources(logging=job.get('logging'))

    job_params = {}
    job_params['labels'] = cls._label_params_from_dict(job.get('labels', {}))
    job_params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
    job_params['inputs'] = cls._input_file_params_from_dict(
        job.get('inputs', {}), False)
    job_params['input-recursives'] = cls._input_file_params_from_dict(
        job.get('input-recursives', {}), True)
    job_params['outputs'] = cls._output_file_params_from_dict(
        job.get('outputs', {}), False)
    job_params['output-recursives'] = cls._output_file_params_from_dict(
        job.get('output-recursives', {}), True)
    job_params['mounts'] = cls._mount_params_from_dict(job.get('mounts', {}))

    task_descriptors = []
    for task in job.get('tasks', []):
      task_metadata = {'task-id': task.get('task-id')}

      # Old instances of the meta.yaml do not have a task create time.
      create_time = task.get('create-time')
      if create_time:
        task_metadata['create-time'] = dsub_util.replace_timezone(
            create_time, pytz.utc)

      if task.get('task-attempt') is not None:
        task_metadata['task-attempt'] = task.get('task-attempt')

      task_params = {}
      task_params['labels'] = cls._label_params_from_dict(
          task.get('labels', {}))
      task_params['envs'] = cls._env_params_from_dict(task.get('envs', {}))
      task_params['inputs'] = cls._input_file_params_from_dict(
          task.get('inputs', {}), False)
      task_params['input-recursives'] = cls._input_file_params_from_dict(
          task.get('input-recursives', {}), True)
      task_params['outputs'] = cls._output_file_params_from_dict(
          task.get('outputs', {}), False)
      task_params['output-recursives'] = cls._output_file_params_from_dict(
          task.get('output-recursives', {}), True)

      task_resources = Resources(logging_path=task.get('logging-path'))

      task_descriptors.append(
          TaskDescriptor(task_metadata, task_params, task_resources))

    return JobDescriptor(job_metadata, job_params, job_resources,
                         task_descriptors)

def from_yaml ( cls , yaml_string ) : try : job = yaml . full_load ( yaml_string ) except AttributeError : job = yaml . load ( yaml_string ) dsub_version = job . get ( 'dsub-version' ) if not dsub_version : return cls . _from_yaml_v0 ( job ) job_metadata = { } for key in [ 'job-id' , 'job-name' , 'task-ids' , 'user-id' , 'dsub-version' , 'user-project' , 'script-name' ] : if job . get ( key ) is not None : job_metadata [ key ] = job . get ( key ) job_metadata [ 'create-time' ] = dsub_util . replace_timezone ( job . get ( 'create-time' ) , pytz . utc ) job_resources = Resources ( logging = job . get ( 'logging' ) ) job_params = { } job_params [ 'labels' ] = cls . _label_params_from_dict ( job . get ( 'labels' , { } ) ) job_params [ 'envs' ] = cls . _env_params_from_dict ( job . get ( 'envs' , { } ) ) job_params [ 'inputs' ] = cls . _input_file_params_from_dict ( job . get ( 'inputs' , { } ) , False ) job_params [ 'input-recursives' ] = cls . _input_file_params_from_dict ( job . get ( 'input-recursives' , { } ) , True ) job_params [ 'outputs' ] = cls . _output_file_params_from_dict ( job . get ( 'outputs' , { } ) , False ) job_params [ 'output-recursives' ] = cls . _output_file_params_from_dict ( job . get ( 'output-recursives' , { } ) , True ) job_params [ 'mounts' ] = cls . _mount_params_from_dict ( job . get ( 'mounts' , { } ) ) task_descriptors = [ ] for task in job . get ( 'tasks' , [ ] ) : task_metadata = { 'task-id' : task . get ( 'task-id' ) } create_time = task . get ( 'create-time' ) if create_time : task_metadata [ 'create-time' ] = dsub_util . replace_timezone ( create_time , pytz . utc ) if task . get ( 'task-attempt' ) is not None : task_metadata [ 'task-attempt' ] = task . get ( 'task-attempt' ) task_params = { } task_params [ 'labels' ] = cls . _label_params_from_dict ( task . get ( 'labels' , { } ) ) task_params [ 'envs' ] = cls . _env_params_from_dict ( task . get ( 'envs' , { } ) ) task_params [ 'inputs' ] = cls . _input_file_params_from_dict ( task . get ( 'inputs' , { } ) , False ) task_params [ 'input-recursives' ] = cls . _input_file_params_from_dict ( task . get ( 'input-recursives' , { } ) , True ) task_params [ 'outputs' ] = cls . _output_file_params_from_dict ( task . get ( 'outputs' , { } ) , False ) task_params [ 'output-recursives' ] = cls . _output_file_params_from_dict ( task . get ( 'output-recursives' , { } ) , True ) task_resources = Resources ( logging_path = task . get ( 'logging-path' ) ) task_descriptors . append ( TaskDescriptor ( task_metadata , task_params , task_resources ) ) return JobDescriptor ( job_metadata , job_params , job_resources , task_descriptors )

Populate and return a JobDescriptor from a YAML string .


def find_task_descriptor(self, task_id):
    """Returns the task_descriptor corresponding to task_id."""

    # It is not guaranteed that the index will be task_id - 1 when --tasks is
    # used with a min/max range.
    for task_descriptor in self.task_descriptors:
      if task_descriptor.task_metadata.get('task-id') == task_id:
        return task_descriptor
    return None

def find_task_descriptor ( self , task_id ) : for task_descriptor in self . task_descriptors : if task_descriptor . task_metadata . get ( 'task-id' ) == task_id : return task_descriptor return None

Returns the task_descriptor corresponding to task_id .


def get_file_environment_variables(file_params):
  """Return a dictionary of environment variables for the user container."""
  env = {}
  for param in file_params:
    # We have no cases where the environment variable provided to user
    # scripts have a trailing slash, so be sure to always strip it.
    # The case that this is specifically handling is --input-recursive and
    # --output-recursive variables, which are directory values.
    env[param.name] = os.path.join(
        DATA_MOUNT_POINT, param.docker_path.rstrip('/')) if param.value else ''
  return env

def get_file_environment_variables ( file_params ) : env = { } for param in file_params : env [ param . name ] = os . path . join ( DATA_MOUNT_POINT , param . docker_path . rstrip ( '/' ) ) if param . value else '' return env

Return a dictionary of environment variables for the user container .


def get_job_and_task_param(job_params, task_params, field):
  """Returns a dict combining the field for job and task params."""
  return job_params.get(field, set()) | task_params.get(field, set())

def get_job_and_task_param ( job_params , task_params , field ) : return job_params . get ( field , set ( ) ) | task_params . get ( field , set ( ) )

Returns a dict combining the field for job and task params .


def _emit_search_criteria(user_ids, job_ids, task_ids, labels):
  """Print the filters used to delete tasks. Use raw flags as arguments."""
  print('Delete running jobs:')
  print('  user:')
  print('    %s\n' % user_ids)
  print('  job-id:')
  print('    %s\n' % job_ids)
  if task_ids:
    print('  task-id:')
    print('    %s\n' % task_ids)
  # Labels are in a LabelParam namedtuple and must be reformated for printing.
  if labels:
    print('  labels:')
    print('    %s\n' % repr(labels))

def
_emit_search_criteria
(
user_ids
,
job_ids
,
task_ids
,
labels
)
:
print
(
'Delete running jobs:'
)
print
(
'  user:'
)
print
(
'    %s\n'
%
user_ids
)
print
(
'  job-id:'
)
print
(
'    %s\n'
%
job_ids
)
if
task_ids
:
print
(
'  task-id:'
)
print
(
'    %s\n'
%
task_ids
)
if
labels
:
print
(
'  labels:'
)
print
(
'    %s\n'
%
repr
(
labels
)
)

Print the filters used to delete tasks . Use raw flags as arguments .


def ddel_tasks(provider,
               user_ids=None,
               job_ids=None,
               task_ids=None,
               labels=None,
               create_time_min=None,
               create_time_max=None):
  """Kill jobs or job tasks.

  This function separates ddel logic from flag parsing and user output. Users
  of ddel who intend to access the data programmatically should use this.

  Args:
    provider: an instantiated dsub provider.
    user_ids: a set of user ids who "own" the job(s) to delete.
    job_ids: a set of job ids to delete.
    task_ids: a set of task ids to delete.
    labels: a set of LabelParam, each must match the job(s) to be cancelled.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.

  Returns:
    list of job ids which were deleted.
  """
  # Delete the requested jobs
  deleted_tasks, error_messages = provider.delete_jobs(
      user_ids, job_ids, task_ids, labels, create_time_min, create_time_max)

  # Emit any errors canceling jobs
  for msg in error_messages:
    print(msg)

  return deleted_tasks

def ddel_tasks ( provider , user_ids = None , job_ids = None , task_ids = None , labels = None , create_time_min = None , create_time_max = None ) : deleted_tasks , error_messages = provider . delete_jobs ( user_ids , job_ids , task_ids , labels , create_time_min , create_time_max ) for msg in error_messages : print ( msg ) return deleted_tasks


def get_action_by_id(op, action_id):
  """Return the operation's array of actions."""
  actions = get_actions(op)
  if actions and 1 <= action_id < len(actions):
    return actions[action_id - 1]

def get_action_by_id ( op , action_id ) : actions = get_actions ( op ) if actions and 1 <= action_id < len ( actions ) : return actions [ action_id - 1 ]

Return the operation s array of actions .


def _get_action_by_name(op, name):
  """Return the value for the specified action."""
  actions = get_actions(op)
  for action in actions:
    if action.get('name') == name:
      return action

def _get_action_by_name ( op , name ) : actions = get_actions ( op ) for action in actions : if action . get ( 'name' ) == name : return action

Return the value for the specified action .


def get_action_environment(op, name):
  """Return the environment for the operation."""
  action = _get_action_by_name(op, name)
  if action:
    return action.get('environment')

def get_action_environment ( op , name ) : action = _get_action_by_name ( op , name ) if action : return action . get ( 'environment' )

Return the environment for the operation .


def get_action_image(op, name):
  """Return the image for the operation."""
  action = _get_action_by_name(op, name)
  if action:
    return action.get('imageUri')

def get_action_image ( op , name ) : action = _get_action_by_name ( op , name ) if action : return action . get ( 'imageUri' )


def get_event_of_type(op, event_type):
  """Return all events of a particular type."""
  events = get_events(op)
  if not events:
    return None

  return [e for e in events if e.get('details', {}).get('@type') == event_type]

def get_event_of_type ( op , event_type ) : events = get_events ( op ) if not events : return None return [ e for e in events if e . get ( 'details' , { } ) . get ( '@type' ) == event_type ]

Return all events of a particular type .


def get_last_update(op):
  """Return the most recent timestamp in the operation."""
  last_update = get_end_time(op)

  if not last_update:
    last_event = get_last_event(op)
    if last_event:
      last_update = last_event['timestamp']

  if not last_update:
    last_update = get_create_time(op)

  return last_update

def get_last_update ( op ) : last_update = get_end_time ( op ) if not last_update : last_event = get_last_event ( op ) if last_event : last_update = last_event [ 'timestamp' ] if not last_update : last_update = get_create_time ( op ) return last_update

Return the most recent timestamp in the operation .