def run_command(cmd, sudo=False):
    '''run_command uses subprocess to send a command to the terminal.

    Parameters
    ==========
    cmd: the command to send, should be a list for subprocess
    error_message: the error message to give to user if fails,
    if none specified, will alert that command failed.

    '''
    if sudo is True:
        cmd = ['sudo'] + cmd

    try:
        output = Popen(cmd, stderr=STDOUT, stdout=PIPE)

    except FileNotFoundError:
        cmd.pop(0)
        output = Popen(cmd, stderr=STDOUT, stdout=PIPE)

    t = output.communicate()[0],output.returncode
    output = {'message':t[0],
              'return_code':t[1]}

    if isinstance(output['message'], bytes):
        output['message'] = output['message'].decode('utf-8')

    return output

def run_command ( cmd , sudo = False ) : if sudo is True : cmd = [ 'sudo' ] + cmd try : output = Popen ( cmd , stderr = STDOUT , stdout = PIPE ) except FileNotFoundError : cmd . pop ( 0 ) output = Popen ( cmd , stderr = STDOUT , stdout = PIPE ) t = output . communicate ( ) [ 0 ] , output . returncode output = { 'message' : t [ 0 ] , 'return_code' : t [ 1 ] } if isinstance ( output [ 'message' ] , bytes ) : output [ 'message' ] = output [ 'message' ] . decode ( 'utf-8' ) return output

run_command uses subprocess to send a command to the terminal .


def _get_metadata(self, image_file=None, dbx_metadata=None):
        '''this is a wrapper around the main client.get_metadata to first parse
           a Dropbox FileMetadata into a dicionary, then pass it on to the
           primary get_metadata function.

           Parameters
           ==========
           image_file: the full path to the image file that had metadata
                       extracted
           metadata: the Dropbox FileMetadata to parse.

        '''
        metadata = dict()

        if dbx_metadata is not None:
            for key in dbx_metadata.__dir__():
                value = getattr(dbx_metadata, key)
                if type(value) in [str, datetime.datetime, bool, int, float]:
                    metadata[key.strip('_')] = value

        return self.get_metadata(image_file, names=metadata)

def _get_metadata ( self , image_file = None , dbx_metadata = None ) : metadata = dict ( ) if dbx_metadata is not None : for key in dbx_metadata . __dir__ ( ) : value = getattr ( dbx_metadata , key ) if type ( value ) in [ str , datetime . datetime , bool , int , float ] : metadata [ key . strip ( '_' ) ] = value return self . get_metadata ( image_file , names = metadata )

this is a wrapper around the main client . get_metadata to first parse a Dropbox FileMetadata into a dicionary then pass it on to the primary get_metadata function .


def _update_secrets(self):
        '''update secrets will look for a dropbox token in the environment at
           SREGISTRY_DROPBOX_TOKEN and if found, create a client. If not,
           an error message is returned and the client exits.
        '''

        # Retrieve the user token. Exit if not found
        token = self._required_get_and_update('SREGISTRY_DROPBOX_TOKEN')

        # Create the dropbox client
        self.dbx = Dropbox(token)

        # Verify that the account is valid
        try:
            self.account = self.dbx.users_get_current_account()
        except AuthError as err:
            bot.error('Account invalid. Exiting.')
            sys.exit(1)

def
_update_secrets
(
self
)
:
token
=
self
.
_required_get_and_update
(
'SREGISTRY_DROPBOX_TOKEN'
)
self
.
dbx
=
Dropbox
(
token
)
try
:
self
.
account
=
self
.
dbx
.
users_get_current_account
(
)
except
AuthError
as
err
:
bot
.
error
(
'Account invalid. Exiting.'
)
sys
.
exit
(
1
)

update secrets will look for a dropbox token in the environment at SREGISTRY_DROPBOX_TOKEN and if found create a client . If not an error message is returned and the client exits .


def print_output(response, output_file=None):
    '''print the output to the console for the user. If the user wants the content
       also printed to an output file, do that.

       Parameters
       ==========
       response: the response from the builder, with metadata added
       output_file: if defined, write output also to file

    '''
    # If successful built, show container uri
    if response['status'] == 'SUCCESS':
        bucket = response['artifacts']['objects']['location']
        obj = response['artifacts']['objects']['paths'][0]
        bot.custom("MD5HASH", response['file_hash'], 'CYAN')
        bot.custom("SIZE", response['size'], 'CYAN')
        bot.custom(response['status'], bucket + obj , 'CYAN')
    else:
        bot.custom(response['status'], 'see logs for details', 'CYAN')

    # Show the logs no matter what
    bot.custom("LOGS", response['logUrl'], 'CYAN')

    # Did the user make the container public?
    if "public_url" in response:
        bot.custom('URL', response['public_url'], 'CYAN')

    # Does the user also need writing to an output file?
    if output_file != None:
        with open(output_file, 'w') as filey:
            if response['status'] == 'SUCCESS':
                filey.writelines('MD5HASH %s\n' % response['file_hash'])
                filey.writelines('SIZE %s\n' % response['size'])
            filey.writelines('%s %s%s\n' % (response['status'], bucket, obj))
            filey.writelines('LOGS %s\n' % response['logUrl'])
            if "public_url" in response:
                filey.writelines('URL %s\n' % response['public_url'])

def
print_output
(
response
,
output_file
=
)
:
if
response
[
'status'
]
==
'SUCCESS'
:
bucket
=
response
[
'artifacts'
]
[
'objects'
]
[
'location'
]
obj
=
response
[
'artifacts'
]
[
'objects'
]
[
'paths'
]
[
0
]
bot
.
custom
(
"MD5HASH"
,
response
[
'file_hash'
]
,
'CYAN'
)
bot
.
custom
(
"SIZE"
,
response
[
'size'
]
,
'CYAN'
)
bot
.
custom
(
response
[
'status'
]
,
bucket
+
obj
,
'CYAN'
)
else
:
bot
.
custom
(
response
[
'status'
]
,
'see logs for details'
,
'CYAN'
)
bot
.
custom
(
"LOGS"
,
response
[
'logUrl'
]
,
'CYAN'
)
if
"public_url"
in
response
:
bot
.
custom
(
'URL'
,
response
[
'public_url'
]
,
'CYAN'
)
if
output_file
!=
:
with
open
(
output_file
,
'w'
)
as
filey
:
if
response
[
'status'
]
==
'SUCCESS'
:
filey
.
writelines
(
'MD5HASH %s\n'
%
response
[
'file_hash'
]
)
filey
.
writelines
(
'SIZE %s\n'
%
response
[
'size'
]
)
filey
.
writelines
(
'%s %s%s\n'
%
(
response
[
'status'
]
,
bucket
,
obj
)
)
filey
.
writelines
(
'LOGS %s\n'
%
response
[
'logUrl'
]
)
if
"public_url"
in
response
:
filey
.
writelines
(
'URL %s\n'
%
response
[
'public_url'
]
)

print the output to the console for the user . If the user wants the content also printed to an output file do that .


def kill(args):
    '''kill is a helper function to call the "kill" function of the client,
       meaning we bring down an instance.
    '''
    from sregistry.main import Client as cli
    if len(args.commands) > 0:
        for name in args.commands:
            cli.destroy(name)
    sys.exit(0)

def kill ( args ) : from sregistry . main import Client as cli if len ( args . commands ) > 0 : for name in args . commands : cli . destroy ( name ) sys . exit ( 0 )

kill is a helper function to call the kill function of the client meaning we bring down an instance .


def list_logs(args, container_name=None):
    '''list a specific log for a builder, or the latest log if none provided

       Parameters
       ==========
       args: the argparse object to look for a container name
       container_name: a default container name set to be None (show latest log)

    '''
    from sregistry.main import Client as cli
    if len(args.commands) > 0:
        container_name = args.commands.pop(0)
    cli.logs(container_name)
    sys.exit(0)

def list_logs ( args , container_name = None ) : from sregistry . main import Client as cli if len ( args . commands ) > 0 : container_name = args . commands . pop ( 0 ) cli . logs ( container_name ) sys . exit ( 0 )

list a specific log for a builder or the latest log if none provided


def get_collections(self):
        '''get a listing of collections that the user has access to.
        '''
        collections = []
        for container in self.conn.get_account()[1]:
            collections.append(container['name'])
        return collections

def get_collections ( self ) : collections = [ ] for container in self . conn . get_account ( ) [ 1 ] : collections . append ( container [ 'name' ] ) return collections

get a listing of collections that the user has access to .


def _update_secrets(self):
        '''update secrets will look for a user and token in the environment
           If we find the values, cache and continue. Otherwise, exit with error
        '''

        # Get the swift authentication type first.  That will determine what we
        # will need to collect for proper authentication
        self.config['SREGISTRY_SWIFT_AUTHTYPE'] = self._required_get_and_update(
                                                     'SREGISTRY_SWIFT_AUTHTYPE')

        # Check what auth version is requested and setup the connection
        if self.config['SREGISTRY_SWIFT_AUTHTYPE'] == 'preauth':

            # Pre-Authenticated Token/URL - Use OS_AUTH_TOKEN/OS_STORAGE_URL
            # Retrieve the user token, user, and base. Exit if not found
            for envar in ['SREGISTRY_SWIFT_OS_AUTH_TOKEN',
                          'SREGISTRY_SWIFT_OS_STORAGE_URL' ]:
                self.config[envar] = self._required_get_and_update(envar)

            self.conn = swiftclient.Connection(
                preauthurl=self.config['SREGISTRY_SWIFT_OS_STORAGE_URL'],
                preauthtoken=self.config['SREGISTRY_SWIFT_OS_AUTH_TOKEN']
            )
        elif self.config['SREGISTRY_SWIFT_AUTHTYPE'] == 'keystonev3':

            # Keystone v3 Authentication
            # Retrieve the user token, user, and base. Exit if not found
            for envar in ['SREGISTRY_SWIFT_USER',
                          'SREGISTRY_SWIFT_TOKEN',
                          'SREGISTRY_SWIFT_URL']:
                self.config[envar] = self._required_get_and_update(envar)

            auth_url = '%s/v3' % self.config['SREGISTRY_SWIFT_URL']
            # Setting to default as a safety.  No v3 environment to test
            # May require ENV vars for real use. - M. Moore
            _os_options = {
                'user_domain_name': 'Default',
                'project_domain_name': 'Default',
                'project_name': 'Default'
            }

            # Save the connection to use for some command
            self.conn = swiftclient.Connection(
                user=self.config['SREGISTRY_SWIFT_USER'],
                key=self.config['SREGISTRY_SWIFT_TOKEN'],
                os_options=_os_options,
                authurl=auth_url,
                auth_version='3'
            )

        elif self.config['SREGISTRY_SWIFT_AUTHTYPE'] == 'keystonev2':

            # Keystone v2 Authentication
            # Retrieve the user token, user, and base. Exit if not found
            for envar in ['SREGISTRY_SWIFT_USER',
                          'SREGISTRY_SWIFT_TOKEN',
                          'SREGISTRY_SWIFT_TENANT',
                          'SREGISTRY_SWIFT_REGION',
                          'SREGISTRY_SWIFT_URL']:
                self.config[envar] = self._required_get_and_update(envar)

            # More human friendly to interact with
            auth_url = '%s/v2.0/' % self.config['SREGISTRY_SWIFT_URL']
            # Set required OpenStack options for tenant/region
            _os_options = {
                'tenant_name': self.config['SREGISTRY_SWIFT_TENANT'],
                'region_name': self.config['SREGISTRY_SWIFT_REGION']
            }

            # Save the connection to use for some command
            self.conn = swiftclient.Connection(
                user=self.config['SREGISTRY_SWIFT_USER'],
                key=self.config['SREGISTRY_SWIFT_TOKEN'],
                os_options=_os_options,
                authurl=auth_url,
                auth_version='2'
            )
        else:

            # Legacy Authentication
            # Retrieve the user token, user, and base. Exit if not found
            for envar in ['SREGISTRY_SWIFT_USER',
                          'SREGISTRY_SWIFT_TOKEN',
                          'SREGISTRY_SWIFT_URL']:
                self.config[envar] = self._required_get_and_update(envar)

            # More human friendly to interact with
            auth_url = '%s/auth/' % self.config['SREGISTRY_SWIFT_URL']

            # Save the connection to use for some command
            self.conn = swiftclient.Connection(
                user=self.config['SREGISTRY_SWIFT_USER'],
                key=self.config['SREGISTRY_SWIFT_TOKEN'],
                authurl=auth_url,
            )

def _update_secrets ( self ) : self . config [ 'SREGISTRY_SWIFT_AUTHTYPE' ] = self . _required_get_and_update ( 'SREGISTRY_SWIFT_AUTHTYPE' ) if self . config [ 'SREGISTRY_SWIFT_AUTHTYPE' ] == 'preauth' : for envar in [ 'SREGISTRY_SWIFT_OS_AUTH_TOKEN' , 'SREGISTRY_SWIFT_OS_STORAGE_URL' ] : self . config [ envar ] = self . _required_get_and_update ( envar ) self . conn = swiftclient . Connection ( preauthurl = self . config [ 'SREGISTRY_SWIFT_OS_STORAGE_URL' ] , preauthtoken = self . config [ 'SREGISTRY_SWIFT_OS_AUTH_TOKEN' ] ) elif self . config [ 'SREGISTRY_SWIFT_AUTHTYPE' ] == 'keystonev3' : for envar in [ 'SREGISTRY_SWIFT_USER' , 'SREGISTRY_SWIFT_TOKEN' , 'SREGISTRY_SWIFT_URL' ] : self . config [ envar ] = self . _required_get_and_update ( envar ) auth_url = '%s/v3' % self . config [ 'SREGISTRY_SWIFT_URL' ] _os_options = { 'user_domain_name' : 'Default' , 'project_domain_name' : 'Default' , 'project_name' : 'Default' } self . conn = swiftclient . Connection ( user = self . config [ 'SREGISTRY_SWIFT_USER' ] , key = self . config [ 'SREGISTRY_SWIFT_TOKEN' ] , os_options = _os_options , authurl = auth_url , auth_version = '3' ) elif self . config [ 'SREGISTRY_SWIFT_AUTHTYPE' ] == 'keystonev2' : for envar in [ 'SREGISTRY_SWIFT_USER' , 'SREGISTRY_SWIFT_TOKEN' , 'SREGISTRY_SWIFT_TENANT' , 'SREGISTRY_SWIFT_REGION' , 'SREGISTRY_SWIFT_URL' ] : self . config [ envar ] = self . _required_get_and_update ( envar ) auth_url = '%s/v2.0/' % self . config [ 'SREGISTRY_SWIFT_URL' ] _os_options = { 'tenant_name' : self . config [ 'SREGISTRY_SWIFT_TENANT' ] , 'region_name' : self . config [ 'SREGISTRY_SWIFT_REGION' ] } self . conn = swiftclient . Connection ( user = self . config [ 'SREGISTRY_SWIFT_USER' ] , key = self . config [ 'SREGISTRY_SWIFT_TOKEN' ] , os_options = _os_options , authurl = auth_url , auth_version = '2' ) else : for envar in [ 'SREGISTRY_SWIFT_USER' , 'SREGISTRY_SWIFT_TOKEN' , 'SREGISTRY_SWIFT_URL' ] : self . config [ envar ] = self . _required_get_and_update ( envar ) auth_url = '%s/auth/' % self . config [ 'SREGISTRY_SWIFT_URL' ] self . conn = swiftclient . Connection ( user = self . config [ 'SREGISTRY_SWIFT_USER' ] , key = self . config [ 'SREGISTRY_SWIFT_TOKEN' ] , authurl = auth_url , )

update secrets will look for a user and token in the environment If we find the values cache and continue . Otherwise exit with error


def _update_secrets(self):
        '''The user is required to have an application secrets file in his
           or her environment. The information isn't saved to the secrets
           file, but the client exists with error if the variable isn't found.
        '''
        env = 'GOOGLE_APPLICATION_CREDENTIALS'
        self._secrets = self._get_and_update_setting(env)
        if self._secrets is None:
            bot.error('You must export %s to use Google Storage client' %env)
            sys.exit(1)

The user is required to have an application secrets file in his or her environment . The information isn t saved to the secrets file but the client exists with error if the variable isn t found .


def get_client(image=None, quiet=False, **kwargs):
    '''
       get the correct client depending on the driver of interest. The
       selected client can be chosen based on the environment variable
       SREGISTRY_CLIENT, and later changed based on the image uri parsed
       If there is no preference, the default is to load the singularity
       hub client.

       Parameters
       ==========
       image: if provided, we derive the correct client based on the uri
       of an image. If not provided, we default to environment, then hub.
       quiet: if True, suppress most output about the client (e.g. speak)

    '''
    from sregistry.defaults import SREGISTRY_CLIENT

    # Give the user a warning:
    if not check_install():
        bot.warning('Singularity is not installed, function might be limited.')

    # If an image is provided, use to determine client
    client_name = get_uri(image)
    if client_name is not None:
        SREGISTRY_CLIENT = client_name

    # If no obvious credential provided, we can use SREGISTRY_CLIENT
    if   SREGISTRY_CLIENT == 'aws': from .aws import Client
    elif SREGISTRY_CLIENT == 'docker': from .docker import Client
    elif SREGISTRY_CLIENT == 'dropbox': from .dropbox import Client
    elif SREGISTRY_CLIENT == 'gitlab': from .gitlab import Client
    elif SREGISTRY_CLIENT == 'globus': from .globus import Client
    elif SREGISTRY_CLIENT == 'nvidia': from .nvidia import Client
    elif SREGISTRY_CLIENT == 'hub': from .hub import Client
    elif SREGISTRY_CLIENT == 'google-drive': from .google_drive import Client
    elif SREGISTRY_CLIENT == 'google-compute': from .google_storage import Client
    elif SREGISTRY_CLIENT == 'google-storage': from .google_storage import Client
    elif SREGISTRY_CLIENT == 'google-build': from .google_build import Client
    elif SREGISTRY_CLIENT == 'registry': from .registry import Client
    elif SREGISTRY_CLIENT == 's3': from .s3 import Client
    elif SREGISTRY_CLIENT == 'swift': from .swift import Client
    else: from .hub import Client

    Client.client_name = SREGISTRY_CLIENT
    Client.quiet = quiet

    # Create credentials cache, if it doesn't exist
    Client._credential_cache = get_credential_cache()

    # Add the database, if wanted
    if SREGISTRY_DATABASE is not None:

        # These are global functions used across modules
        from sregistry.database import (
            init_db, add, cp, get, mv, rm, rmi,
            images, inspect,
            rename,
            get_container,
            get_collection,
            get_or_create_collection
        )

        # Actions
        Client._init_db = init_db
        Client.add = add
        Client.cp = cp
        Client.get = get
        Client.inspect = inspect
        Client.mv = mv
        Client.rename = rename
        Client.rm = rm
        Client.rmi = rmi
        Client.images = images

        # Collections
        Client.get_or_create_collection = get_or_create_collection
        Client.get_container = get_container
        Client.get_collection = get_collection

    # If no database, import dummy functions that return the equivalent
    else:
        from sregistry.database import ( add, init_db )
        Client.add = add
        Client._init_db = init_db

    # Initialize the database
    cli = Client()

    if hasattr(Client, '_init_db'):
        cli._init_db(SREGISTRY_DATABASE)
    return cli

get the correct client depending on the driver of interest . The selected client can be chosen based on the environment variable SREGISTRY_CLIENT and later changed based on the image uri parsed If there is no preference the default is to load the singularity hub client .


def ipython(args):
    '''give the user an ipython shell, optionally with an endpoint of choice.
    '''

    # The client will announce itself (backend/database) unless it's get
    from sregistry.main import get_client
    client = get_client(args.endpoint)
    client.announce(args.command)
    from IPython import embed
    embed()

def ipython ( args ) : from sregistry . main import get_client client = get_client ( args . endpoint ) client . announce ( args . command ) from IPython import embed embed ( )

give the user an ipython shell optionally with an endpoint of choice .


def get_manifests(self, repo_name, digest=None):
    '''get_manifests calls get_manifest for each of the schema versions,
       including v2 and v1. Version 1 includes image layers and metadata,
       and version 2 must be parsed for a specific manifest, and the 2nd
       call includes the layers. If a digest is not provided
       latest is used.

       Parameters
       ==========
       repo_name: reference to the <username>/<repository>:<tag> to obtain
       digest: a tag or shasum version

    '''

    if not hasattr(self, 'manifests'):
        self.manifests = {}

    # Obtain schema version 1 (metadata) and 2, and image config
    schemaVersions = ['v1', 'v2', 'config']
    for schemaVersion in schemaVersions:
        manifest = self._get_manifest(repo_name, digest, schemaVersion)
        if manifest is not None:

            # If we don't have a config yet, try to get from version 2 manifest
            if schemaVersion == "v2" and "config" in manifest:
                bot.debug('Attempting to get config as blob in verison 2 manifest')
                url = self._get_layerLink(repo_name, manifest['config']['digest'])
                headers = {'Accept': manifest['config']['mediaType']}
                self.manifests['config'] = self._get(url, headers=headers)

            self.manifests[schemaVersion] = manifest

    return self.manifests

get_manifests calls get_manifest for each of the schema versions including v2 and v1 . Version 1 includes image layers and metadata and version 2 must be parsed for a specific manifest and the 2nd call includes the layers . If a digest is not provided latest is used .


def get_manifest(self, repo_name, digest=None, version="v1"):
    '''
       get_manifest should return an image manifest
       for a particular repo and tag.  The image details
       are extracted when the client is generated.

       Parameters
       ==========
       repo_name: reference to the <username>/<repository>:<tag> to obtain
       digest: a tag or shasum version
       version: one of v1, v2, and config (for image config)

    '''

    accepts = {'config': "application/vnd.docker.container.image.v1+json",
               'v1': "application/vnd.docker.distribution.manifest.v1+json",
               'v2': "application/vnd.docker.distribution.manifest.v2+json" }

    url = self._get_manifest_selfLink(repo_name, digest)

    bot.verbose("Obtaining manifest: %s %s" % (url, version))
    headers = {'Accept': accepts[version] }

    try:
        manifest = self._get(url, headers=headers, quiet=True)
        manifest['selfLink'] = url
    except:
        manifest = None

    return manifest

get_manifest should return an image manifest for a particular repo and tag . The image details are extracted when the client is generated .


def get_download_cache(self, destination, subfolder='docker'):
    '''determine the user preference for atomic download of layers. If
       the user has set a singularity cache directory, honor it. Otherwise,
       use the Singularity default.
    '''
    # First priority after user specification is Singularity Cache
    if destination is None:
        destination = self._get_setting('SINGULARITY_CACHEDIR',
                                         SINGULARITY_CACHE)

        # If not set, the user has disabled (use tmp)
        destination = get_tmpdir(destination)

    if not destination.endswith(subfolder):
        destination = "%s/%s" %(destination, subfolder)

    # Create subfolders, if don't exist
    mkdir_p(destination)
    return destination

def get_download_cache ( self , destination , subfolder = 'docker' ) : if destination is None : destination = self . _get_setting ( 'SINGULARITY_CACHEDIR' , SINGULARITY_CACHE ) destination = get_tmpdir ( destination ) if not destination . endswith ( subfolder ) : destination = "%s/%s" % ( destination , subfolder ) mkdir_p ( destination ) return destination

determine the user preference for atomic download of layers . If the user has set a singularity cache directory honor it . Otherwise use the Singularity default .


def extract_env(self):
    '''extract the environment from the manifest, or return None.
       Used by functions env_extract_image, and env_extract_tar
    '''
    environ = self._get_config('Env')
    if environ is not None:
        if not isinstance(environ, list):
            environ = [environ]

        lines = []
        for line in environ:
            line = re.findall("(?P<var_name>.+?)=(?P<var_value>.+)", line)
            line = ['export %s="%s"' % (x[0], x[1]) for x in line]
            lines = lines + line

        environ = "\n".join(lines)
        bot.verbose3("Found Docker container environment!")
    return environ

def
extract_env
(
self
)
:
environ
=
self
.
_get_config
(
'Env'
)
if
environ
is
not
:
if
not
isinstance
(
environ
,
list
)
:
environ
=
[
environ
]
lines
=
[
]
for
line
in
environ
:
line
=
re
.
findall
(
"(?P<var_name>.+?)=(?P<var_value>.+)"
,
line
)
line
=
[
'export %s="%s"'
%
(
x
[
0
]
,
x
[
1
]
)
for
x
in
line
]
lines
=
lines
+
line
environ
=
"\n"
.
join
(
lines
)
bot
.
verbose3
(
"Found Docker container environment!"
)
return
environ

extract the environment from the manifest or return None . Used by functions env_extract_image and env_extract_tar


def _update_base(self):
        '''update the base, including the URL for GitLab and the API endpoint.
        '''
        self.base = self._get_and_update_setting('SREGISTRY_GITLAB_BASE',
                                                 "https://gitlab.com/")
        self.api_base = "%s/api/v4" % self.base.strip('/')
        self.artifacts = self._get_and_update_setting('SREGISTRY_GITLAB_FOLDER',
                                                      'build')

        self.job = self._get_and_update_setting('SREGISTRY_GITLAB_JOB', 'build')

        bot.debug('      Api: %s' % self.api_base)
        bot.debug('Artifacts: %s' % self.artifacts)
        bot.debug('      Job: %s' % self.job)

def
_update_base
(
self
)
:
self
.
base
=
self
.
_get_and_update_setting
(
'SREGISTRY_GITLAB_BASE'
,
"https://gitlab.com/"
)
self
.
api_base
=
"%s/api/v4"
%
self
.
base
.
strip
(
'/'
)
self
.
artifacts
=
self
.
_get_and_update_setting
(
'SREGISTRY_GITLAB_FOLDER'
,
'build'
)
self
.
job
=
self
.
_get_and_update_setting
(
'SREGISTRY_GITLAB_JOB'
,
'build'
)
bot
.
debug
(
'      Api: %s'
%
self
.
api_base
)
bot
.
debug
(
'Artifacts: %s'
%
self
.
artifacts
)
bot
.
debug
(
'      Job: %s'
%
self
.
job
)

update the base including the URL for GitLab and the API endpoint .


def _update_secrets(self):
        '''update secrets will update metadata needed for pull and search
        '''
        self.token = self._required_get_and_update('SREGISTRY_GITLAB_TOKEN')
        self.headers["Private-Token"] = self.token

def _update_secrets ( self ) : self . token = self . _required_get_and_update ( 'SREGISTRY_GITLAB_TOKEN' ) self . headers [ "Private-Token" ] = self . token


def _get_metadata(self):
        '''since the user needs a job id and other parameters, save this
           for them.
        '''
        metadata = {'SREGISTRY_GITLAB_FOLDER': self.artifacts,
                    'api_base': self.api_base,
                    'SREGISTRY_GITLAB_BASE': self.base,
                    'SREGISTRY_GITLAB_JOB': self.job }
        return metadata

def _get_metadata ( self ) : metadata = { 'SREGISTRY_GITLAB_FOLDER' : self . artifacts , 'api_base' : self . api_base , 'SREGISTRY_GITLAB_BASE' : self . base , 'SREGISTRY_GITLAB_JOB' : self . job } return metadata

since the user needs a job id and other parameters save this for them .


def get_settings(self, client_name=None):
    '''get all settings, either for a particular client if a name is provided,
       or across clients.

       Parameters
       ==========
       client_name: the client name to return settings for (optional)

    '''
    settings = read_client_secrets()
    if client_name is not None and client_name in settings:
        return settings[client_name]
    return settings

def get_settings ( self , client_name = None ) : settings = read_client_secrets ( ) if client_name is not None and client_name in settings : return settings [ client_name ] return settings

get all settings either for a particular client if a name is provided or across clients .


def required_get_and_update(self, name, default=None):
    '''a wrapper to get_and_update, but if not successful, will print an
       error and exit.
    '''
    setting = self._get_and_update_setting(name, default=None)
    if setting in [None, ""]:
        bot.exit('You must export %s' % name)
    return setting

def
required_get_and_update
(
self
,
name
,
default
=
)
:
setting
=
self
.
_get_and_update_setting
(
name
,
default
=
)
if
setting
in
[
,
""
]
:
bot
.
exit
(
'You must export %s'
%
name
)
return
setting

a wrapper to get_and_update but if not successful will print an error and exit .


def update_setting(self, name, value):
    '''Just update a setting, doesn't need to be returned.
    '''

    if value is not None:
        updates = {name : value}
        update_client_secrets(backend=self.client_name,
                              updates=updates)

def update_setting ( self , name , value ) : if value is not None : updates = { name : value } update_client_secrets ( backend = self . client_name , updates = updates )

Just update a setting doesn t need to be returned .


def authorize(self, names, payload=None, request_type="push"):
    '''Authorize a client based on encrypting the payload with the client
       token, which should be matched on the receiving server'''

    if self.secrets is not None:

        if "registry" in self.secrets:

            # Use the payload to generate a digest   push|collection|name|tag|user
            timestamp = generate_timestamp()
            credential = generate_credential(self.secrets['registry']['username'])
            credential = "%s/%s/%s" %(request_type,credential,timestamp)

            if payload is None:
                payload = "%s|%s|%s|%s|%s|" %(request_type,
                                              names['collection'],
                                              timestamp,
                                              names['image'],
                                              names['tag'])

            signature = generate_signature(payload,self.secrets['registry']['token'])
            return "SREGISTRY-HMAC-SHA256 Credential=%s,Signature=%s" %(credential,signature)

Authorize a client based on encrypting the payload with the client token which should be matched on the receiving server


def list_builders(self, project=None, zone='us-west1-a'):
    '''list builders, or instances for the project. They should start with
       sregistry-builder

       Parameters
       ==========
       project: specify a project, will default to environment first
       zone: the zone to use, defaults to us-west1-a if environment not set

    '''
    builders = []
    instances = self._get_instances(project, zone)

    for instance in instances['items']:
        builders.append([instance['name'], instance['status']])

    bot.info("[google-compute] Found %s instances" %(len(builders)))
    bot.table(builders)
    bot.newline()

def
list_builders
(
self
,
project
=
,
zone
=
'us-west1-a'
)
:
builders
=
[
]
instances
=
self
.
_get_instances
(
project
,
zone
)
for
instance
in
instances
[
'items'
]
:
builders
.
append
(
[
instance
[
'name'
]
,
instance
[
'status'
]
]
)
bot
.
info
(
"[google-compute] Found %s instances"
%
(
len
(
builders
)
)
)
bot
.
table
(
builders
)
bot
.
newline
(
)

list builders or instances for the project . They should start with sregistry - builder


def load_templates(self, name):
    '''load a particular template based on a name. We look for a name IN data,
       so the query name can be a partial string of the full name.

       Parameters
       ==========
       name: the name of a template to look up
    '''
    configs = self._get_templates()
    templates = []

    # The user wants to retrieve a particular configuration
    matches = [x for x in configs['data'] if name in x['name']]
    if len(matches) > 0:
        for match in matches:
            response = self._get(match['id'])
            templates.append(response)
        return templates

    bot.info('No matches found for %s' %name)

def
load_templates
(
self
,
name
)
:
configs
=
self
.
_get_templates
(
)
templates
=
[
]
matches
=
[
x
for
x
in
configs
[
'data'
]
if
name
in
x
[
'name'
]
]
if
len
(
matches
)
>
0
:
for
match
in
matches
:
response
=
self
.
_get
(
match
[
'id'
]
)
templates
.
append
(
response
)
return
templates
bot
.
info
(
'No matches found for %s'
%
name
)

load a particular template based on a name . We look for a name IN data so the query name can be a partial string of the full name .


def get_ipaddress(self, name, retries=3, delay=3):
    '''get the ip_address of an inserted instance. Will try three times with
       delay to give the instance time to start up.

       Parameters
       ==========
       name: the name of the instance to get the ip address for.
       retries: the number of retries before giving up
       delay: the delay between retry

       Note from @vsoch: this function is pretty nasty.

    '''
    for rr in range(retries):

        # Retrieve list of instances
        instances = self._get_instances()

        for instance in instances['items']:
            if instance['name'] == name:

                # Iterate through network interfaces
                for network in instance['networkInterfaces']:
                    if network['name'] == 'nic0':

                        # Access configurations
                        for subnet in network['accessConfigs']:
                            if subnet['name'] == 'External NAT':
                                if 'natIP' in subnet:
                                    return subnet['natIP']

        sleep(delay)
    bot.warning('Did not find IP address, check Cloud Console!')

get the ip_address of an inserted instance . Will try three times with delay to give the instance time to start up .


def run_build(self, config):
    '''run a build, meaning inserting an instance. Retry if there is failure

       Parameters
       ==========
       config: the configuration dictionary generated by setup_build

    '''
    project = self._get_project()
    zone = self._get_zone()

    bot.custom(prefix='INSTANCE', message=config['name'], color="CYAN")
    bot.info(config['description'])

    response = self._compute_service.instances().insert(project=project,
                                                        zone=zone,
                                                        body=config).execute()

    # Direct the user to the web portal with log
    ipaddress = self._get_ipaddress(config['name'])
    bot.info('Robot Logger: http://%s' %ipaddress)
    bot.info('Allow a few minutes for web server install, beepboop!')
    return response

run a build meaning inserting an instance . Retry if there is failure


def list_containers(self):
    '''return a list of containers, determined by finding the metadata field
       "type" with value "container." We alert the user to no containers
       if results is empty, and exit

       {'metadata': {'items':
                              [
                               {'key': 'type', 'value': 'container'}, ...
                              ]
                    }
       }

    '''
    results = []
    for image in self._bucket.list_blobs():
        if image.metadata is not None:
            if "type" in image.metadata:
                if image.metadata['type'] == "container":
                    results.append(image)

    if len(results) == 0:
        bot.info("No containers found, based on metadata type:container")

    return results

return a list of containers determined by finding the metadata field type with value container . We alert the user to no containers if results is empty and exit


def search_all(self):
    '''a "list all" search that doesn't require a query. Here we return to
       the user all objects that have custom metadata value of "container"

       IMPORTANT: the upload function adds this metadata. For a container to
       be found by the client, it must have the type as container in metadata.
    '''

    results = self._list_containers()

    bot.info("[gs://%s] Containers" %self._bucket_name)

    rows = []
    for i in results:
        size = round(i.size / (1024*1024.0))
        size = ("%s MB" %size).rjust(10)
        rows.append([size, i.metadata['name']])

    bot.table(rows)
    return rows

def
search_all
(
self
)
:
results
=
self
.
_list_containers
(
)
bot
.
info
(
"[gs://%s] Containers"
%
self
.
_bucket_name
)
rows
=
[
]
for
i
in
results
:
size
=
round
(
i
.
size
/
(
1024
*
1024.0
)
)
size
=
(
"%s MB"
%
size
)
.
rjust
(
10
)
rows
.
append
(
[
size
,
i
.
metadata
[
'name'
]
]
)
bot
.
table
(
rows
)
return
rows

a list all search that doesn t require a query . Here we return to the user all objects that have custom metadata value of container


def main(args,parser,subparser):
    '''the list command corresponds with listing images for an external
       resource. This is different from listing images that are local to the
       database, which should be done with "images"
    '''
    from sregistry.main import get_client
    cli = get_client(quiet=args.quiet)

    for query in args.query:
        if query in ['','*']:
            query = None

        cli.ls(query=query)

def main ( args , parser , subparser ) : from sregistry . main import get_client cli = get_client ( quiet = args . quiet ) for query in args . query : if query in [ '' , '*' ] : query = None cli . ls ( query = query )

the list command corresponds with listing images for an external resource . This is different from listing images that are local to the database which should be done with images


def main(args, parser, subparser):
    '''sharing an image means sending a remote share from an image you
       control to a contact, usually an email.
    '''
    from sregistry.main import get_client
    images = args.image

    if not isinstance(images,list):
        images = [images]

    for image in images:
        print(image)

        # Detect any uri, and refresh client if necessary
        cli = get_client(image, quiet=args.quiet)
        cli.announce(args.command)
        cli.share(image, share_to=args.share_to)

def main ( args , parser , subparser ) : from sregistry . main import get_client images = args . image if not isinstance ( images , list ) : images = [ images ] for image in images : print ( image ) cli = get_client ( image , quiet = args . quiet ) cli . announce ( args . command ) cli . share ( image , share_to = args . share_to )

sharing an image means sending a remote share from an image you control to a contact usually an email .


def init_db(self, db_path):
    '''initialize the database, with the default database path or custom of

       the format sqlite:////scif/data/expfactory.db

    The custom path can be set with the environment variable SREGISTRY_DATABASE
    when a user creates the client, we must initialize this db
    the database should use the .singularity cache folder to cache
    layers and images, and .singularity/sregistry.db as a database
    '''

    # Database Setup, use default if uri not provided
    self.database = 'sqlite:///%s' % db_path
    self.storage = SREGISTRY_STORAGE

    bot.debug("Database located at %s" % self.database)
    self.engine = create_engine(self.database, convert_unicode=True)
    self.session = scoped_session(sessionmaker(autocommit=False,
                                               autoflush=False,
                                               bind=self.engine))

    Base.query = self.session.query_property()

    # import all modules here that might define models so that
    # they will be registered properly on the metadata.  Otherwise
    # you will have to import them first before calling init_db()
    Base.metadata.create_all(bind=self.engine)
    self.Base = Base

def
init_db
(
self
,
db_path
)
:
self
.
database
=
'sqlite:///%s'
%
db_path
self
.
storage
=
SREGISTRY_STORAGE
bot
.
debug
(
"Database located at %s"
%
self
.
database
)
self
.
engine
=
create_engine
(
self
.
database
,
convert_unicode
=
True
)
self
.
session
=
scoped_session
(
sessionmaker
(
autocommit
=
False
,
autoflush
=
False
,
bind
=
self
.
engine
)
)
Base
.
query
=
self
.
session
.
query_property
(
)
Base
.
metadata
.
create_all
(
bind
=
self
.
engine
)
self
.
Base
=
Base

initialize the database with the default database path or custom of


def get_build_template():
    '''get default build template.
    '''
    base = get_installdir()
    name = "%s/main/templates/build/singularity-cloudbuild.json" % base

    if os.path.exists(name):
        bot.debug("Found template %s" %name)
        return read_json(name)

    bot.warning("Template %s not found." % name)


def search(self, query=None, args=None):
    '''query will show images determined by the extension of img
       or simg.

       Parameters
       ==========
       query: the container name (path) or uri to search for
       args.endpoint: can be an endpoint id and optional path, e.g.:

             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e:.singularity'
             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e'

       if not defined, we show the user endpoints to choose from

       Usage
       =====
       If endpoint is defined with a query, then we search the given endpoint
       for a container of interested (designated by ending in .img or .simg

       If no endpoint is provided but instead just a query, we use the query
       to search endpoints.

    '''

    # No query is defined
    if query is None:

        # Option 1: No query or endpoints lists all shared and personal
        if args.endpoint is None:
            bot.info('Listing shared endpoints. Add query to expand search.')
            return self._list_endpoints()

        # Option 2: An endpoint without query will just list containers there
        else:
            return self._list_endpoint(args.endpoint)

    # Option 3: A query without an endpoint will search endpoints for it
    if args.endpoint is None:
        bot.info('You must specify an endpoint id to query!')
        return self._list_endpoints(query)

    # Option 4: A query with an endpoint will search the endpoint for pattern
    return self._list_endpoint(endpoint=args.endpoint,
                               query=query)

query will show images determined by the extension of img or simg .


def list_endpoints(self, query=None):
    '''list all endpoints, providing a list of endpoints to the user to
       better filter the search. This function takes no arguments,
       as the user has not provided an endpoint id or query.
    '''
    bot.info('Please select an endpoint id to query from')

    endpoints = self._get_endpoints(query)

    # Iterate through endpoints to provide user a list

    bot.custom(prefix="Globus", message="Endpoints", color="CYAN")
    rows = []
    for kind,eps in endpoints.items():
        for epid,epmeta in eps.items():
            rows.append([epid, '[%s]' %kind, epmeta['name']])

    bot.table(rows)
    return rows

list all endpoints providing a list of endpoints to the user to better filter the search . This function takes no arguments as the user has not provided an endpoint id or query .


def list_endpoint(self, endpoint, query=None):
    '''An endpoint is required here to list files within. Optionally, we can
       take a path relative to the endpoint root.

       Parameters
       ==========
       endpoint: a single endpoint ID or an endpoint id and relative path.
                 If no path is provided, we use '', which defaults to scratch.

       query: if defined, limit files to those that have query match

    '''
    if not hasattr(self, 'transfer_client'):
        self._init_transfer_client()

    # Separate endpoint id from the desired path

    endpoint, path = self._parse_endpoint_name(endpoint)

    # Get a list of files at endpoint, under specific path
    try:
        result = self.transfer_client.operation_ls(endpoint, path=path)
    except TransferAPIError as err:

        # Tell the user what went wrong!
        bot.custom(prefix='ERROR', message=err, color='RED')
        sys.exit(1)

    rows = []

    for filey in result:

        # Highlight container contenders with purple
        name = filey['name']
        if query is None or query in name:
            if name.endswith('img'):
                name = bot.addColor('PURPLE',name)

            rows.append([filey['type'],
                         filey['permissions'],
                         str(filey['size']),
                         name ])

    if len(rows) > 0:
        rows = [["type","[perm]","[size]","[name]"]] + rows
        bot.custom(prefix="Endpoint Listing %s" %path, message='', color="CYAN")
        bot.table(rows)
    else:
        bot.info('No content was found at the selected endpoint.')
    return rows

An endpoint is required here to list files within . Optionally we can take a path relative to the endpoint root .


def share(self, query, share_to=None):
    '''share will use the client to get a shareable link for an image of choice.
       the functions returns a url of choice to send to a recipient.
    '''

    names = parse_image_name(remove_uri(query))

    # Dropbox path is the path in storage with a slash
    dropbox_path = '/%s' % names['storage']

    # First ensure that exists
    if self.exists(dropbox_path) is True:

        # Create new shared link
        try:
            share = self.dbx.sharing_create_shared_link_with_settings(dropbox_path)

        # Already exists!
        except ApiError as err:
            share = self.dbx.sharing_create_shared_link(dropbox_path)

        bot.info(share.url)
    return share.url

share will use the client to get a shareable link for an image of choice . the functions returns a url of choice to send to a recipient .


def read_client_secrets():
    '''for private or protected registries, a client secrets file is required
       to be located at .sregistry. If no secrets are found, we use default
       of Singularity Hub, and return a dummy secrets.
    '''
    client_secrets = _default_client_secrets()

    # If token file not provided, check environment
    secrets = get_secrets_file()

    # If exists, load
    if secrets is not None:
        client_secrets = read_json(secrets)

    # Otherwise, initialize
    else:
        from sregistry.defaults import SREGISTRY_CLIENT_SECRETS
        write_json(client_secrets, SREGISTRY_CLIENT_SECRETS)

    return client_secrets

def read_client_secrets ( ) : client_secrets = _default_client_secrets ( ) secrets = get_secrets_file ( ) if secrets is not None : client_secrets = read_json ( secrets ) else : from sregistry . defaults import SREGISTRY_CLIENT_SECRETS write_json ( client_secrets , SREGISTRY_CLIENT_SECRETS ) return client_secrets

for private or protected registries a client secrets file is required to be located at . sregistry . If no secrets are found we use default of Singularity Hub and return a dummy secrets .


def _get_services(self, version='v1'):
        '''get version 1 of the google compute and storage service

        Parameters
        ==========
        version: version to use (default is v1)
        '''
        self._bucket_service = storage.Client()
        creds = GoogleCredentials.get_application_default()
        self._storage_service = discovery_build('storage', version, credentials=creds)
        self._compute_service = discovery_build('compute', version, credentials=creds)

def _get_services ( self , version = 'v1' ) : self . _bucket_service = storage . Client ( ) creds = GoogleCredentials . get_application_default ( ) self . _storage_service = discovery_build ( 'storage' , version , credentials = creds ) self . _compute_service = discovery_build ( 'compute' , version , credentials = creds )


def delete_object(service, bucket_name, object_name):
    '''delete object will delete a file from a bucket

       Parameters
       ==========
       storage_service: the service obtained with get_storage_service
       bucket_name: the name of the bucket
       object_name: the "name" parameter of the object.

    '''
    try:
        operation = service.objects().delete(bucket=bucket_name,
                                             object=object_name).execute()
    except HttpError as e:
        pass
        operation = e
    return operation

def delete_object ( service , bucket_name , object_name ) : try : operation = service . objects ( ) . delete ( bucket = bucket_name , object = object_name ) . execute ( ) except HttpError as e : pass operation = e return operation


def delete(self, name):
    '''delete an image from Google Storage.

       Parameters
       ==========
       name: the name of the file (or image) to delete

    '''

    bot.debug("DELETE %s" % name)

    for file_object in files:
        if isinstance(file_object, dict):
            if "kind" in file_object:
                if file_object['kind'] == "storage#object":
                    object_name = "/".join(file_object['id'].split('/')[:-1])
                    object_name = re.sub('%s/' %self._bucket['name'],'', object_name,1)

                    delete_object(service=self._bucket_service,
                                  bucket_name=bucket['name'],
                                  object_name=object_name)

def
delete
(
self
,
name
)
:
bot
.
debug
(
"DELETE %s"
%
name
)
for
file_object
in
files
:
if
isinstance
(
file_object
,
dict
)
:
if
"kind"
in
file_object
:
if
file_object
[
'kind'
]
==
"storage#object"
:
object_name
=
"/"
.
join
(
file_object
[
'id'
]
.
split
(
'/'
)
[
:
1
]
)
object_name
=
re
.
sub
(
'%s/'
%
self
.
_bucket
[
'name'
]
,
''
,
object_name
,
1
)
delete_object
(
service
=
self
.
_bucket_service
,
bucket_name
=
bucket
[
'name'
]
,
object_name
=
object_name
)


def destroy(self, name):
    '''destroy an instance, meaning take down the instance and stop the build.

       Parameters
       ==========
       name: the name of the instance to stop building.
    '''

    instances = self._get_instances()
    project = self._get_project()
    zone = self._get_zone()
    found = False

    if 'items' in instances:
        for instance in instances['items']:
            if instance['name'] == name:
                found = True
                break

    if found:
        bot.info('Killing instance %s' %name)
        return self._compute_service.instances().delete(project=project,
                                                        zone=zone,
                                                        instance=name).execute()

def
destroy
(
self
,
name
)
:
instances
=
self
.
_get_instances
(
)
project
=
self
.
_get_project
(
)
zone
=
self
.
_get_zone
(
)
found
=
False
if
'items'
in
instances
:
for
instance
in
instances
[
'items'
]
:
if
instance
[
'name'
]
==
name
:
found
=
True
break
if
found
:
bot
.
info
(
'Killing instance %s'
%
name
)
return
self
.
_compute_service
.
instances
(
)
.
delete
(
project
=
project
,
zone
=
zone
,
instance
=
name
)
.
execute
(
)

destroy an instance meaning take down the instance and stop the build .


def get_subparsers(parser):
    '''get_subparser will get a dictionary of subparsers, to help with printing help
    '''

    actions = [action for action in parser._actions
               if isinstance(action, argparse._SubParsersAction)]

    subparsers = dict()
    for action in actions:
        # get all subparsers and print help
        for choice, subparser in action.choices.items():
            subparsers[choice] = subparser

    return subparsers

def get_subparsers ( parser ) : actions = [ action for action in parser . _actions if isinstance ( action , argparse . _SubParsersAction ) ] subparsers = dict ( ) for action in actions : for choice , subparser in action . choices . items ( ) : subparsers [ choice ] = subparser return subparsers

get_subparser will get a dictionary of subparsers to help with printing help


def generate(self, delim='-', length=4, chars='0123456789'):
        '''
        Generate a robot name. Inspiration from Haikunator, but much more
                 poorly implemented ;)

        Parameters
        ==========
        delim: Delimiter
        length: TokenLength
        chars: TokenChars
        '''

        descriptor = self._select(self._descriptors)
        noun = self._select(self._nouns)
        numbers = ''.join((self._select(chars) for _ in range(length)))
        return delim.join([descriptor, noun, numbers])

def generate ( self , delim = '-' , length = 4 , chars = '0123456789' ) : descriptor = self . _select ( self . _descriptors ) noun = self . _select ( self . _nouns ) numbers = '' . join ( ( self . _select ( chars ) for _ in range ( length ) ) ) return delim . join ( [ descriptor , noun , numbers ] )

Generate a robot name . Inspiration from Haikunator but much more poorly implemented ; )


def get_tmpdir(requested_tmpdir=None, prefix="", create=True):
    '''get a temporary directory for an operation. If SREGISTRY_TMPDIR
       is set, return that. Otherwise, return the output of tempfile.mkdtemp

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), we will need to
       create a subfolder *within* the SREGISTRY_TMPDIR.
       create: boolean to determine if we should create folder (True)
    '''
    from sregistry.defaults import SREGISTRY_TMPDIR

    # First priority for the base goes to the user requested.
    tmpdir = requested_tmpdir or SREGISTRY_TMPDIR

    prefix = prefix or "sregistry-tmp"
    prefix = "%s.%s" %(prefix, next(tempfile._get_candidate_names()))
    tmpdir = os.path.join(tmpdir, prefix)

    if not os.path.exists(tmpdir) and create is True:
        os.mkdir(tmpdir)

    return tmpdir

def get_tmpdir ( requested_tmpdir = None , prefix = "" , create = True ) : from sregistry . defaults import SREGISTRY_TMPDIR tmpdir = requested_tmpdir or SREGISTRY_TMPDIR prefix = prefix or "sregistry-tmp" prefix = "%s.%s" % ( prefix , next ( tempfile . _get_candidate_names ( ) ) ) tmpdir = os . path . join ( tmpdir , prefix ) if not os . path . exists ( tmpdir ) and create is True : os . mkdir ( tmpdir ) return tmpdir

get a temporary directory for an operation . If SREGISTRY_TMPDIR is set return that . Otherwise return the output of tempfile . mkdtemp


def extract_tar(archive, output_folder, handle_whiteout=False):
    '''extract a tar archive to a specified output folder

        Parameters
        ==========
        archive: the archive file to extract
        output_folder: the output folder to extract to
        handle_whiteout: use docker2oci variation to handle whiteout files

    '''
    from .terminal import run_command

    # Do we want to remove whiteout files?
    if handle_whiteout is True:
        return _extract_tar(archive, output_folder)

    # If extension is .tar.gz, use -xzf
    args = '-xf'
    if archive.endswith(".tar.gz"):
        args = '-xzf'

    # Just use command line, more succinct.
    command = ["tar", args, archive, "-C", output_folder, "--exclude=dev/*"]
    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

def
extract_tar
(
archive
,
output_folder
,
handle_whiteout
=
False
)
:
from
.
terminal
import
run_command
if
handle_whiteout
is
True
:
return
_extract_tar
(
archive
,
output_folder
)
args
=
'-xf'
if
archive
.
endswith
(
".tar.gz"
)
:
args
=
'-xzf'
command
=
[
"tar"
,
args
,
archive
,
"-C"
,
output_folder
,
"--exclude=dev/*"
]
if
not
bot
.
is_quiet
(
)
:
print
(
"Extracting %s"
%
archive
)
return
run_command
(
command
)


def _extract_tar(archive, output_folder):
    '''use blob2oci to handle whiteout files for extraction. Credit for this
       script goes to docker2oci by Olivier Freyermouth, and see script
       folder for license.

       Parameters
       ==========
       archive: the archive to extract
       output_folder the output folder (sandbox) to extract to

    '''
    from .terminal import ( run_command, which )

    result = which('blob2oci')
    if result['return_code'] != 0:
        bot.error('Cannot find blob2oci script on path, exiting.')
        sys.exit(1)

    script = result['message']
    command = ['exec' ,script, '--layer', archive, '--extract', output_folder]

    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

use blob2oci to handle whiteout files for extraction . Credit for this script goes to docker2oci by Olivier Freyermouth and see script folder for license .


def get_file_hash(filename):
    '''find the SHA256 hash string of a file
    '''
    hasher = hashlib.sha256()
    with open(filename, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

def get_file_hash ( filename ) : hasher = hashlib . sha256 ( ) with open ( filename , "rb" ) as f : for chunk in iter ( lambda : f . read ( 4096 ) , b"" ) : hasher . update ( chunk ) return hasher . hexdigest ( )


def read_file(filename, mode="r", readlines=True):
    '''write_file will open a file, "filename" and write content, "content"
       and properly close the file
    '''
    with open(filename, mode) as filey:
        if readlines is True:
            content = filey.readlines()
        else:
            content = filey.read()
    return content

def read_file ( filename , mode = "r" , readlines = True ) : with open ( filename , mode ) as filey : if readlines is True : content = filey . readlines ( ) else : content = filey . read ( ) return content

write_file will open a file filename and write content content and properly close the file


def read_json(filename, mode='r'):
    '''read_json reads in a json file and returns
       the data structure as dict.
    '''
    with open(filename, mode) as filey:
        data = json.load(filey)
    return data

def read_json ( filename , mode = 'r' ) : with open ( filename , mode ) as filey : data = json . load ( filey ) return data

read_json reads in a json file and returns the data structure as dict .


def clean_up(files):
    '''clean up will delete a list of files, only if they exist
    '''
    if not isinstance(files, list):
        files = [files]

    for f in files:
        if os.path.exists(f):
            bot.verbose3("Cleaning up %s" % f)
            os.remove(f)

def
clean_up
(
files
)
:
if
not
isinstance
(
files
,
list
)
:
files
=
[
files
]
for
f
in
files
:
if
os
.
path
.
exists
(
f
)
:
bot
.
verbose3
(
"Cleaning up %s"
%
f
)
os
.
remove
(
f
)

clean up will delete a list of files only if they exist


def push(self, path, name, tag=None):
    '''push an image to an S3 endpoint'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

    # Create extra metadata, this is how we identify the image later
    # *important* bug in boto3 will return these capitalized
    # see https://github.com/boto/boto3/issues/1709
    metadata = {'sizemb': "%s" % image_size,
                'client': 'sregistry' }

    self.bucket.upload_file(path, names['storage_uri'], {"Metadata": metadata })

def
push
(
self
,
path
,
name
,
tag
=
)
:
path
=
os
.
path
.
abspath
(
path
)
image
=
os
.
path
.
basename
(
path
)
bot
.
debug
(
"PUSH %s"
%
path
)
if
not
os
.
path
.
exists
(
path
)
:
bot
.
error
(
'%s does not exist.'
%
path
)
sys
.
exit
(
1
)
names
=
parse_image_name
(
remove_uri
(
name
)
,
tag
=
tag
)
image_size
=
os
.
path
.
getsize
(
path
)
>>
20
metadata
=
{
'sizemb'
:
"%s"
%
image_size
,
'client'
:
'sregistry'
}
self
.
bucket
.
upload_file
(
path
,
names
[
'storage_uri'
]
,
{
"Metadata"
:
metadata
}
)


def get_or_create_collection(self, name):
    '''get a collection if it exists. If it doesn't exist, create it first.

    Parameters
    ==========
    name: the collection name, usually parsed from get_image_names()['name']

    '''
    from sregistry.database.models import Collection
    collection = self.get_collection(name)

    # If it doesn't exist, create it
    if collection is None:
        collection = Collection(name=name)
        self.session.add(collection)
        self.session.commit()

    return collection

def get_or_create_collection ( self , name ) : from sregistry . database . models import Collection collection = self . get_collection ( name ) if collection is None : collection = Collection ( name = name ) self . session . add ( collection ) self . session . commit ( ) return collection

get a collection if it exists . If it doesn t exist create it first .


def get_collection(self, name):
    '''get a collection, if it exists, otherwise return None.
    '''
    from sregistry.database.models import Collection
    return Collection.query.filter(Collection.name == name).first()

def get_collection ( self , name ) : from sregistry . database . models import Collection return Collection . query . filter ( Collection . name == name ) . first ( )

get a collection if it exists otherwise return None .


def get_container(self, name, collection_id, tag="latest", version=None):
    '''get a container, otherwise return None.
    '''
    from sregistry.database.models import Container
    if version is None:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag).first()
    else:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag,
                                              version = version).first()
    return container

def get_container ( self , name , collection_id , tag = "latest" , version = None ) : from sregistry . database . models import Container if version is None : container = Container . query . filter_by ( collection_id = collection_id , name = name , tag = tag ) . first ( ) else : container = Container . query . filter_by ( collection_id = collection_id , name = name , tag = tag , version = version ) . first ( ) return container


def images(self, query=None):
    '''List local images in the database, optionally with a query.

       Paramters
       =========
       query: a string to search for in the container or collection name|tag|uri

    '''
    from sregistry.database.models import Collection, Container

    rows = []
    if query is not None:
        like = "%" + query + "%"
        containers = Container.query.filter(or_(Container.name == query,
                                                Container.tag.like(like),
                                                Container.uri.like(like),
                                                Container.name.like(like))).all()
    else:
        containers = Container.query.all()

    if len(containers) > 0:
        message = "  [date]   [client]\t[uri]"
        bot.custom(prefix='Containers:', message=message, color="RED")
        for c in containers:
            uri = c.get_uri()
            created_at = c.created_at.strftime('%B %d, %Y')
            rows.append([created_at, "   [%s]" %c.client, uri])
        bot.table(rows)
    return containers

def
images
(
self
,
query
=
)
:
from
sregistry
.
database
.
models
import
Collection
,
Container
rows
=
[
]
if
query
is
not
:
like
=
"%"
+
query
+
"%"
containers
=
Container
.
query
.
filter
(
or_
(
Container
.
name
==
query
,
Container
.
tag
.
like
(
like
)
,
Container
.
uri
.
like
(
like
)
,
Container
.
name
.
like
(
like
)
)
)
.
all
(
)
else
:
containers
=
Container
.
query
.
all
(
)
if
len
(
containers
)
>
0
:
message
=
"  [date]   [client]\t[uri]"
bot
.
custom
(
prefix
=
'Containers:'
,
message
=
message
,
color
=
"RED"
)
for
c
in
containers
:
uri
=
c
.
get_uri
(
)
created_at
=
c
.
created_at
.
strftime
(
'%B %d, %Y'
)
rows
.
append
(
[
created_at
,
"   [%s]"
%
c
.
client
,
uri
]
)
bot
.
table
(
rows
)
return
containers

List local images in the database optionally with a query .


def inspect(self, name):
    '''Inspect a local image in the database, which typically includes the
       basic fields in the model.

    '''
    print(name)
    container = self.get(name)
    if container is not None:
        collection = container.collection.name
        fields = container.__dict__.copy()
        fields['collection'] = collection
        fields['metrics'] = json.loads(fields['metrics'])
        del fields['_sa_instance_state']
        fields['created_at'] = str(fields['created_at'])
        print(json.dumps(fields, indent=4, sort_keys=True))
        return fields

def inspect ( self , name ) : print ( name ) container = self . get ( name ) if container is not None : collection = container . collection . name fields = container . __dict__ . copy ( ) fields [ 'collection' ] = collection fields [ 'metrics' ] = json . loads ( fields [ 'metrics' ] ) del fields [ '_sa_instance_state' ] fields [ 'created_at' ] = str ( fields [ 'created_at' ] ) print ( json . dumps ( fields , indent = 4 , sort_keys = True ) ) return fields

Inspect a local image in the database which typically includes the basic fields in the model .


def rename(self, image_name, path):
    '''rename performs a move, but ensures the path is maintained in storage

       Parameters
       ==========
       image_name: the image name (uri) to rename to.
       path: the name to rename (basename is taken)

    '''
    container = self.get(image_name, quiet=True)

    if container is not None:
        if container.image is not None:

            # The original directory for the container stays the same
            dirname = os.path.dirname(container.image)

            # But we derive a new filename and uri

            names = parse_image_name( remove_uri (path) )
            storage = os.path.join( self.storage,
                                    os.path.dirname(names['storage']) )

            # This is the collection folder

            if not os.path.exists(storage):
                os.mkdir(storage)

            # Here we get the new full path, rename the container file

            fullpath = os.path.abspath(os.path.join(dirname, names['storage']))
            container = self.cp(move_to=fullpath,
                                container=container,
                                command="rename")

            # On successful rename of file, update the uri

            if container is not None:
                container.uri = names['uri']
                self.session.commit()
                return container

    bot.warning('%s not found' %(image_name))

def
rename
(
self
,
image_name
,
path
)
:
container
=
self
.
get
(
image_name
,
quiet
=
True
)
if
container
is
not
:
if
container
.
image
is
not
:
dirname
=
os
.
path
.
dirname
(
container
.
image
)
names
=
parse_image_name
(
remove_uri
(
path
)
)
storage
=
os
.
path
.
join
(
self
.
storage
,
os
.
path
.
dirname
(
names
[
'storage'
]
)
)
if
not
os
.
path
.
exists
(
storage
)
:
os
.
mkdir
(
storage
)
fullpath
=
os
.
path
.
abspath
(
os
.
path
.
join
(
dirname
,
names
[
'storage'
]
)
)
container
=
self
.
cp
(
move_to
=
fullpath
,
container
=
container
,
command
=
"rename"
)
if
container
is
not
:
container
.
uri
=
names
[
'uri'
]
self
.
session
.
commit
(
)
return
container
bot
.
warning
(
'%s not found'
%
(
image_name
)
)

rename performs a move but ensures the path is maintained in storage


def mv(self, image_name, path):
    '''Move an image from it's current location to a new path.
       Removing the image from organized storage is not the recommended approach
       however is still a function wanted by some.

       Parameters
       ==========
       image_name: the parsed image name.
       path: the location to move the image to

    '''

    container = self.get(image_name, quiet=True)

    if container is not None:

        name = container.uri or container.get_uri()
        image = container.image or ''

        # Only continue if image file exists
        if os.path.exists(image):

            # Default assume directory, use image name and path fully
            filename = os.path.basename(image)
            filedir = os.path.abspath(path)

            # If it's a file, use filename provided
            if not os.path.isdir(path):
                filename = os.path.basename(path)
                filedir = os.path.dirname(path)

            # If directory is empty, assume $PWD
            if filedir == '':
                filedir = os.getcwd()

            # Copy to the fullpath from the storage
            fullpath = os.path.abspath(os.path.join(filedir,filename))
            return self.cp(move_to=fullpath,
                           container=container,
                           command="move")

    bot.warning('%s not found' %(image_name))

def
mv
(
self
,
image_name
,
path
)
:
container
=
self
.
get
(
image_name
,
quiet
=
True
)
if
container
is
not
:
name
=
container
.
uri
or
container
.
get_uri
(
)
image
=
container
.
image
or
''
if
os
.
path
.
exists
(
image
)
:
filename
=
os
.
path
.
basename
(
image
)
filedir
=
os
.
path
.
abspath
(
path
)
if
not
os
.
path
.
isdir
(
path
)
:
filename
=
os
.
path
.
basename
(
path
)
filedir
=
os
.
path
.
dirname
(
path
)
if
filedir
==
''
:
filedir
=
os
.
getcwd
(
)
fullpath
=
os
.
path
.
abspath
(
os
.
path
.
join
(
filedir
,
filename
)
)
return
self
.
cp
(
move_to
=
fullpath
,
container
=
container
,
command
=
"move"
)
bot
.
warning
(
'%s not found'
%
(
image_name
)
)

Move an image from it s current location to a new path . Removing the image from organized storage is not the recommended approach however is still a function wanted by some .


def rmi(self, image_name):
    '''Remove an image from the database and filesystem.
    '''
    container = self.rm(image_name, delete=True)
    if container is not None:
        bot.info("[rmi] %s" % container)

def
rmi
(
self
,
image_name
)
:
container
=
self
.
rm
(
image_name
,
delete
=
True
)
if
container
is
not
:
bot
.
info
(
"[rmi] %s"
%
container
)

Remove an image from the database and filesystem .


def add(self, image_path=None,
              image_uri=None,
              image_name=None,
              url=None,
              metadata=None,
              save=True,
              copy=False):

    '''get or create a container, including the collection to add it to.
    This function can be used from a file on the local system, or via a URL
    that has been downloaded. Either way, if one of url, version, or image_file
    is not provided, the model is created without it. If a version is not
    provided but a file path is, then the file hash is used.

    Parameters
    ==========
    image_path: full path to image file
    image_name: if defined, the user wants a custom name (and not based on uri)
    metadata: any extra metadata to keep for the image (dict)
    save: if True, move the image to the cache if it's not there
    copy: If True, copy the image instead of moving it.

    image_name: a uri that gets parsed into a names object that looks like:

    {'collection': 'vsoch',
     'image': 'hello-world',
     'storage': 'vsoch/hello-world-latest.img',
     'tag': 'latest',
     'version': '12345'
     'uri': 'vsoch/hello-world:latest@12345'}

    After running add, the user will take some image in a working
    directory, add it to the database, and have it available for search
    and use under SREGISTRY_STORAGE/<collection>/<container>

    If the container was retrieved from a webby place, it should have version
    If no version is found, the file hash is used.
    '''

    from sregistry.database.models import (
        Container,
        Collection
    )

    # We can only save if the image is provided
    if image_path is not None:
        if not os.path.exists(image_path) and save is True:
            bot.error('Cannot find %s' %image_path)
            sys.exit(1)

    # An image uri is required for version, tag, etc.
    if image_uri is None:
        bot.error('You must provide an image uri <collection>/<namespace>')
        sys.exit(1)

    names = parse_image_name( remove_uri(image_uri) )
    bot.debug('Adding %s to registry' % names['uri'])

    # If Singularity is installed, inspect image for metadata
    metadata = self.get_metadata(image_path, names=names)
    collection = self.get_or_create_collection(names['collection'])

    # Get a hash of the file for the version, or use provided
    version = names.get('version')
    if version == None:
        if image_path != None:
            version = get_image_hash(image_path)
        else:
            version = ''  # we can't determine a version, not in API/no file
        names = parse_image_name( remove_uri(image_uri), version=version )

    # If save, move to registry storage first
    if save is True and image_path is not None:

        # If the user hasn't defined a custom name
        if image_name is None:
            image_name = self._get_storage_name(names)

        if copy is True:
            copyfile(image_path, image_name)
        else:
            shutil.move(image_path, image_name)

        image_path = image_name

    # Just in case the client didn't provide it, see if we have in metadata
    if url is None and "url" in metadata:
        url = metadata['url']

    # First check that we don't have one already!
    container = self.get_container(name=names['image'],
                                   collection_id=collection.id,
                                   tag=names['tag'],
                                   version=version)

    # The container did not exist, create it
    if container is None:
        action = "new"
        container = Container(metrics=json.dumps(metadata),
                              name=names['image'],
                              image=image_path,
                              client=self.client_name,
                              tag=names['tag'],
                              version=version,
                              url=url,
                              uri=names['uri'],
                              collection_id=collection.id)

        self.session.add(container)
        collection.containers.append(container)

    # The container existed, update it.
    else:
        action="update"
        metrics=json.loads(container.metrics)
        metrics.update(metadata)
        container.url= url
        container.client=self.client_name
        if image_path is not None:
            container.image=image_path
        container.metrics=json.dumps(metrics)

    self.session.commit()
    bot.info("[container][%s] %s" % (action,names['uri']))
    return container

get or create a container including the collection to add it to . This function can be used from a file on the local system or via a URL that has been downloaded . Either way if one of url version or image_file is not provided the model is created without it . If a version is not provided but a file path is then the file hash is used .


def push(self, path, name, tag=None):
    '''push an image to Singularity Registry'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Interaction with a registry requires secrets
    self.require_secrets()

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

# COLLECTION ###################################################################

    # If the registry is provided in the uri, use it
    if names['registry'] == None:
        names['registry'] = self.base

    # If the base doesn't start with http or https, add it
    names = self._add_https(names)

    # Prepare push request, this will return a collection ID if permission
    url = '%s/push/' % names['registry']
    auth_url = '%s/upload/chunked_upload' % names['registry']
    SREGISTRY_EVENT = self.authorize(request_type="push",
                                     names=names)

    # Data fields for collection
    fields = { 'collection': names['collection'],
               'name':names['image'],
               'tag': names['tag']}

    headers = { 'Authorization': SREGISTRY_EVENT }

    r = requests.post(auth_url, json=fields, headers=headers)

    # Always tell the user what's going on!
    message = self._read_response(r)
    print('\n[1. Collection return status {0} {1}]'.format(r.status_code, message))

    # Get the collection id, if created, and continue with upload
    if r.status_code != 200:
        sys.exit(1)

# UPLOAD #######################################################################

    url = '%s/upload' % names['registry'].replace('/api','')
    bot.debug('Seting upload URL to {0}'.format(url))

    cid = r.json()['cid']
    upload_to = os.path.basename(names['storage'])

    SREGISTRY_EVENT = self.authorize(request_type="upload",
                                     names=names)

    encoder = MultipartEncoder(fields={'SREGISTRY_EVENT': SREGISTRY_EVENT,
                                       'name': names['image'],
                                       'collection': str(cid),
                                       'tag': names['tag'],
                                       'file1': (upload_to, open(path, 'rb'), 'text/plain')})

    progress_callback = create_callback(encoder, self.quiet)
    monitor = MultipartEncoderMonitor(encoder, progress_callback)
    headers = {'Content-Type': monitor.content_type,
               'Authorization': SREGISTRY_EVENT }

    try:
        r = requests.post(url, data=monitor, headers=headers)
        r.raise_for_status()
        message = r.json()['message']
        print('\n[Return status {0} {1}]'.format(r.status_code, message))
    except requests.HTTPError as e:
        print('\nUpload failed: {0}.'.format(e))
    except KeyboardInterrupt:
        print('\nUpload cancelled.')
    except Exception as e:
        print(e)


def parse_header(recipe, header="from", remove_header=True):
    '''take a recipe, and return the complete header, line. If
       remove_header is True, only return the value.

       Parameters
       ==========
       recipe: the recipe file
       headers: the header key to find and parse
       remove_header: if true, remove the key

    '''
    parsed_header = None
    fromline = [x for x in recipe.split('\n') if "%s:" %header in x.lower()]

    # Case 1: We did not find the fromline
    if len(fromline) == 0:
        return ""

    # Case 2: We found it!
    if len(fromline) > 0:
        fromline = fromline[0]
        parsed_header = fromline.strip()

    # Does the user want to clean it up?
    if remove_header is True:
        parsed_header = fromline.split(':', 1)[-1].strip()
    return parsed_header

def parse_header ( recipe , header = "from" , remove_header = True ) : parsed_header = None fromline = [ x for x in recipe . split ( '\n' ) if "%s:" % header in x . lower ( ) ] if len ( fromline ) == 0 : return "" if len ( fromline ) > 0 : fromline = fromline [ 0 ] parsed_header = fromline . strip ( ) if remove_header is True : parsed_header = fromline . split ( ':' , 1 ) [ - 1 ] . strip ( ) return parsed_header

take a recipe and return the complete header line . If remove_header is True only return the value .


def find_single_recipe(filename, pattern="Singularity", manifest=None):
    '''find_single_recipe will parse a single file, and if valid,
       return an updated manifest

       Parameters
       ==========
       filename: the filename to assess for a recipe
       pattern: a default pattern to search for
       manifest: an already started manifest

    '''

    if pattern is None:
        pattern = "Singularity*"

    recipe = None
    file_basename = os.path.basename(filename)
    if fnmatch.fnmatch(file_basename, pattern):
        recipe = {'path': os.path.abspath(filename),
                  'modified':os.path.getmtime(filename)}

    # If we already have the recipe, only add if more recent
    if manifest is not None and recipe is not None:
        container_uri = '/'.join(filename.split('/')[-2:])
        if container_uri in manifest:
            if manifest[container_uri]['modified'] < os.path.getmtime(filename):
                manifest[container_uri] = recipe
        else:
            manifest[container_uri] = recipe
        return manifest

    return recipe

def find_single_recipe ( filename , pattern = "Singularity" , manifest = None ) : if pattern is None : pattern = "Singularity*" recipe = None file_basename = os . path . basename ( filename ) if fnmatch . fnmatch ( file_basename , pattern ) : recipe = { 'path' : os . path . abspath ( filename ) , 'modified' : os . path . getmtime ( filename ) } if manifest is not None and recipe is not None : container_uri = '/' . join ( filename . split ( '/' ) [ - 2 : ] ) if container_uri in manifest : if manifest [ container_uri ] [ 'modified' ] < os . path . getmtime ( filename ) : manifest [ container_uri ] = recipe else : manifest [ container_uri ] = recipe return manifest return recipe

find_single_recipe will parse a single file and if valid return an updated manifest


def create_build_package(package_files):
    '''given a list of files, copy them to a temporary folder,
       compress into a .tar.gz, and rename based on the file hash.
       Return the full path to the .tar.gz in the temporary folder.

       Parameters
       ==========
       package_files: a list of files to include in the tar.gz

    '''
    # Ensure package files all exist
    for package_file in package_files:
        if not os.path.exists(package_file):
            bot.exit('Cannot find %s.' % package_file)

    bot.log('Generating build package for %s files...' % len(package_files))
    build_dir = get_tmpdir(prefix="sregistry-build")
    build_tar = '%s/build.tar.gz' % build_dir
    tar = tarfile.open(build_tar, "w:gz")

    # Create the tar.gz
    for package_file in package_files:
        tar.add(package_file)
    tar.close()

    # Get hash (sha256), and rename file
    sha256 = get_file_hash(build_tar)
    hash_tar =  "%s/%s.tar.gz" %(build_dir, sha256)
    shutil.move(build_tar, hash_tar)
    return hash_tar

given a list of files copy them to a temporary folder compress into a . tar . gz and rename based on the file hash . Return the full path to the . tar . gz in the temporary folder .


def run_build(self, config, bucket, names):
    '''run a build, meaning creating a build. Retry if there is failure
    '''

    project = self._get_project()

    #          prefix,    message, color
    bot.custom('PROJECT', project, "CYAN")
    bot.custom('BUILD  ', config['steps'][0]['name'], "CYAN")

    response = self._build_service.projects().builds().create(body=config,
                                              projectId=project).execute()

    build_id = response['metadata']['build']['id']
    status = response['metadata']['build']['status']
    bot.log("build %s: %s" % (build_id, status))

    start = time.time()
    while status not in ['COMPLETE', 'FAILURE', 'SUCCESS']:
        time.sleep(15)
        response = self._build_service.projects().builds().get(id=build_id,
                                                  projectId=project).execute()

        build_id = response['id']
        status = response['status']
        bot.log("build %s: %s" % (build_id, status))

    end = time.time()
    bot.log('Total build time: %s seconds' % (round(end - start, 2)))

    # If successful, update blob metadata and visibility
    if status == 'SUCCESS':

        # Does the user want to keep the container private?
        env = 'SREGISTRY_GOOGLE_STORAGE_PRIVATE'
        blob = bucket.blob(response['artifacts']['objects']['paths'][0])

        # Make Public, if desired
        if self._get_and_update_setting(env) == None:
            blob.make_public()
            response['public_url'] = blob.public_url

        # Add the metadata directly to the object
        update_blob_metadata(blob, response, config, bucket, names)
        response['media_link'] = blob.media_link
        response['size'] = blob.size
        response['file_hash'] = blob.md5_hash

    return response

def
run_build
(
self
,
config
,
bucket
,
names
)
:
project
=
self
.
_get_project
(
)
bot
.
custom
(
'PROJECT'
,
project
,
"CYAN"
)
bot
.
custom
(
'BUILD  '
,
config
[
'steps'
]
[
0
]
[
'name'
]
,
"CYAN"
)
response
=
self
.
_build_service
.
projects
(
)
.
builds
(
)
.
create
(
body
=
config
,
projectId
=
project
)
.
execute
(
)
build_id
=
response
[
'metadata'
]
[
'build'
]
[
'id'
]
status
=
response
[
'metadata'
]
[
'build'
]
[
'status'
]
bot
.
log
(
"build %s: %s"
%
(
build_id
,
status
)
)
start
=
time
.
time
(
)
while
status
not
in
[
'COMPLETE'
,
'FAILURE'
,
'SUCCESS'
]
:
time
.
sleep
(
15
)
response
=
self
.
_build_service
.
projects
(
)
.
builds
(
)
.
get
(
id
=
build_id
,
projectId
=
project
)
.
execute
(
)
build_id
=
response
[
'id'
]
status
=
response
[
'status'
]
bot
.
log
(
"build %s: %s"
%
(
build_id
,
status
)
)
end
=
time
.
time
(
)
bot
.
log
(
'Total build time: %s seconds'
%
(
round
(
end
start
,
2
)
)
)
if
status
==
'SUCCESS'
:
env
=
'SREGISTRY_GOOGLE_STORAGE_PRIVATE'
blob
=
bucket
.
blob
(
response
[
'artifacts'
]
[
'objects'
]
[
'paths'
]
[
0
]
)
if
self
.
_get_and_update_setting
(
env
)
==
:
blob
.
make_public
(
)
response
[
'public_url'
]
=
blob
.
public_url
update_blob_metadata
(
blob
,
response
,
config
,
bucket
,
names
)
response
[
'media_link'
]
=
blob
.
media_link
response
[
'size'
]
=
blob
.
size
response
[
'file_hash'
]
=
blob
.
md5_hash
return
response

run a build meaning creating a build . Retry if there is failure


def update_blob_metadata(blob, response, config, bucket, names):
    '''a specific function to take a blob, along with a SUCCESS response
       from Google build, the original config, and update the blob
       metadata with the artifact file name, dependencies, and image hash.
    '''

    manifest = os.path.basename(response['results']['artifactManifest'])
    manifest = json.loads(bucket.blob(manifest).download_as_string())

    metadata = {'file_hash': manifest['file_hash'][0]['file_hash'][0]['value'],
                'artifactManifest': response['results']['artifactManifest'],
                'location': manifest['location'],
                'storageSourceBucket': config['source']['storageSource']['bucket'],
                'storageSourceObject': config['source']['storageSource']['object'],
                'buildCommand': ' '.join(config['steps'][0]['args']),
                'builder': config['steps'][0]['name'],
                'media_link': blob.media_link,
                'self_link': blob.self_link,
                'size': blob.size,
                'name': names['tag_uri'],
                'type': "container"} # identifier that the blob is a container

    blob.metadata = metadata
    blob._properties['metadata'] = metadata
    blob.patch()

def
update_blob_metadata
(
blob
,
response
,
config
,
bucket
,
names
)
:
manifest
=
os
.
path
.
basename
(
response
[
'results'
]
[
'artifactManifest'
]
)
manifest
=
json
.
loads
(
bucket
.
blob
(
manifest
)
.
download_as_string
(
)
)
metadata
=
{
'file_hash'
:
manifest
[
'file_hash'
]
[
0
]
[
'file_hash'
]
[
0
]
[
'value'
]
,
'artifactManifest'
:
response
[
'results'
]
[
'artifactManifest'
]
,
'location'
:
manifest
[
'location'
]
,
'storageSourceBucket'
:
config
[
'source'
]
[
'storageSource'
]
[
'bucket'
]
,
'storageSourceObject'
:
config
[
'source'
]
[
'storageSource'
]
[
'object'
]
,
'buildCommand'
:
' '
.
join
(
config
[
'steps'
]
[
0
]
[
'args'
]
)
,
'builder'
:
config
[
'steps'
]
[
0
]
[
'name'
]
,
'media_link'
:
blob
.
media_link
,
'self_link'
:
blob
.
self_link
,
'size'
:
blob
.
size
,
'name'
:
names
[
'tag_uri'
]
,
'type'
:
"container"
}
blob
.
metadata
=
metadata
blob
.
_properties
[
'metadata'
]
=
metadata
blob
.
patch
(
)

a specific function to take a blob along with a SUCCESS response from Google build the original config and update the blob metadata with the artifact file name dependencies and image hash .


def format_container_name(name, special_characters=None):
    '''format_container_name will take a name supplied by the user,
    remove all special characters (except for those defined by "special-characters"
    and return the new image name.
    '''
    if special_characters is None:
        special_characters = []
    return ''.join(e.lower()
                   for e in name if e.isalnum() or e in special_characters)

def format_container_name ( name , special_characters = None ) : if special_characters is None : special_characters = [ ] return '' . join ( e . lower ( ) for e in name if e . isalnum ( ) or e in special_characters )

format_container_name will take a name supplied by the user remove all special characters ( except for those defined by special - characters and return the new image name .


def useColor(self):
        '''useColor will determine if color should be added
        to a print. Will check if being run in a terminal, and
        if has support for asci'''
        COLORIZE = get_user_color_preference()
        if COLORIZE is not None:
            return COLORIZE
        streams = [self.errorStream, self.outputStream]
        for stream in streams:
            if not hasattr(stream, 'isatty'):
                return False
            if not stream.isatty():
                return False
        return True

def useColor ( self ) : COLORIZE = get_user_color_preference ( ) if COLORIZE is not None : return COLORIZE streams = [ self . errorStream , self . outputStream ] for stream in streams : if not hasattr ( stream , 'isatty' ) : return False if not stream . isatty ( ) : return False return True

useColor will determine if color should be added to a print . Will check if being run in a terminal and if has support for asci


def emitError(self, level):
        '''determine if a level should print to
        stderr, includes all levels but INFO and QUIET'''
        if level in [ABORT,
                     ERROR,
                     WARNING,
                     VERBOSE,
                     VERBOSE1,
                     VERBOSE2,
                     VERBOSE3,
                     DEBUG]:
            return True
        return False

def emitError ( self , level ) : if level in [ ABORT , ERROR , WARNING , VERBOSE , VERBOSE1 , VERBOSE2 , VERBOSE3 , DEBUG ] : return True return False

determine if a level should print to stderr includes all levels but INFO and QUIET


def write(self, stream, message):
        '''write will write a message to a stream,
        first checking the encoding
        '''
        if isinstance(message, bytes):
            message = message.decode('utf-8')
        stream.write(message)

def write ( self , stream , message ) : if isinstance ( message , bytes ) : message = message . decode ( 'utf-8' ) stream . write ( message )

write will write a message to a stream first checking the encoding


def table(self, rows, col_width=2):
        '''table will print a table of entries. If the rows is
        a dictionary, the keys are interpreted as column names. if
        not, a numbered list is used.
        '''

        labels = [str(x) for x in range(1,len(rows)+1)]
        if isinstance(rows, dict):
            labels = list(rows.keys())
            rows = list(rows.values())

        for row in rows:
            label = labels.pop(0)
            label = label.ljust(col_width)
            message = "\t".join(row)
            self.custom(prefix=label,
                        message=message)

def table ( self , rows , col_width = 2 ) : labels = [ str ( x ) for x in range ( 1 , len ( rows ) + 1 ) ] if isinstance ( rows , dict ) : labels = list ( rows . keys ( ) ) rows = list ( rows . values ( ) ) for row in rows : label = labels . pop ( 0 ) label = label . ljust ( col_width ) message = "\t" . join ( row ) self . custom ( prefix = label , message = message )

table will print a table of entries . If the rows is a dictionary the keys are interpreted as column names . if not a numbered list is used .


def push(self, path, name, tag=None):
    '''push an image to Globus endpoint. In this case, the name is the
       globus endpoint id and path.

       --name <endpointid>:/path/for/image

    '''

    # Split the name into endpoint and rest

    endpoint, remote = self._parse_endpoint_name(name)

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    # Flatten image uri into image name

    q = parse_image_name(image)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Ensure we have a transfer client
    if not hasattr(self, 'transfer_client'):
        self._init_transfer_client()

    # The user must have a personal endpoint

    endpoints = self._get_endpoints()

    if len(endpoints['my-endpoints']) == 0:
        bot.error('You must have a personal endpoint to transfer the container')
        sys.exit(1)

    # Take the first endpoint that is active

    source_endpoint = None
    for eid,contender in endpoints['my-endpoints'].items():
       if contender['gcp_connected'] is True:
           source_endpoint = contender
           break

    # Exit if none are active, required!

    if source_endpoint is None:
        bot.error('No activated local endpoints online! Go online to transfer')
        sys.exit(1)

    # The destination endpoint should have an .singularity/shub folder set
    self._create_endpoint_cache(endpoint)

    # SREGISTRY_STORAGE must be an endpoint
    # if the image isn't already there, add it first

    added = self.add(image_path=path,
                     image_uri=q['uri'],
                     copy=True)

    label = "Singularity Registry Transfer for %s" %added.name
    tdata = globus_sdk.TransferData(self.transfer_client,
                                    source_endpoint['id'],
                                    endpoint,
                                    label=label,
                                    sync_level="checksum")
    image = ".singularity/shub/%s" %image
    tdata.add_item(added.image, image)
    bot.info('Requesting transfer from local %s to %s:%s' %(SREGISTRY_STORAGE,
                                                            endpoint, image))
    transfer_result = self.transfer_client.submit_transfer(tdata)
    bot.info(transfer_result['message'])
    return transfer_result

push an image to Globus endpoint . In this case the name is the globus endpoint id and path .


def get_template(name):
    '''return a default template for some function in sregistry
       If there is no template, None is returned.

       Parameters
       ==========
       name: the name of the template to retrieve

    '''
    name = name.lower()
    templates = dict()

    templates['tarinfo'] = {"gid": 0,
                            "uid": 0,
                            "uname": "root",
                            "gname": "root",
                            "mode": 493}

    if name in templates:
        bot.debug("Found template for %s" % (name))
        return templates[name]
    else:
        bot.warning("Cannot find template %s" % (name))

def
get_template
(
name
)
:
name
=
name
.
lower
(
)
templates
=
dict
(
)
templates
[
'tarinfo'
]
=
{
"gid"
:
0
,
"uid"
:
0
,
"uname"
:
"root"
,
"gname"
:
"root"
,
"mode"
:
493
}
if
name
in
templates
:
bot
.
debug
(
"Found template for %s"
%
(
name
)
)
return
templates
[
name
]
else
:
bot
.
warning
(
"Cannot find template %s"
%
(
name
)
)

return a default template for some function in sregistry If there is no template None is returned .


def get_manifest(self, repo_name, tag):
    '''return the image manifest via the aws client, saved in self.manifest
    '''

    image = None
    repo = self.aws.describe_images(repositoryName=repo_name)
    if 'imageDetails' in repo:
        for contender in repo.get('imageDetails'):
            if tag in contender['imageTags']:
                image = contender
                break

    # if the image isn't found, we need to exit
    if image is None:
        bot.exit('Cannot find %s:%s, is the uri correct?' %(repo_name, digest))

    digest = image['imageDigest']
    digests = self.aws.batch_get_image(repositoryName=repo_name,
                                       imageIds=[{"imageDigest": digest,
                                                  "imageTag": tag}])

    self.manifest = json.loads(digests['images'][0]['imageManifest'])
    return self.manifest

return the image manifest via the aws client saved in self . manifest


def get_build_template(name=None, manager='apt'):
    '''get a particular build template, by default we return templates
       that are based on package managers.

       Parameters
       ==========
       name: the full path of the template file to use.
       manager: the package manager to use in the template (yum or apt)

    '''
    base = get_installdir()
    if name is None:
        name = "%s/main/templates/build/singularity-builder-%s.sh" %(base,
                                                                     manager)

    if os.path.exists(name):
        bot.debug("Found template %s" %name)
        return ''.join(read_file(name))

    bot.warning("Template %s not found." %name)

get a particular build template by default we return templates that are based on package managers .