def resolve(cls, all_known_repos, name):
        """We require the list of all remote repo paths to be passed in
        to this because otherwise we would need to import the spec assembler
        in this module, which would give us circular imports."""
        match = None
        for repo in all_known_repos:
            if repo.remote_path == name: # user passed in a full name
                return repo
            if name == repo.short_name:
                if match is None:
                    match = repo
                else:
                    raise RuntimeError('Short repo name {} is ambiguous. It matches both {} and {}'.format(name,
                                                                                match.remote_path,
                                                                                repo.remote_path))
        if match is None:
            raise RuntimeError('Short repo name {} does not match any known repos'.format(name))
        return match

We require the list of all remote repo paths to be passed in to this because otherwise we would need to import the spec assembler in this module which would give us circular imports .


def ensure_local_repo(self):
        """Given a Dusty repo object, clone the remote into Dusty's local repos
        directory if it does not already exist."""
        if os.path.exists(self.managed_path):
            logging.debug('Repo {} already exists'.format(self.remote_path))
            return

        logging.info('Initiating clone of local repo {}'.format(self.remote_path))

        repo_path_parent = parent_dir(self.managed_path)
        if not os.path.exists(repo_path_parent):
            os.makedirs(repo_path_parent)
        with git_error_handling():
            git.Repo.clone_from(self.assemble_remote_path(), self.managed_path)

def
ensure_local_repo
(
self
)
:
if
os
.
path
.
exists
(
self
.
managed_path
)
:
logging
.
debug
(
'Repo {} already exists'
.
format
(
self
.
remote_path
)
)
return
logging
.
info
(
'Initiating clone of local repo {}'
.
format
(
self
.
remote_path
)
)
repo_path_parent
=
parent_dir
(
self
.
managed_path
)
if
not
os
.
path
.
exists
(
repo_path_parent
)
:
os
.
makedirs
(
repo_path_parent
)
with
git_error_handling
(
)
:
git
.
Repo
.
clone_from
(
self
.
assemble_remote_path
(
)
,
self
.
managed_path
)

Given a Dusty repo object clone the remote into Dusty s local repos directory if it does not already exist .


def update_local_repo_async(self, task_queue, force=False):
        """Local repo updating suitable for asynchronous, parallel execution.
        We still need to run `ensure_local_repo` synchronously because it
        does a bunch of non-threadsafe filesystem operations."""
        self.ensure_local_repo()
        task_queue.enqueue_task(self.update_local_repo, force=force)

def update_local_repo_async ( self , task_queue , force = False ) : self . ensure_local_repo ( ) task_queue . enqueue_task ( self . update_local_repo , force = force )

Local repo updating suitable for asynchronous parallel execution . We still need to run ensure_local_repo synchronously because it does a bunch of non - threadsafe filesystem operations .


def update_managed_repos(force=False):
    """For any active, managed repos, update the Dusty-managed
    copy to bring it up to date with the latest master."""
    log_to_client('Pulling latest updates for all active managed repos:')
    update_specs_repo_and_known_hosts()
    repos_to_update = get_all_repos(active_only=True, include_specs_repo=False)
    with parallel_task_queue() as queue:
        log_to_client('Updating managed repos')
        for repo in repos_to_update:
            if not repo.is_overridden:
                repo.update_local_repo_async(queue, force=force)

For any active managed repos update the Dusty - managed copy to bring it up to date with the latest master .


def prep_for_start_local_env(pull_repos):
    """Daemon-side command to ensure we're running the latest
    versions of any managed repos, including the
    specs repo, before we do anything else in the up flow."""
    if pull_repos:
        update_managed_repos(force=True)
    assembled_spec = spec_assembler.get_assembled_specs()
    if not assembled_spec[constants.CONFIG_BUNDLES_KEY]:
        raise RuntimeError('No bundles are activated. Use `dusty bundles` to activate bundles before running `dusty up`.')
    virtualbox.initialize_docker_vm()

Daemon - side command to ensure we re running the latest versions of any managed repos including the specs repo before we do anything else in the up flow .


def start_local_env(recreate_containers):
    """This command will use the compilers to get compose specs
    will pass those specs to the systems that need them. Those
    systems will in turn launch the services needed to make the
    local environment go."""

    assembled_spec = spec_assembler.get_assembled_specs()
    required_absent_assets = virtualbox.required_absent_assets(assembled_spec)
    if required_absent_assets:
        raise RuntimeError('Assets {} are specified as required but are not set. Set them with `dusty assets set`'.format(required_absent_assets))

    docker_ip = virtualbox.get_docker_vm_ip()

    # Stop will fail if we've never written a Composefile before
    if os.path.exists(constants.COMPOSEFILE_PATH):
        try:
            stop_apps_or_services(rm_containers=recreate_containers)
        except CalledProcessError as e:
            log_to_client("WARNING: docker-compose stop failed")
            log_to_client(str(e))

    daemon_warnings.clear_namespace('disk')
    df_info = virtualbox.get_docker_vm_disk_info(as_dict=True)
    if 'M' in df_info['free'] or 'K' in df_info['free']:
        warning_msg = 'VM is low on disk. Available disk: {}'.format(df_info['free'])
        daemon_warnings.warn('disk', warning_msg)
        log_to_client(warning_msg)

    log_to_client("Compiling together the assembled specs")
    active_repos = spec_assembler.get_all_repos(active_only=True, include_specs_repo=False)
    log_to_client("Compiling the port specs")
    port_spec = port_spec_compiler.get_port_spec_document(assembled_spec, docker_ip)
    log_to_client("Compiling the nginx config")
    docker_bridge_ip = virtualbox.get_docker_bridge_ip()
    nginx_config = nginx_compiler.get_nginx_configuration_spec(port_spec, docker_bridge_ip)
    log_to_client("Creating setup and script bash files")
    make_up_command_files(assembled_spec, port_spec)
    log_to_client("Compiling docker-compose config")
    compose_config = compose_compiler.get_compose_dict(assembled_spec, port_spec)

    log_to_client("Saving port forwarding to hosts file")
    hosts.update_hosts_file_from_port_spec(port_spec)
    log_to_client("Configuring NFS")
    nfs.configure_nfs()
    log_to_client("Saving updated nginx config to the VM")
    nginx.update_nginx_from_config(nginx_config)
    log_to_client("Saving Docker Compose config and starting all containers")
    compose.update_running_containers_from_spec(compose_config, recreate_containers=recreate_containers)

    log_to_client("Your local environment is now started!")

This command will use the compilers to get compose specs will pass those specs to the systems that need them . Those systems will in turn launch the services needed to make the local environment go .


def stop_apps_or_services(app_or_service_names=None, rm_containers=False):
    """Stop any currently running Docker containers associated with
    Dusty, or associated with the provided apps_or_services. Does not remove
    the service's containers."""
    if app_or_service_names:
        log_to_client("Stopping the following apps or services: {}".format(', '.join(app_or_service_names)))
    else:
        log_to_client("Stopping all running containers associated with Dusty")

    compose.stop_running_services(app_or_service_names)
    if rm_containers:
        compose.rm_containers(app_or_service_names)

Stop any currently running Docker containers associated with Dusty or associated with the provided apps_or_services . Does not remove the service s containers .


def restart_apps_or_services(app_or_service_names=None):
    """Restart any containers associated with Dusty, or associated with
    the provided app_or_service_names."""
    if app_or_service_names:
        log_to_client("Restarting the following apps or services: {}".format(', '.join(app_or_service_names)))
    else:
        log_to_client("Restarting all active containers associated with Dusty")

    if app_or_service_names:
        specs = spec_assembler.get_assembled_specs()
        specs_list = [specs['apps'][app_name] for app_name in app_or_service_names if app_name in specs['apps']]
        repos = set()
        for spec in specs_list:
            if spec['repo']:
                repos = repos.union(spec_assembler.get_same_container_repos_from_spec(spec))
        nfs.update_nfs_with_repos(repos)
    else:
        nfs.update_nfs_with_repos(spec_assembler.get_all_repos(active_only=True, include_specs_repo=False))
    compose.restart_running_services(app_or_service_names)

Restart any containers associated with Dusty or associated with the provided app_or_service_names .


def case_insensitive_rename(src, dst):
    """A hack to allow us to rename paths in a case-insensitive filesystem like HFS."""
    temp_dir = tempfile.mkdtemp()
    shutil.rmtree(temp_dir)
    shutil.move(src, temp_dir)
    shutil.move(temp_dir, dst)

def case_insensitive_rename ( src , dst ) : temp_dir = tempfile . mkdtemp ( ) shutil . rmtree ( temp_dir ) shutil . move ( src , temp_dir ) shutil . move ( temp_dir , dst )

A hack to allow us to rename paths in a case - insensitive filesystem like HFS .


def _compose_dict_for_nginx(port_specs):
    """Return a dictionary containing the Compose spec required to run
    Dusty's nginx container used for host forwarding."""
    spec = {'image': constants.NGINX_IMAGE,
            'volumes': ['{}:{}'.format(constants.NGINX_CONFIG_DIR_IN_VM, constants.NGINX_CONFIG_DIR_IN_CONTAINER)],
            'command': 'nginx -g "daemon off;" -c /etc/nginx/conf.d/nginx.primary',
            'container_name': 'dusty_{}_1'.format(constants.DUSTY_NGINX_NAME)}
    all_host_ports = set([nginx_spec['host_port'] for nginx_spec in port_specs['nginx']])
    if all_host_ports:
        spec['ports'] = []
        for port in all_host_ports:
            spec['ports'].append('{0}:{0}'.format(port))
    return {constants.DUSTY_NGINX_NAME: spec}

Return a dictionary containing the Compose spec required to run Dusty s nginx container used for host forwarding .


def get_compose_dict(assembled_specs, port_specs):
    """ This function returns a dictionary representation of a docker-compose.yml file, based on assembled_specs from
    the spec_assembler, and port_specs from the port_spec compiler """
    compose_dict = _compose_dict_for_nginx(port_specs)
    for app_name in assembled_specs['apps'].keys():
        compose_dict[app_name] = _composed_app_dict(app_name, assembled_specs, port_specs)
    for service_spec in assembled_specs['services'].values():
        compose_dict[service_spec.name] = _composed_service_dict(service_spec)
    return compose_dict

def get_compose_dict ( assembled_specs , port_specs ) : compose_dict = _compose_dict_for_nginx ( port_specs ) for app_name in assembled_specs [ 'apps' ] . keys ( ) : compose_dict [ app_name ] = _composed_app_dict ( app_name , assembled_specs , port_specs ) for service_spec in assembled_specs [ 'services' ] . values ( ) : compose_dict [ service_spec . name ] = _composed_service_dict ( service_spec ) return compose_dict

This function returns a dictionary representation of a docker - compose . yml file based on assembled_specs from the spec_assembler and port_specs from the port_spec compiler


def _conditional_links(assembled_specs, app_name):
    """ Given the assembled specs and app_name, this function will return all apps and services specified in
    'conditional_links' if they are specified in 'apps' or 'services' in assembled_specs. That means that
    some other part of the system has declared them as necessary, so they should be linked to this app """
    link_to_apps = []
    potential_links = assembled_specs['apps'][app_name]['conditional_links']
    for potential_link in potential_links['apps']:
        if potential_link in assembled_specs['apps']:
            link_to_apps.append(potential_link)
    for potential_link in potential_links['services']:
        if potential_link in assembled_specs['services']:
            link_to_apps.append(potential_link)
    return link_to_apps

def _conditional_links ( assembled_specs , app_name ) : link_to_apps = [ ] potential_links = assembled_specs [ 'apps' ] [ app_name ] [ 'conditional_links' ] for potential_link in potential_links [ 'apps' ] : if potential_link in assembled_specs [ 'apps' ] : link_to_apps . append ( potential_link ) for potential_link in potential_links [ 'services' ] : if potential_link in assembled_specs [ 'services' ] : link_to_apps . append ( potential_link ) return link_to_apps

Given the assembled specs and app_name this function will return all apps and services specified in conditional_links if they are specified in apps or services in assembled_specs . That means that some other part of the system has declared them as necessary so they should be linked to this app


def _get_build_path(app_spec):
    """ Given a spec for an app, returns the value of the `build` field for docker-compose.
    If the path is relative, it is expanded and added to the path of the app's repo. """
    if os.path.isabs(app_spec['build']):
        return app_spec['build']
    return os.path.join(Repo(app_spec['repo']).local_path, app_spec['build'])

def _get_build_path ( app_spec ) : if os . path . isabs ( app_spec [ 'build' ] ) : return app_spec [ 'build' ] return os . path . join ( Repo ( app_spec [ 'repo' ] ) . local_path , app_spec [ 'build' ] )

Given a spec for an app returns the value of the build field for docker - compose . If the path is relative it is expanded and added to the path of the app s repo .


def _composed_app_dict(app_name, assembled_specs, port_specs):
    """ This function returns a dictionary of the docker-compose.yml specifications for one app """
    logging.info("Compose Compiler: Compiling dict for app {}".format(app_name))
    app_spec = assembled_specs['apps'][app_name]
    compose_dict = app_spec["compose"]
    _apply_env_overrides(env_overrides_for_app_or_service(app_name), compose_dict)
    if 'image' in app_spec and 'build' in app_spec:
        raise RuntimeError("image and build are both specified in the spec for {}".format(app_name))
    elif 'image' in app_spec:
        logging.info
        compose_dict['image'] = app_spec['image']
    elif 'build' in app_spec:
        compose_dict['build'] = _get_build_path(app_spec)
    else:
        raise RuntimeError("Neither image nor build was specified in the spec for {}".format(app_name))
    compose_dict['entrypoint'] = []
    compose_dict['command'] = _compile_docker_command(app_spec)
    compose_dict['container_name'] = "dusty_{}_1".format(app_name)
    logging.info("Compose Compiler: compiled command {}".format(compose_dict['command']))
    compose_dict['links'] = _links_for_app(app_spec, assembled_specs)
    logging.info("Compose Compiler: links {}".format(compose_dict['links']))
    compose_dict['volumes'] = compose_dict['volumes'] + _get_compose_volumes(app_name, assembled_specs)
    logging.info("Compose Compiler: volumes {}".format(compose_dict['volumes']))
    port_list = _get_ports_list(app_name, port_specs)
    if port_list:
        compose_dict['ports'] = port_list
    logging.info("Compose Compiler: ports {}".format(port_list))
    compose_dict['user'] = 'root'
    return compose_dict

This function returns a dictionary of the docker - compose . yml specifications for one app


def _composed_service_dict(service_spec):
    """This function returns a dictionary of the docker_compose specifications
    for one service. Currently, this is just the Dusty service spec with
    an additional volume mount to support Dusty's cp functionality."""
    compose_dict = service_spec.plain_dict()
    _apply_env_overrides(env_overrides_for_app_or_service(service_spec.name), compose_dict)
    compose_dict.setdefault('volumes', []).append(_get_cp_volume_mount(service_spec.name))
    compose_dict['container_name'] = "dusty_{}_1".format(service_spec.name)
    return compose_dict

def _composed_service_dict ( service_spec ) : compose_dict = service_spec . plain_dict ( ) _apply_env_overrides ( env_overrides_for_app_or_service ( service_spec . name ) , compose_dict ) compose_dict . setdefault ( 'volumes' , [ ] ) . append ( _get_cp_volume_mount ( service_spec . name ) ) compose_dict [ 'container_name' ] = "dusty_{}_1" . format ( service_spec . name ) return compose_dict

This function returns a dictionary of the docker_compose specifications for one service . Currently this is just the Dusty service spec with an additional volume mount to support Dusty s cp functionality .


def _get_ports_list(app_name, port_specs):
    """ Returns a list of formatted port mappings for an app """
    if app_name not in port_specs['docker_compose']:
        return []
    return ["{}:{}".format(port_spec['mapped_host_port'], port_spec['in_container_port'])
            for port_spec in port_specs['docker_compose'][app_name]]

def _get_ports_list ( app_name , port_specs ) : if app_name not in port_specs [ 'docker_compose' ] : return [ ] return [ "{}:{}" . format ( port_spec [ 'mapped_host_port' ] , port_spec [ 'in_container_port' ] ) for port_spec in port_specs [ 'docker_compose' ] [ app_name ] ]


def _get_compose_volumes(app_name, assembled_specs):
    """ This returns formatted volume specifications for a docker-compose app. We mount the app
    as well as any libs it needs so that local code is used in our container, instead of whatever
    code was in the docker image.

    Additionally, we create a volume for the /cp directory used by Dusty to facilitate
    easy file transfers using `dusty cp`."""
    volumes = []
    volumes.append(_get_cp_volume_mount(app_name))
    volumes += get_app_volume_mounts(app_name, assembled_specs)
    return volumes

def _get_compose_volumes ( app_name , assembled_specs ) : volumes = [ ] volumes . append ( _get_cp_volume_mount ( app_name ) ) volumes += get_app_volume_mounts ( app_name , assembled_specs ) return volumes

This returns formatted volume specifications for a docker - compose app . We mount the app as well as any libs it needs so that local code is used in our container instead of whatever code was in the docker image .


def _env_vars_from_file(filename):
    """
    This code is copied from Docker Compose, so that we're exactly compatible
    with their `env_file` option
    """
    def split_env(env):
        if '=' in env:
            return env.split('=', 1)
        else:
            return env, None
    env = {}
    for line in open(filename, 'r'):
        line = line.strip()
        if line and not line.startswith('#'):
            k, v = split_env(line)
            env[k] = v
    return env

def _env_vars_from_file ( filename ) : def split_env ( env ) : if '=' in env : return env . split ( '=' , 1 ) else : return env , None env = { } for line in open ( filename , 'r' ) : line = line . strip ( ) if line and not line . startswith ( '#' ) : k , v = split_env ( line ) env [ k ] = v return env

This code is copied from Docker Compose so that we re exactly compatible with their env_file option


def _expand_libs_in_apps(specs):
    """
    Expands specs.apps.depends.libs to include any indirectly required libs
    """
    for app_name, app_spec in specs['apps'].iteritems():
        if 'depends' in app_spec and 'libs' in app_spec['depends']:
            app_spec['depends']['libs'] = _get_dependent('libs', app_name, specs, 'apps')

def _expand_libs_in_apps ( specs ) : for app_name , app_spec in specs [ 'apps' ] . iteritems ( ) : if 'depends' in app_spec and 'libs' in app_spec [ 'depends' ] : app_spec [ 'depends' ] [ 'libs' ] = _get_dependent ( 'libs' , app_name , specs , 'apps' )

Expands specs . apps . depends . libs to include any indirectly required libs


def _expand_libs_in_libs(specs):
    """
    Expands specs.libs.depends.libs to include any indirectly required libs
    """
    for lib_name, lib_spec in specs['libs'].iteritems():
        if 'depends' in lib_spec and 'libs' in lib_spec['depends']:
            lib_spec['depends']['libs'] = _get_dependent('libs', lib_name, specs, 'libs')

def _expand_libs_in_libs ( specs ) : for lib_name , lib_spec in specs [ 'libs' ] . iteritems ( ) : if 'depends' in lib_spec and 'libs' in lib_spec [ 'depends' ] : lib_spec [ 'depends' ] [ 'libs' ] = _get_dependent ( 'libs' , lib_name , specs , 'libs' )

Expands specs . libs . depends . libs to include any indirectly required libs


def _get_referenced_libs(specs):
    """
    Returns all libs that are referenced in specs.apps.depends.libs
    """
    active_libs = set()
    for app_spec in specs['apps'].values():
        for lib in app_spec['depends']['libs']:
            active_libs.add(lib)
    return active_libs

def _get_referenced_libs ( specs ) : active_libs = set ( ) for app_spec in specs [ 'apps' ] . values ( ) : for lib in app_spec [ 'depends' ] [ 'libs' ] : active_libs . add ( lib ) return active_libs

Returns all libs that are referenced in specs . apps . depends . libs


def _get_referenced_services(specs):
    """
    Returns all services that are referenced in specs.apps.depends.services,
    or in specs.bundles.services
    """
    active_services = set()
    for app_spec in specs['apps'].values():
        for service in app_spec['depends']['services']:
            active_services.add(service)
    for bundle_spec in specs['bundles'].values():
        for service in bundle_spec['services']:
            active_services.add(service)
    return active_services

def _get_referenced_services ( specs ) : active_services = set ( ) for app_spec in specs [ 'apps' ] . values ( ) : for service in app_spec [ 'depends' ] [ 'services' ] : active_services . add ( service ) for bundle_spec in specs [ 'bundles' ] . values ( ) : for service in bundle_spec [ 'services' ] : active_services . add ( service ) return active_services

Returns all services that are referenced in specs . apps . depends . services or in specs . bundles . services


def _add_active_assets(specs):
    """
    This function adds an assets key to the specs, which is filled in with a dictionary
    of all assets defined by apps and libs in the specs
    """
    specs['assets'] = {}
    for spec in specs.get_apps_and_libs():
        for asset in spec['assets']:
            if not specs['assets'].get(asset['name']):
                specs['assets'][asset['name']] = {}
                specs['assets'][asset['name']]['required_by'] = set()
                specs['assets'][asset['name']]['used_by'] = set()
            specs['assets'][asset['name']]['used_by'].add(spec.name)
            if asset['required']:
                specs['assets'][asset['name']]['required_by'].add(spec.name)

def _add_active_assets ( specs ) : specs [ 'assets' ] = { } for spec in specs . get_apps_and_libs ( ) : for asset in spec [ 'assets' ] : if not specs [ 'assets' ] . get ( asset [ 'name' ] ) : specs [ 'assets' ] [ asset [ 'name' ] ] = { } specs [ 'assets' ] [ asset [ 'name' ] ] [ 'required_by' ] = set ( ) specs [ 'assets' ] [ asset [ 'name' ] ] [ 'used_by' ] = set ( ) specs [ 'assets' ] [ asset [ 'name' ] ] [ 'used_by' ] . add ( spec . name ) if asset [ 'required' ] : specs [ 'assets' ] [ asset [ 'name' ] ] [ 'required_by' ] . add ( spec . name )

This function adds an assets key to the specs which is filled in with a dictionary of all assets defined by apps and libs in the specs


def _get_expanded_active_specs(specs):
    """
    This function removes any unnecessary bundles, apps, libs, and services that aren't needed by
    the activated_bundles.  It also expands inside specs.apps.depends.libs all libs that are needed
    indirectly by each app
    """
    _filter_active(constants.CONFIG_BUNDLES_KEY, specs)
    _filter_active('apps', specs)
    _expand_libs_in_apps(specs)
    _filter_active('libs', specs)
    _filter_active('services', specs)
    _add_active_assets(specs)

def _get_expanded_active_specs ( specs ) : _filter_active ( constants . CONFIG_BUNDLES_KEY , specs ) _filter_active ( 'apps' , specs ) _expand_libs_in_apps ( specs ) _filter_active ( 'libs' , specs ) _filter_active ( 'services' , specs ) _add_active_assets ( specs )

This function removes any unnecessary bundles apps libs and services that aren t needed by the activated_bundles . It also expands inside specs . apps . depends . libs all libs that are needed indirectly by each app


def get_repo_of_app_or_library(app_or_library_name):
    """ This function takes an app or library name and will return the corresponding repo
    for that app or library"""
    specs = get_specs()
    repo_name = specs.get_app_or_lib(app_or_library_name)['repo']
    if not repo_name:
        return None
    return Repo(repo_name)

def get_repo_of_app_or_library ( app_or_library_name ) : specs = get_specs ( ) repo_name = specs . get_app_or_lib ( app_or_library_name ) [ 'repo' ] if not repo_name : return None return Repo ( repo_name )


def get_same_container_repos_from_spec(app_or_library_spec):
    """Given the spec of an app or library, returns all repos that are guaranteed
    to live in the same container"""
    repos = set()
    app_or_lib_repo = get_repo_of_app_or_library(app_or_library_spec.name)
    if app_or_lib_repo is not None:
        repos.add(app_or_lib_repo)
    for dependent_name in app_or_library_spec['depends']['libs']:
        repos.add(get_repo_of_app_or_library(dependent_name))
    return repos

def get_same_container_repos_from_spec ( app_or_library_spec ) : repos = set ( ) app_or_lib_repo = get_repo_of_app_or_library ( app_or_library_spec . name ) if app_or_lib_repo is not None : repos . add ( app_or_lib_repo ) for dependent_name in app_or_library_spec [ 'depends' ] [ 'libs' ] : repos . add ( get_repo_of_app_or_library ( dependent_name ) ) return repos

Given the spec of an app or library returns all repos that are guaranteed to live in the same container


def get_same_container_repos(app_or_library_name):
    """Given the name of an app or library, returns all repos that are guaranteed
    to live in the same container"""
    specs = get_expanded_libs_specs()
    spec = specs.get_app_or_lib(app_or_library_name)
    return get_same_container_repos_from_spec(spec)

def get_same_container_repos ( app_or_library_name ) : specs = get_expanded_libs_specs ( ) spec = specs . get_app_or_lib ( app_or_library_name ) return get_same_container_repos_from_spec ( spec )

Given the name of an app or library returns all repos that are guaranteed to live in the same container


def _dusty_hosts_config(hosts_specs):
    """Return a string of all host rules required to match
    the given spec. This string is wrapped in the Dusty hosts
    header and footer so it can be easily removed later."""
    rules =  ''.join(['{} {}\n'.format(spec['forwarded_ip'], spec['host_address']) for spec in hosts_specs])
    return config_file.create_config_section(rules)

def
_dusty_hosts_config
(
hosts_specs
)
:
rules
=
''
.
join
(
[
'{} {}\n'
.
format
(
spec
[
'forwarded_ip'
]
,
spec
[
'host_address'
]
)
for
spec
in
hosts_specs
]
)
return
config_file
.
create_config_section
(
rules
)

Return a string of all host rules required to match the given spec . This string is wrapped in the Dusty hosts header and footer so it can be easily removed later .


def update_hosts_file_from_port_spec(port_spec):
    """Given a port spec, update the hosts file specified at
    constants.HOST_PATH to contain the port mappings specified
    in the spec. Any existing Dusty configurations are replaced."""
    logging.info('Updating hosts file to match port spec')
    hosts_specs = port_spec['hosts_file']
    current_hosts = config_file.read(constants.HOSTS_PATH)
    cleared_hosts = config_file.remove_current_dusty_config(current_hosts)
    updated_hosts = cleared_hosts + _dusty_hosts_config(hosts_specs)
    config_file.write(constants.HOSTS_PATH, updated_hosts)

Given a port spec update the hosts file specified at constants . HOST_PATH to contain the port mappings specified in the spec . Any existing Dusty configurations are replaced .


def _move_temp_binary_to_path(tmp_binary_path):
    """Moves the temporary binary to the location of the binary that's currently being run.
    Preserves owner, group, and permissions of original binary"""
    # pylint: disable=E1101
    binary_path = _get_binary_location()
    if not binary_path.endswith(constants.DUSTY_BINARY_NAME):
        raise RuntimeError('Refusing to overwrite binary {}'.format(binary_path))
    st = os.stat(binary_path)
    permissions = st.st_mode
    owner = st.st_uid
    group = st.st_gid
    shutil.move(tmp_binary_path, binary_path)
    os.chown(binary_path, owner, group)
    os.chmod(binary_path, permissions)
    return binary_path

def
_move_temp_binary_to_path
(
tmp_binary_path
)
:
binary_path
=
_get_binary_location
(
)
if
not
binary_path
.
endswith
(
constants
.
DUSTY_BINARY_NAME
)
:
raise
RuntimeError
(
'Refusing to overwrite binary {}'
.
format
(
binary_path
)
)
st
=
os
.
stat
(
binary_path
)
permissions
=
st
.
st_mode
owner
=
st
.
st_uid
group
=
st
.
st_gid
shutil
.
move
(
tmp_binary_path
,
binary_path
)
os
.
chown
(
binary_path
,
owner
,
group
)
os
.
chmod
(
binary_path
,
permissions
)
return
binary_path

Moves the temporary binary to the location of the binary that s currently being run . Preserves owner group and permissions of original binary


def parallel_task_queue(pool_size=multiprocessing.cpu_count()):
    """Context manager for setting up a TaskQueue. Upon leaving the
    context manager, all tasks that were enqueued will be executed
    in parallel subject to `pool_size` concurrency constraints."""
    task_queue = TaskQueue(pool_size)
    yield task_queue
    task_queue.execute()

def parallel_task_queue ( pool_size = multiprocessing . cpu_count ( ) ) : task_queue = TaskQueue ( pool_size ) yield task_queue task_queue . execute ( )

Context manager for setting up a TaskQueue . Upon leaving the context manager all tasks that were enqueued will be executed in parallel subject to pool_size concurrency constraints .


def _nginx_location_spec(port_spec, bridge_ip):
    """This will output the nginx location config string for specific port spec """
    location_string_spec = "\t \t location / { \n"
    for location_setting in ['proxy_http_version 1.1;',
                             'proxy_set_header Upgrade $http_upgrade;',
                             'proxy_set_header Connection "upgrade";',
                             'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;',
                             'proxy_set_header Host $http_host;',
                             _nginx_proxy_string(port_spec, bridge_ip)]:
        location_string_spec += "\t \t \t {} \n".format(location_setting)
    location_string_spec += "\t \t } \n"
    return location_string_spec


def _nginx_http_spec(port_spec, bridge_ip):
    """This will output the nginx HTTP config string for specific port spec """
    server_string_spec = "\t server {\n"
    server_string_spec += "\t \t {}\n".format(_nginx_max_file_size_string())
    server_string_spec += "\t \t {}\n".format(_nginx_listen_string(port_spec))
    server_string_spec += "\t \t {}\n".format(_nginx_server_name_string(port_spec))
    server_string_spec += _nginx_location_spec(port_spec, bridge_ip)
    server_string_spec += _custom_502_page()
    server_string_spec += "\t }\n"
    return server_string_spec

def
_nginx_http_spec
(
port_spec
,
bridge_ip
)
:
server_string_spec
=
"\t server {\n"
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_max_file_size_string
(
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_listen_string
(
port_spec
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_server_name_string
(
port_spec
)
)
server_string_spec
+=
_nginx_location_spec
(
port_spec
,
bridge_ip
)
server_string_spec
+=
_custom_502_page
(
)
server_string_spec
+=
"\t }\n"
return
server_string_spec


def _nginx_stream_spec(port_spec, bridge_ip):
    """This will output the nginx stream config string for specific port spec """
    server_string_spec = "\t server {\n"
    server_string_spec += "\t \t {}\n".format(_nginx_listen_string(port_spec))
    server_string_spec += "\t \t {}\n".format(_nginx_proxy_string(port_spec, bridge_ip))
    server_string_spec += "\t }\n"
    return server_string_spec

def
_nginx_stream_spec
(
port_spec
,
bridge_ip
)
:
server_string_spec
=
"\t server {\n"
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_listen_string
(
port_spec
)
)
server_string_spec
+=
"\t \t {}\n"
.
format
(
_nginx_proxy_string
(
port_spec
,
bridge_ip
)
)
server_string_spec
+=
"\t }\n"
return
server_string_spec


def get_nginx_configuration_spec(port_spec_dict, docker_bridge_ip):
    """This function will take in a port spec as specified by the port_spec compiler and
    will output an nginx web proxy config string. This string can then be written to a file
    and used running nginx """
    nginx_http_config, nginx_stream_config = "", ""
    for port_spec in port_spec_dict['nginx']:
        if port_spec['type'] == 'http':
            nginx_http_config += _nginx_http_spec(port_spec, docker_bridge_ip)
        elif port_spec['type'] == 'stream':
            nginx_stream_config += _nginx_stream_spec(port_spec, docker_bridge_ip)
    return {'http': nginx_http_config, 'stream': nginx_stream_config}

def get_nginx_configuration_spec ( port_spec_dict , docker_bridge_ip ) : nginx_http_config , nginx_stream_config = "" , "" for port_spec in port_spec_dict [ 'nginx' ] : if port_spec [ 'type' ] == 'http' : nginx_http_config += _nginx_http_spec ( port_spec , docker_bridge_ip ) elif port_spec [ 'type' ] == 'stream' : nginx_stream_config += _nginx_stream_spec ( port_spec , docker_bridge_ip ) return { 'http' : nginx_http_config , 'stream' : nginx_stream_config }

This function will take in a port spec as specified by the port_spec compiler and will output an nginx web proxy config string . This string can then be written to a file and used running nginx


def _load_ssh_auth_post_yosemite(mac_username):
    """Starting with Yosemite, launchd was rearchitected and now only one
    launchd process runs for all users. This allows us to much more easily
    impersonate a user through launchd and extract the environment
    variables from their running processes."""
    user_id = subprocess.check_output(['id', '-u', mac_username])
    ssh_auth_sock = subprocess.check_output(['launchctl', 'asuser', user_id, 'launchctl', 'getenv', 'SSH_AUTH_SOCK']).rstrip()
    _set_ssh_auth_sock(ssh_auth_sock)

def _load_ssh_auth_post_yosemite ( mac_username ) : user_id = subprocess . check_output ( [ 'id' , '-u' , mac_username ] ) ssh_auth_sock = subprocess . check_output ( [ 'launchctl' , 'asuser' , user_id , 'launchctl' , 'getenv' , 'SSH_AUTH_SOCK' ] ) . rstrip ( ) _set_ssh_auth_sock ( ssh_auth_sock )

Starting with Yosemite launchd was rearchitected and now only one launchd process runs for all users . This allows us to much more easily impersonate a user through launchd and extract the environment variables from their running processes .


def _load_ssh_auth_pre_yosemite():
    """For OS X versions before Yosemite, many launchd processes run simultaneously under
    different users and different permission models. The simpler `asuser` trick we use
    in Yosemite doesn't work, since it gets routed to the wrong launchd. We instead need
    to find the running ssh-agent process and use its PID to navigate ourselves
    to the correct launchd."""
    for process in psutil.process_iter():
        if process.name() == 'ssh-agent':
            ssh_auth_sock = subprocess.check_output(['launchctl', 'bsexec', str(process.pid), 'launchctl', 'getenv', 'SSH_AUTH_SOCK']).rstrip()
            if ssh_auth_sock:
                _set_ssh_auth_sock(ssh_auth_sock)
                break
    else:
        daemon_warnings.warn('ssh', 'No running ssh-agent found linked to SSH_AUTH_SOCK')

For OS X versions before Yosemite many launchd processes run simultaneously under different users and different permission models . The simpler asuser trick we use in Yosemite doesn t work since it gets routed to the wrong launchd . We instead need to find the running ssh - agent process and use its PID to navigate ourselves to the correct launchd .


def check_and_load_ssh_auth():
    """
    Will check the mac_username config value; if it is present, will load that user's
    SSH_AUTH_SOCK environment variable to the current environment.  This allows git clones
    to behave the same for the daemon as they do for the user
    """

    mac_username = get_config_value(constants.CONFIG_MAC_USERNAME_KEY)
    if not mac_username:
        logging.info("Can't setup ssh authorization; no mac_username specified")
        return
    if not _running_on_mac(): # give our Linux unit tests a way to not freak out
        logging.info("Skipping SSH load, we are not running on Mac")
        return

    if _mac_version_is_post_yosemite():
        _load_ssh_auth_post_yosemite(mac_username)
    else:
        _load_ssh_auth_pre_yosemite()

Will check the mac_username config value ; if it is present will load that user s SSH_AUTH_SOCK environment variable to the current environment . This allows git clones to behave the same for the daemon as they do for the user


def _cleanup_path(path):
    """Recursively delete a path upon exiting this context
    manager. Supports targets that are files or directories."""
    try:
        yield
    finally:
        if os.path.exists(path):
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

def _cleanup_path ( path ) : try : yield finally : if os . path . exists ( path ) : if os . path . isdir ( path ) : shutil . rmtree ( path ) else : os . remove ( path )

Recursively delete a path upon exiting this context manager . Supports targets that are files or directories .


def copy_between_containers(source_name, source_path, dest_name, dest_path):
    """Copy a file from the source container to an intermediate staging
    area on the local filesystem, then from that staging area to the
    destination container.

    These moves take place without demotion for two reasons:
      1. There should be no permissions vulnerabilities with copying
         between containers because it is assumed the non-privileged
         user has full access to all Dusty containers.
      2. The temp dir created by mkdtemp is owned by the owner of the
         Dusty daemon process, so if we demoted our moves to/from that location
         they would encounter permission errors."""
    if not container_path_exists(source_name, source_path):
        raise RuntimeError('ERROR: Path {} does not exist inside container {}.'.format(source_path, source_name))
    temp_path = os.path.join(tempfile.mkdtemp(), str(uuid.uuid1()))
    with _cleanup_path(temp_path):
        copy_to_local(temp_path, source_name, source_path, demote=False)
        copy_from_local(temp_path, dest_name, dest_path, demote=False)

Copy a file from the source container to an intermediate staging area on the local filesystem then from that staging area to the destination container .


def copy_from_local(local_path, remote_name, remote_path, demote=True):
    """Copy a path from the local filesystem to a path inside a Dusty
    container. The files on the local filesystem must be accessible
    by the user specified in mac_username."""
    if not os.path.exists(local_path):
        raise RuntimeError('ERROR: Path {} does not exist'.format(local_path))
    temp_identifier = str(uuid.uuid1())
    if os.path.isdir(local_path):
        sync_local_path_to_vm(local_path, os.path.join(vm_cp_path(remote_name), temp_identifier), demote=demote)
        move_dir_inside_container(remote_name, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier), remote_path)
    else:
        sync_local_path_to_vm(local_path, os.path.join(vm_cp_path(remote_name), temp_identifier), demote=demote)
        move_file_inside_container(remote_name, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier), remote_path)

def
copy_from_local
(
local_path
,
remote_name
,
remote_path
,
demote
=
True
)
:
if
not
os
.
path
.
exists
(
local_path
)
:
raise
RuntimeError
(
'ERROR: Path {} does not exist'
.
format
(
local_path
)
)
temp_identifier
=
str
(
uuid
.
uuid1
(
)
)
if
os
.
path
.
isdir
(
local_path
)
:
sync_local_path_to_vm
(
local_path
,
os
.
path
.
join
(
vm_cp_path
(
remote_name
)
,
temp_identifier
)
,
demote
=
demote
)
move_dir_inside_container
(
remote_name
,
os
.
path
.
join
(
constants
.
CONTAINER_CP_DIR
,
temp_identifier
)
,
remote_path
)
else
:
sync_local_path_to_vm
(
local_path
,
os
.
path
.
join
(
vm_cp_path
(
remote_name
)
,
temp_identifier
)
,
demote
=
demote
)
move_file_inside_container
(
remote_name
,
os
.
path
.
join
(
constants
.
CONTAINER_CP_DIR
,
temp_identifier
)
,
remote_path
)

Copy a path from the local filesystem to a path inside a Dusty container . The files on the local filesystem must be accessible by the user specified in mac_username .


def copy_to_local(local_path, remote_name, remote_path, demote=True):
    """Copy a path from inside a Dusty container to a path on the
    local filesystem. The path on the local filesystem must be
    wrist-accessible by the user specified in mac_username."""
    if not container_path_exists(remote_name, remote_path):
        raise RuntimeError('ERROR: Path {} does not exist inside container {}.'.format(remote_path, remote_name))
    temp_identifier = str(uuid.uuid1())
    copy_path_inside_container(remote_name, remote_path, os.path.join(constants.CONTAINER_CP_DIR, temp_identifier))
    vm_path = os.path.join(vm_cp_path(remote_name), temp_identifier)
    is_dir = vm_path_is_directory(vm_path)
    sync_local_path_from_vm(local_path, vm_path, demote=demote, is_dir=is_dir)

Copy a path from inside a Dusty container to a path on the local filesystem . The path on the local filesystem must be wrist - accessible by the user specified in mac_username .


def _mount_repo(repo, wait_for_server=False):
    """
    This function will create the VM directory where a repo will be mounted, if it
    doesn't exist.  If wait_for_server is set, it will wait up to 10 seconds for
    the nfs server to start, by retrying mounts that fail with 'Connection Refused'.

    If wait_for_server is not set, it will attempt to run the mount command once
    """
    check_call_on_vm('sudo mkdir -p {}'.format(repo.vm_path))
    if wait_for_server:
        for i in range(0,10):
            try:
                _run_mount_command(repo)
                return
            except CalledProcessError as e:
                if 'Connection refused' in e.output:
                    logging.info('Failed to mount repo; waiting for nfsd to restart')
                    time.sleep(1)
                else:
                    logging.info(e.output)
                    raise e
        log_to_client('Failed to mount repo {}'.format(repo.short_name))
        raise RuntimeError('Unable to mount repo with NFS')
    else:
        _run_mount_command(repo)

This function will create the VM directory where a repo will be mounted if it doesn t exist . If wait_for_server is set it will wait up to 10 seconds for the nfs server to start by retrying mounts that fail with Connection Refused .


def get_port_spec_document(expanded_active_specs, docker_vm_ip):
    """ Given a dictionary containing the expanded dusty DAG specs this function will
    return a dictionary containing the port mappings needed by downstream methods.  Currently
    this includes docker_compose, virtualbox, nginx and hosts_file."""
    forwarding_port = 65000
    port_spec = {'docker_compose':{}, 'nginx':[], 'hosts_file':[]}
    host_full_addresses, host_names, stream_host_ports = set(), set(), set()
    # No matter the order of apps in expanded_active_specs, we want to produce a consistent
    # port_spec with respect to the apps and the ports they are outputted on
    for app_name in sorted(expanded_active_specs['apps'].keys()):
        app_spec = expanded_active_specs['apps'][app_name]
        if 'host_forwarding' not in app_spec:
            continue
        port_spec['docker_compose'][app_name] = []
        for host_forwarding_spec in app_spec['host_forwarding']:
            # These functions are just used for validating the set of specs all works together
            _add_full_addresses(host_forwarding_spec, host_full_addresses)
            if host_forwarding_spec['type'] == 'stream':
                _add_stream_host_port(host_forwarding_spec, stream_host_ports)

            port_spec['docker_compose'][app_name].append(_docker_compose_port_spec(host_forwarding_spec, forwarding_port))
            port_spec['nginx'].append(_nginx_port_spec(host_forwarding_spec, forwarding_port, docker_vm_ip))

            _add_host_names(host_forwarding_spec, docker_vm_ip, port_spec, host_names)
            forwarding_port += 1
    return port_spec

def get_port_spec_document ( expanded_active_specs , docker_vm_ip ) : forwarding_port = 65000 port_spec = { 'docker_compose' : { } , 'nginx' : [ ] , 'hosts_file' : [ ] } host_full_addresses , host_names , stream_host_ports = set ( ) , set ( ) , set ( ) for app_name in sorted ( expanded_active_specs [ 'apps' ] . keys ( ) ) : app_spec = expanded_active_specs [ 'apps' ] [ app_name ] if 'host_forwarding' not in app_spec : continue port_spec [ 'docker_compose' ] [ app_name ] = [ ] for host_forwarding_spec in app_spec [ 'host_forwarding' ] : _add_full_addresses ( host_forwarding_spec , host_full_addresses ) if host_forwarding_spec [ 'type' ] == 'stream' : _add_stream_host_port ( host_forwarding_spec , stream_host_ports ) port_spec [ 'docker_compose' ] [ app_name ] . append ( _docker_compose_port_spec ( host_forwarding_spec , forwarding_port ) ) port_spec [ 'nginx' ] . append ( _nginx_port_spec ( host_forwarding_spec , forwarding_port , docker_vm_ip ) ) _add_host_names ( host_forwarding_spec , docker_vm_ip , port_spec , host_names ) forwarding_port += 1 return port_spec

Given a dictionary containing the expanded dusty DAG specs this function will return a dictionary containing the port mappings needed by downstream methods . Currently this includes docker_compose virtualbox nginx and hosts_file .


def init_yaml_constructor():
    """
    This dark magic is used to make yaml.safe_load encode all strings as utf-8,
    where otherwise python unicode strings would be returned for non-ascii chars
    """
    def utf_encoding_string_constructor(loader, node):
        return loader.construct_scalar(node).encode('utf-8')
    yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', utf_encoding_string_constructor)

def init_yaml_constructor ( ) : def utf_encoding_string_constructor ( loader , node ) : return loader . construct_scalar ( node ) . encode ( 'utf-8' ) yaml . SafeLoader . add_constructor ( u'tag:yaml.org,2002:str' , utf_encoding_string_constructor )

This dark magic is used to make yaml . safe_load encode all strings as utf - 8 where otherwise python unicode strings would be returned for non - ascii chars


def registry_from_image(image_name):
    """Returns the Docker registry host associated with
    a given image name."""
    if '/' not in image_name: # official image
        return constants.PUBLIC_DOCKER_REGISTRY
    prefix = image_name.split('/')[0]
    if '.' not in prefix: # user image on official repository, e.g. thieman/clojure
        return constants.PUBLIC_DOCKER_REGISTRY
    return prefix

def registry_from_image ( image_name ) : if '/' not in image_name : return constants . PUBLIC_DOCKER_REGISTRY prefix = image_name . split ( '/' ) [ 0 ] if '.' not in prefix : return constants . PUBLIC_DOCKER_REGISTRY return prefix

Returns the Docker registry host associated with a given image name .


def get_authed_registries():
    """Reads the local Docker client config for the current user
    and returns all registries to which the user may be logged in.
    This is intended to be run client-side, not by the daemon."""
    result = set()
    if not os.path.exists(constants.DOCKER_CONFIG_PATH):
        return result
    config = json.load(open(constants.DOCKER_CONFIG_PATH, 'r'))
    for registry in config.get('auths', {}).iterkeys():
        try:
            parsed = urlparse(registry)
        except Exception:
            log_to_client('Error parsing registry {} from Docker config, will skip this registry').format(registry)
        # This logic assumes the auth is either of the form
        # gamechanger.io (no scheme, no path after host) or
        # of the form https://index.docker.io/v1/ (scheme,
        # netloc parses correctly, additional path does not matter).
        # These are the formats I saw in my personal config file,
        # not sure what other formats it might accept.
        result.add(parsed.netloc) if parsed.netloc else result.add(parsed.path)
    return result

Reads the local Docker client config for the current user and returns all registries to which the user may be logged in . This is intended to be run client - side not by the daemon .


def streaming_to_client():
    """Puts the client logger into streaming mode, which sends
    unbuffered input through to the socket one character at a time.
    We also disable propagation so the root logger does not
    receive many one-byte emissions. This context handler
    was originally created for streaming Compose up's
    terminal output through to the client and should only be
    used for similarly complex circumstances."""
    for handler in client_logger.handlers:
        if hasattr(handler, 'append_newlines'):
            break
    else:
        handler = None
    old_propagate = client_logger.propagate
    client_logger.propagate = False
    if handler is not None:
        old_append = handler.append_newlines
        handler.append_newlines = False
    yield
    client_logger.propagate = old_propagate
    if handler is not None:
        handler.append_newlines = old_append

def streaming_to_client ( ) : for handler in client_logger . handlers : if hasattr ( handler , 'append_newlines' ) : break else : handler = None old_propagate = client_logger . propagate client_logger . propagate = False if handler is not None : old_append = handler . append_newlines handler . append_newlines = False yield client_logger . propagate = old_propagate if handler is not None : handler . append_newlines = old_append

Puts the client logger into streaming mode which sends unbuffered input through to the socket one character at a time . We also disable propagation so the root logger does not receive many one - byte emissions . This context handler was originally created for streaming Compose up s terminal output through to the client and should only be used for similarly complex circumstances .


def pty_fork(*args):
    """Runs a subprocess with a PTY attached via fork and exec.
    The output from the PTY is streamed through log_to_client.
    This should not be necessary for most subprocesses, we
    built this to handle Compose up which only streams pull
    progress if it is attached to a TTY."""

    updated_env = copy(os.environ)
    updated_env.update(get_docker_env())
    args += (updated_env,)
    executable = args[0]
    demote_fn = demote_to_user(get_config_value(constants.CONFIG_MAC_USERNAME_KEY))

    child_pid, pty_fd = pty.fork()
    if child_pid == 0:
        demote_fn()
        os.execle(_executable_path(executable), *args)
    else:
        child_process = psutil.Process(child_pid)
        terminal = os.fdopen(pty_fd, 'r', 0)
        with streaming_to_client():
            while child_process.status() == 'running':
                output = terminal.read(1)
                log_to_client(output)
        _, exit_code = os.waitpid(child_pid, 0)
        if exit_code != 0:
            raise subprocess.CalledProcessError(exit_code, ' '.join(args[:-1]))

def
pty_fork
(
*
args
)
:
updated_env
=
copy
(
os
.
environ
)
updated_env
.
update
(
get_docker_env
(
)
)
args
+=
(
updated_env
,
)
executable
=
args
[
0
]
demote_fn
=
demote_to_user
(
get_config_value
(
constants
.
CONFIG_MAC_USERNAME_KEY
)
)
child_pid
,
pty_fd
=
pty
.
fork
(
)
if
child_pid
==
0
:
demote_fn
(
)
os
.
execle
(
_executable_path
(
executable
)
,
*
args
)
else
:
child_process
=
psutil
.
Process
(
child_pid
)
terminal
=
os
.
fdopen
(
pty_fd
,
'r'
,
0
)
with
streaming_to_client
(
)
:
while
child_process
.
status
(
)
==
'running'
:
output
=
terminal
.
read
(
1
)
log_to_client
(
output
)
_
,
exit_code
=
os
.
waitpid
(
child_pid
,
0
)
if
exit_code
!=
0
:
raise
subprocess
.
CalledProcessError
(
exit_code
,
' '
.
join
(
args
[
:
1
]
)
)

Runs a subprocess with a PTY attached via fork and exec . The output from the PTY is streamed through log_to_client . This should not be necessary for most subprocesses we built this to handle Compose up which only streams pull progress if it is attached to a TTY .


def _compile_docker_commands(app_name, assembled_specs, port_spec):
    """ This is used to compile the command that will be run when the docker container starts
    up. This command has to install any libs that the app uses, run the `always` command, and
    run the `once` command if the container is being launched for the first time """
    app_spec = assembled_specs['apps'][app_name]
    commands = ['set -e']
    commands += _lib_install_commands_for_app(app_name, assembled_specs)
    if app_spec['mount']:
        commands.append("cd {}".format(container_code_path(app_spec)))
        commands.append("export PATH=$PATH:{}".format(container_code_path(app_spec)))
    commands += _copy_assets_commands_for_app(app_spec, assembled_specs)
    commands += _get_once_commands(app_spec, port_spec)
    commands += _get_always_commands(app_spec)
    return commands

def
_compile_docker_commands
(
app_name
,
assembled_specs
,
port_spec
)
:
app_spec
=
assembled_specs
[
'apps'
]
[
app_name
]
commands
=
[
'set -e'
]
commands
+=
_lib_install_commands_for_app
(
app_name
,
assembled_specs
)
if
app_spec
[
'mount'
]
:
commands
.
append
(
"cd {}"
.
format
(
container_code_path
(
app_spec
)
)
)
commands
.
append
(
"export PATH=$PATH:{}"
.
format
(
container_code_path
(
app_spec
)
)
)
commands
+=
_copy_assets_commands_for_app
(
app_spec
,
assembled_specs
)
commands
+=
_get_once_commands
(
app_spec
,
port_spec
)
commands
+=
_get_always_commands
(
app_spec
)
return
commands

This is used to compile the command that will be run when the docker container starts up . This command has to install any libs that the app uses run the always command and run the once command if the container is being launched for the first time


def _increase_file_handle_limit():
    """Raise the open file handles permitted by the Dusty daemon process
    and its child processes. The number we choose here needs to be within
    the OS X default kernel hard limit, which is 10240."""
    logging.info('Increasing file handle limit to {}'.format(constants.FILE_HANDLE_LIMIT))
    resource.setrlimit(resource.RLIMIT_NOFILE,
                       (constants.FILE_HANDLE_LIMIT, resource.RLIM_INFINITY))

def
_increase_file_handle_limit
(
)
:
logging
.
info
(
'Increasing file handle limit to {}'
.
format
(
constants
.
FILE_HANDLE_LIMIT
)
)
resource
.
setrlimit
(
resource
.
RLIMIT_NOFILE
,
(
constants
.
FILE_HANDLE_LIMIT
,
resource
.
RLIM_INFINITY
)
)

Raise the open file handles permitted by the Dusty daemon process and its child processes . The number we choose here needs to be within the OS X default kernel hard limit which is 10240 .


def _start_http_server():
    """Start the daemon's HTTP server on a separate thread.
    This server is only used for servicing container status
    requests from Dusty's custom 502 page."""
    logging.info('Starting HTTP server at {}:{}'.format(constants.DAEMON_HTTP_BIND_IP,
                                                        constants.DAEMON_HTTP_BIND_PORT))
    thread = threading.Thread(target=http_server.app.run, args=(constants.DAEMON_HTTP_BIND_IP,
                                                                constants.DAEMON_HTTP_BIND_PORT))
    thread.daemon = True
    thread.start()

def
_start_http_server
(
)
:
logging
.
info
(
'Starting HTTP server at {}:{}'
.
format
(
constants
.
DAEMON_HTTP_BIND_IP
,
constants
.
DAEMON_HTTP_BIND_PORT
)
)
thread
=
threading
.
Thread
(
target
=
http_server
.
app
.
run
,
args
=
(
constants
.
DAEMON_HTTP_BIND_IP
,
constants
.
DAEMON_HTTP_BIND_PORT
)
)
thread
.
daemon
=
True
thread
.
start
(
)

Start the daemon s HTTP server on a separate thread . This server is only used for servicing container status requests from Dusty s custom 502 page .


def get_docker_client():
    """Ripped off and slightly modified based on docker-py's
    kwargs_from_env utility function."""
    env = get_docker_env()
    host, cert_path, tls_verify = env['DOCKER_HOST'], env['DOCKER_CERT_PATH'], env['DOCKER_TLS_VERIFY']

    params = {'base_url': host.replace('tcp://', 'https://'),
              'timeout': None,
              'version': 'auto'}
    if tls_verify and cert_path:
        params['tls'] = docker.tls.TLSConfig(
            client_cert=(os.path.join(cert_path, 'cert.pem'),
                         os.path.join(cert_path, 'key.pem')),
            ca_cert=os.path.join(cert_path, 'ca.pem'),
            verify=True,
            ssl_version=None,
            assert_hostname=False)
    return docker.Client(**params)

def get_docker_client ( ) : env = get_docker_env ( ) host , cert_path , tls_verify = env [ 'DOCKER_HOST' ] , env [ 'DOCKER_CERT_PATH' ] , env [ 'DOCKER_TLS_VERIFY' ] params = { 'base_url' : host . replace ( 'tcp://' , 'https://' ) , 'timeout' : None , 'version' : 'auto' } if tls_verify and cert_path : params [ 'tls' ] = docker . tls . TLSConfig ( client_cert = ( os . path . join ( cert_path , 'cert.pem' ) , os . path . join ( cert_path , 'key.pem' ) ) , ca_cert = os . path . join ( cert_path , 'ca.pem' ) , verify = True , ssl_version = None , assert_hostname = False ) return docker . Client ( ** params )

Ripped off and slightly modified based on docker - py s kwargs_from_env utility function .


def get_dusty_containers(services, include_exited=False):
    """Get a list of containers associated with the list
    of services. If no services are provided, attempts to
    return all containers associated with Dusty."""
    client = get_docker_client()
    if services:
        containers = [get_container_for_app_or_service(service, include_exited=include_exited) for service in services]
        return [container for container in containers if container]
    else:
        return [container
                for container in client.containers(all=include_exited)
                if any(name.startswith('/dusty') for name in container.get('Names', []))]

def get_dusty_containers ( services , include_exited = False ) : client = get_docker_client ( ) if services : containers = [ get_container_for_app_or_service ( service , include_exited = include_exited ) for service in services ] return [ container for container in containers if container ] else : return [ container for container in client . containers ( all = include_exited ) if any ( name . startswith ( '/dusty' ) for name in container . get ( 'Names' , [ ] ) ) ]

Get a list of containers associated with the list of services . If no services are provided attempts to return all containers associated with Dusty .


def configure_nfs_server():
    """
    This function is used with `dusty up`.  It will check all active repos to see if
    they are exported.  If any are missing, it will replace current dusty exports with
    exports that are needed for currently active repos, and restart
    the nfs server
    """
    repos_for_export = get_all_repos(active_only=True, include_specs_repo=False)

    current_exports = _get_current_exports()
    needed_exports = _get_exports_for_repos(repos_for_export)

    _ensure_managed_repos_dir_exists()

    if not needed_exports.difference(current_exports):
        if not _server_is_running():
            _restart_server()
        return

    _write_exports_config(needed_exports)
    _restart_server()

def configure_nfs_server ( ) : repos_for_export = get_all_repos ( active_only = True , include_specs_repo = False ) current_exports = _get_current_exports ( ) needed_exports = _get_exports_for_repos ( repos_for_export ) _ensure_managed_repos_dir_exists ( ) if not needed_exports . difference ( current_exports ) : if not _server_is_running ( ) : _restart_server ( ) return _write_exports_config ( needed_exports ) _restart_server ( )

This function is used with dusty up . It will check all active repos to see if they are exported . If any are missing it will replace current dusty exports with exports that are needed for currently active repos and restart the nfs server


def _ensure_managed_repos_dir_exists():
    """
    Our exports file will be invalid if this folder doesn't exist, and the NFS server
    will not run correctly.
    """
    if not os.path.exists(constants.REPOS_DIR):
        os.makedirs(constants.REPOS_DIR)

def _ensure_managed_repos_dir_exists ( ) : if not os . path . exists ( constants . REPOS_DIR ) : os . makedirs ( constants . REPOS_DIR )

Our exports file will be invalid if this folder doesn t exist and the NFS server will not run correctly .


def register_consumer():
    """Given a hostname and port attempting to be accessed,
    return a unique consumer ID for accessing logs from
    the referenced container."""
    global _consumers
    hostname, port = request.form['hostname'], request.form['port']

    app_name = _app_name_from_forwarding_info(hostname, port)
    containers = get_dusty_containers([app_name], include_exited=True)
    if not containers:
        raise ValueError('No container exists for app {}'.format(app_name))
    container = containers[0]

    new_id = uuid1()
    new_consumer = Consumer(container['Id'], datetime.utcnow())
    _consumers[str(new_id)] = new_consumer

    response = jsonify({'app_name': app_name, 'consumer_id': new_id})
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST'
    return response

def
register_consumer
(
)
:
global
_consumers
hostname
,
port
=
request
.
form
[
'hostname'
]
,
request
.
form
[
'port'
]
app_name
=
_app_name_from_forwarding_info
(
hostname
,
port
)
containers
=
get_dusty_containers
(
[
app_name
]
,
include_exited
=
True
)
if
not
containers
:
raise
ValueError
(
'No container exists for app {}'
.
format
(
app_name
)
)
container
=
containers
[
0
]
new_id
=
uuid1
(
)
new_consumer
=
Consumer
(
container
[
'Id'
]
,
datetime
.
utcnow
(
)
)
_consumers
[
str
(
new_id
)
]
=
new_consumer
response
=
jsonify
(
{
'app_name'
:
app_name
,
'consumer_id'
:
new_id
}
)
response
.
headers
[
'Access-Control-Allow-Origin'
]
=
'*'
response
.
headers
[
'Access-Control-Allow-Methods'
]
=
'GET, POST'
return
response

Given a hostname and port attempting to be accessed return a unique consumer ID for accessing logs from the referenced container .


def consume(consumer_id):
    """Given an existing consumer ID, return any new lines from the
    log since the last time the consumer was consumed."""
    global _consumers
    consumer = _consumers[consumer_id]

    client = get_docker_client()
    try:
        status = client.inspect_container(consumer.container_id)['State']['Status']
    except Exception as e:
        status = 'unknown'
    new_logs = client.logs(consumer.container_id,
                           stdout=True,
                           stderr=True,
                           stream=False,
                           timestamps=False,
                           since=calendar.timegm(consumer.offset.timetuple()))

    updated_consumer = Consumer(consumer.container_id, datetime.utcnow())
    _consumers[str(consumer_id)] = updated_consumer

    response = jsonify({'logs': new_logs, 'status': status})
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST'
    return response

def
consume
(
consumer_id
)
:
global
_consumers
consumer
=
_consumers
[
consumer_id
]
client
=
get_docker_client
(
)
try
:
status
=
client
.
inspect_container
(
consumer
.
container_id
)
[
'State'
]
[
'Status'
]
except
Exception
as
e
:
status
=
'unknown'
new_logs
=
client
.
logs
(
consumer
.
container_id
,
stdout
=
True
,
stderr
=
True
,
stream
=
False
,
timestamps
=
False
,
since
=
calendar
.
timegm
(
consumer
.
offset
.
timetuple
(
)
)
)
updated_consumer
=
Consumer
(
consumer
.
container_id
,
datetime
.
utcnow
(
)
)
_consumers
[
str
(
consumer_id
)
]
=
updated_consumer
response
=
jsonify
(
{
'logs'
:
new_logs
,
'status'
:
status
}
)
response
.
headers
[
'Access-Control-Allow-Origin'
]
=
'*'
response
.
headers
[
'Access-Control-Allow-Methods'
]
=
'GET, POST'
return
response

Given an existing consumer ID return any new lines from the log since the last time the consumer was consumed .


def get_app_volume_mounts(app_name, assembled_specs, test=False):
    """ This returns a list of formatted volume specs for an app. These mounts declared in the apps' spec
    and mounts declared in all lib specs the app depends on"""
    app_spec = assembled_specs['apps'][app_name]
    volumes = [get_command_files_volume_mount(app_name, test=test)]
    volumes.append(get_asset_volume_mount(app_name))
    repo_mount = _get_app_repo_volume_mount(app_spec)
    if repo_mount:
        volumes.append(repo_mount)
    volumes += _get_app_libs_volume_mounts(app_name, assembled_specs)
    return volumes

def get_app_volume_mounts ( app_name , assembled_specs , test = False ) : app_spec = assembled_specs [ 'apps' ] [ app_name ] volumes = [ get_command_files_volume_mount ( app_name , test = test ) ] volumes . append ( get_asset_volume_mount ( app_name ) ) repo_mount = _get_app_repo_volume_mount ( app_spec ) if repo_mount : volumes . append ( repo_mount ) volumes += _get_app_libs_volume_mounts ( app_name , assembled_specs ) return volumes

This returns a list of formatted volume specs for an app . These mounts declared in the apps spec and mounts declared in all lib specs the app depends on


def get_lib_volume_mounts(base_lib_name, assembled_specs):
    """ Returns a list of the formatted volume specs for a lib"""
    volumes = [_get_lib_repo_volume_mount(assembled_specs['libs'][base_lib_name])]
    volumes.append(get_command_files_volume_mount(base_lib_name, test=True))
    for lib_name in assembled_specs['libs'][base_lib_name]['depends']['libs']:
        lib_spec = assembled_specs['libs'][lib_name]
        volumes.append(_get_lib_repo_volume_mount(lib_spec))
    return volumes

def get_lib_volume_mounts ( base_lib_name , assembled_specs ) : volumes = [ _get_lib_repo_volume_mount ( assembled_specs [ 'libs' ] [ base_lib_name ] ) ] volumes . append ( get_command_files_volume_mount ( base_lib_name , test = True ) ) for lib_name in assembled_specs [ 'libs' ] [ base_lib_name ] [ 'depends' ] [ 'libs' ] : lib_spec = assembled_specs [ 'libs' ] [ lib_name ] volumes . append ( _get_lib_repo_volume_mount ( lib_spec ) ) return volumes


def _get_app_libs_volume_mounts(app_name, assembled_specs):
    """ Returns a list of the formatted volume mounts for all libs that an app uses """
    volumes = []
    for lib_name in assembled_specs['apps'][app_name]['depends']['libs']:
        lib_spec = assembled_specs['libs'][lib_name]
        volumes.append("{}:{}".format(Repo(lib_spec['repo']).vm_path, container_code_path(lib_spec)))
    return volumes

def _get_app_libs_volume_mounts ( app_name , assembled_specs ) : volumes = [ ] for lib_name in assembled_specs [ 'apps' ] [ app_name ] [ 'depends' ] [ 'libs' ] : lib_spec = assembled_specs [ 'libs' ] [ lib_name ] volumes . append ( "{}:{}" . format ( Repo ( lib_spec [ 'repo' ] ) . vm_path , container_code_path ( lib_spec ) ) ) return volumes


def _dusty_vm_exists():
    """We use VBox directly instead of Docker Machine because it
    shaves about 0.5 seconds off the runtime of this check."""
    existing_vms = check_output_demoted(['VBoxManage', 'list', 'vms'])
    for line in existing_vms.splitlines():
        if '"{}"'.format(constants.VM_MACHINE_NAME) in line:
            return True
    return False

def _dusty_vm_exists ( ) : existing_vms = check_output_demoted ( [ 'VBoxManage' , 'list' , 'vms' ] ) for line in existing_vms . splitlines ( ) : if '"{}"' . format ( constants . VM_MACHINE_NAME ) in line : return True return False

We use VBox directly instead of Docker Machine because it shaves about 0 . 5 seconds off the runtime of this check .


def _init_docker_vm():
    """Initialize the Dusty VM if it does not already exist."""
    if not _dusty_vm_exists():
        log_to_client('Initializing new Dusty VM with Docker Machine')
        machine_options = ['--driver', 'virtualbox',
                           '--virtualbox-cpu-count', '-1',
                           '--virtualbox-boot2docker-url', constants.CONFIG_BOOT2DOCKER_URL,
                           '--virtualbox-memory', str(get_config_value(constants.CONFIG_VM_MEM_SIZE)),
                           '--virtualbox-hostonly-nictype', constants.VM_NIC_TYPE]
        check_call_demoted(['docker-machine', 'create'] + machine_options + [constants.VM_MACHINE_NAME],
                           redirect_stderr=True)

Initialize the Dusty VM if it does not already exist .


def _start_docker_vm():
    """Start the Dusty VM if it is not already running."""
    is_running = docker_vm_is_running()
    if not is_running:
        log_to_client('Starting docker-machine VM {}'.format(constants.VM_MACHINE_NAME))
        _apply_nat_dns_host_resolver()
        _apply_nat_net_less_greedy_subnet()
        check_and_log_output_and_error_demoted(['docker-machine', 'start', constants.VM_MACHINE_NAME], quiet_on_success=True)
    return is_running

def
_start_docker_vm
(
)
:
is_running
=
docker_vm_is_running
(
)
if
not
is_running
:
log_to_client
(
'Starting docker-machine VM {}'
.
format
(
constants
.
VM_MACHINE_NAME
)
)
_apply_nat_dns_host_resolver
(
)
_apply_nat_net_less_greedy_subnet
(
)
check_and_log_output_and_error_demoted
(
[
'docker-machine'
,
'start'
,
constants
.
VM_MACHINE_NAME
]
,
quiet_on_success
=
True
)
return
is_running

Start the Dusty VM if it is not already running .


def docker_vm_is_running():
    """Using VBoxManage is 0.5 seconds or so faster than Machine."""
    running_vms = check_output_demoted(['VBoxManage', 'list', 'runningvms'])
    for line in running_vms.splitlines():
        if '"{}"'.format(constants.VM_MACHINE_NAME) in line:
            return True
    return False

def docker_vm_is_running ( ) : running_vms = check_output_demoted ( [ 'VBoxManage' , 'list' , 'runningvms' ] ) for line in running_vms . splitlines ( ) : if '"{}"' . format ( constants . VM_MACHINE_NAME ) in line : return True return False

Using VBoxManage is 0 . 5 seconds or so faster than Machine .


def _get_localhost_ssh_port():
    """Something in the VM chain, either VirtualBox or Machine, helpfully
    sets up localhost-to-VM forwarding on port 22. We can inspect this
    rule to determine the port on localhost which gets forwarded to
    22 in the VM."""
    for line in _get_vm_config():
        if line.startswith('Forwarding'):
            spec = line.split('=')[1].strip('"')
            name, protocol, host, host_port, target, target_port = spec.split(',')
            if name == 'ssh' and protocol == 'tcp' and target_port == '22':
                return host_port
    raise ValueError('Could not determine localhost port for SSH forwarding')

Something in the VM chain either VirtualBox or Machine helpfully sets up localhost - to - VM forwarding on port 22 . We can inspect this rule to determine the port on localhost which gets forwarded to 22 in the VM .


def _get_host_only_mac_address():
    """Returns the MAC address assigned to the host-only adapter,
    using output from VBoxManage. Returned MAC address has no colons
    and is lower-cased."""
    # Get the number of the host-only adapter
    vm_config = _get_vm_config()
    for line in vm_config:
        if line.startswith('hostonlyadapter'):
            adapter_number = int(line[15:16])
            break
    else:
        raise ValueError('No host-only adapter is defined for the Dusty VM')

    for line in vm_config:
        if line.startswith('macaddress{}'.format(adapter_number)):
            return line.split('=')[1].strip('"').lower()
    raise ValueError('Could not find MAC address for adapter number {}'.format(adapter_number))

Returns the MAC address assigned to the host - only adapter using output from VBoxManage . Returned MAC address has no colons and is lower - cased .


def _ip_for_mac_from_ip_addr_show(ip_addr_show, target_mac):
    """Given the rather-complex output from an 'ip addr show' command
    on the VM, parse the output to determine the IP address
    assigned to the interface with the given MAC."""
    return_next_ip = False
    for line in ip_addr_show.splitlines():
        line = line.strip()
        if line.startswith('link/ether'):
            line_mac = line.split(' ')[1].replace(':', '')
            if line_mac == target_mac:
                return_next_ip = True
        elif return_next_ip and line.startswith('inet') and not line.startswith('inet6'):
            ip = line.split(' ')[1].split('/')[0]
            return ip

def
_ip_for_mac_from_ip_addr_show
(
ip_addr_show
,
target_mac
)
:
return_next_ip
=
False
for
line
in
ip_addr_show
.
splitlines
(
)
:
line
=
line
.
strip
(
)
if
line
.
startswith
(
'link/ether'
)
:
line_mac
=
line
.
split
(
' '
)
[
1
]
.
replace
(
':'
,
''
)
if
line_mac
==
target_mac
:
return_next_ip
=
True
elif
return_next_ip
and
line
.
startswith
(
'inet'
)
and
not
line
.
startswith
(
'inet6'
)
:
ip
=
line
.
split
(
' '
)
[
1
]
.
split
(
'/'
)
[
0
]
return
ip

Given the rather - complex output from an ip addr show command on the VM parse the output to determine the IP address assigned to the interface with the given MAC .


def _get_host_only_ip():
    """Determine the host-only IP of the Dusty VM through Virtualbox and SSH
    directly, bypassing Docker Machine. We do this because Docker Machine is
    much slower, taking about 600ms total. We are basically doing the same
    flow Docker Machine does in its own code."""
    mac = _get_host_only_mac_address()
    ip_addr_show = check_output_demoted(['ssh', '-o', 'StrictHostKeyChecking=no',
                                         '-o', 'UserKnownHostsFile=/dev/null',
                                         '-i', _vm_key_path(), '-p', _get_localhost_ssh_port(),
                                         'docker@127.0.0.1', 'ip addr show'])
    return _ip_for_mac_from_ip_addr_show(ip_addr_show, mac)

def
_get_host_only_ip
(
)
:
mac
=
_get_host_only_mac_address
(
)
ip_addr_show
=
check_output_demoted
(
[
'ssh'
,
'-o'
,
'StrictHostKeyChecking=no'
,
'-o'
,
'UserKnownHostsFile=/dev/null'
,
'-i'
,
_vm_key_path
(
)
,
'-p'
,
_get_localhost_ssh_port
(
)
,
'docker@127.0.0.1'
,
'ip addr show'
]
)
return
_ip_for_mac_from_ip_addr_show
(
ip_addr_show
,
mac
)

Determine the host - only IP of the Dusty VM through Virtualbox and SSH directly bypassing Docker Machine . We do this because Docker Machine is much slower taking about 600ms total . We are basically doing the same flow Docker Machine does in its own code .


def create_local_copy(cookie_file):
    """Make a local copy of the sqlite cookie database and return the new filename.
    This is necessary in case this database is still being written to while the user browses
    to avoid sqlite locking errors.
    """
    # if type of cookie_file is a list, use the first element in the list
    if isinstance(cookie_file, list):
        cookie_file = cookie_file[0]

    # check if cookie file exists
    if os.path.exists(cookie_file):
        # copy to random name in tmp folder
        tmp_cookie_file = tempfile.NamedTemporaryFile(suffix='.sqlite').name
        open(tmp_cookie_file, 'wb').write(open(cookie_file, 'rb').read())
        return tmp_cookie_file
    else:
        raise BrowserCookieError('Can not find cookie file at: ' + cookie_file)

def
create_local_copy
(
cookie_file
)
:
if
isinstance
(
cookie_file
,
list
)
:
cookie_file
=
cookie_file
[
0
]
if
os
.
path
.
exists
(
cookie_file
)
:
tmp_cookie_file
=
tempfile
.
NamedTemporaryFile
(
suffix
=
'.sqlite'
)
.
name
open
(
tmp_cookie_file
,
'wb'
)
.
write
(
open
(
cookie_file
,
'rb'
)
.
read
(
)
)
return
tmp_cookie_file
else
:
raise
BrowserCookieError
(
'Can not find cookie file at: '
+
cookie_file
)

Make a local copy of the sqlite cookie database and return the new filename . This is necessary in case this database is still being written to while the user browses to avoid sqlite locking errors .


def create_cookie(host, path, secure, expires, name, value):
    """Shortcut function to create a cookie
    """
    return http.cookiejar.Cookie(0, name, value, None, False, host, host.startswith('.'), host.startswith('.'), path,
                                 True, secure, expires, False, None, None, {})

def create_cookie ( host , path , secure , expires , name , value ) : return http . cookiejar . Cookie ( 0 , name , value , None , False , host , host . startswith ( '.' ) , host . startswith ( '.' ) , path , True , secure , expires , False , None , None , { } )


def load(domain_name=""):
    """Try to load cookies from all supported browsers and return combined cookiejar
    Optionally pass in a domain name to only load cookies from the specified domain
    """
    cj = http.cookiejar.CookieJar()
    for cookie_fn in [chrome, firefox]:
        try:
            for cookie in cookie_fn(domain_name=domain_name):
                cj.set_cookie(cookie)
        except BrowserCookieError:
            pass
    return cj

def load ( domain_name = "" ) : cj = http . cookiejar . CookieJar ( ) for cookie_fn in [ chrome , firefox ] : try : for cookie in cookie_fn ( domain_name = domain_name ) : cj . set_cookie ( cookie ) except BrowserCookieError : pass return cj


def load(self):
        """Load sqlite cookies into a cookiejar
        """
        con = sqlite3.connect(self.tmp_cookie_file)
        cur = con.cursor()
        try:
            # chrome <=55
            cur.execute('SELECT host_key, path, secure, expires_utc, name, value, encrypted_value '
                        'FROM cookies WHERE host_key like "%{}%";'.format(self.domain_name))
        except sqlite3.OperationalError:
            # chrome >=56
            cur.execute('SELECT host_key, path, is_secure, expires_utc, name, value, encrypted_value '
                        'FROM cookies WHERE host_key like "%{}%";'.format(self.domain_name))

        cj = http.cookiejar.CookieJar()
        for item in cur.fetchall():
            host, path, secure, expires, name = item[:5]
            value = self._decrypt(item[5], item[6])
            c = create_cookie(host, path, secure, expires, name, value)
            cj.set_cookie(c)
        con.close()
        return cj


def _decrypt(self, value, encrypted_value):
        """Decrypt encoded cookies
        """

        if sys.platform == 'win32':
            return self._decrypt_windows_chrome(value, encrypted_value)

        if value or (encrypted_value[:3] != b'v10'):
            return value

        # Encrypted cookies should be prefixed with 'v10' according to the
        # Chromium code. Strip it off.
        encrypted_value = encrypted_value[3:]
        encrypted_value_half_len = int(len(encrypted_value) / 2)

        cipher = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(self.key, self.iv))
        decrypted = cipher.feed(encrypted_value[:encrypted_value_half_len])
        decrypted += cipher.feed(encrypted_value[encrypted_value_half_len:])
        decrypted += cipher.feed()
        return decrypted.decode("utf-8")

def _decrypt ( self , value , encrypted_value ) : if sys . platform == 'win32' : return self . _decrypt_windows_chrome ( value , encrypted_value ) if value or ( encrypted_value [ : 3 ] != b'v10' ) : return value encrypted_value = encrypted_value [ 3 : ] encrypted_value_half_len = int ( len ( encrypted_value ) / 2 ) cipher = pyaes . Decrypter ( pyaes . AESModeOfOperationCBC ( self . key , self . iv ) ) decrypted = cipher . feed ( encrypted_value [ : encrypted_value_half_len ] ) decrypted += cipher . feed ( encrypted_value [ encrypted_value_half_len : ] ) decrypted += cipher . feed ( ) return decrypted . decode ( "utf-8" )


def _randone(d, limit=20, grouprefs=None):
    if grouprefs is None:
        grouprefs = {}
    """docstring for _randone"""
    ret = ''
    for i in d:
        if i[0] == sre_parse.IN:
            ret += choice(_in(i[1]))
        elif i[0] == sre_parse.LITERAL:
            ret += unichr(i[1])
        elif i[0] == sre_parse.CATEGORY:
            ret += choice(CATEGORIES.get(i[1], ['']))
        elif i[0] == sre_parse.ANY:
            ret += choice(CATEGORIES['category_any'])
        elif i[0] == sre_parse.MAX_REPEAT or i[0] == sre_parse.MIN_REPEAT:
            if i[1][1] + 1 - i[1][0] >= limit:
                min, max = i[1][0], i[1][0] + limit - 1
            else:
                min, max = i[1][0], i[1][1]
            for _ in range(randint(min, max)):
                ret += _randone(list(i[1][2]), limit, grouprefs)
        elif i[0] == sre_parse.BRANCH:
            ret += _randone(choice(i[1][1]), limit, grouprefs)
        elif i[0] == sre_parse.SUBPATTERN or i[0] == sre_parse.ASSERT:
            subexpr = i[1][1]
            if IS_PY36_OR_GREATER and i[0] == sre_parse.SUBPATTERN:
                subexpr = i[1][3]
            subp = _randone(subexpr, limit, grouprefs)
            if i[1][0]:
                grouprefs[i[1][0]] = subp
            ret += subp
        elif i[0] == sre_parse.AT:
            continue
        elif i[0] == sre_parse.NOT_LITERAL:
            c = list(CATEGORIES['category_any'])
            if unichr(i[1]) in c:
                c.remove(unichr(i[1]))
            ret += choice(c)
        elif i[0] == sre_parse.GROUPREF:
            ret += grouprefs[i[1]]
        elif i[0] == sre_parse.ASSERT_NOT:
            pass
        else:
            print('[!] cannot handle expression "%s"' % str(i))

    return ret

def
_randone
(
d
,
limit
=
20
,
grouprefs
=
)
:
if
grouprefs
is
:
grouprefs
=
{
}
ret
=
''
for
i
in
d
:
if
i
[
0
]
==
sre_parse
.
IN
:
ret
+=
choice
(
_in
(
i
[
1
]
)
)
elif
i
[
0
]
==
sre_parse
.
LITERAL
:
ret
+=
unichr
(
i
[
1
]
)
elif
i
[
0
]
==
sre_parse
.
CATEGORY
:
ret
+=
choice
(
CATEGORIES
.
get
(
i
[
1
]
,
[
''
]
)
)
elif
i
[
0
]
==
sre_parse
.
ANY
:
ret
+=
choice
(
CATEGORIES
[
'category_any'
]
)
elif
i
[
0
]
==
sre_parse
.
MAX_REPEAT
or
i
[
0
]
==
sre_parse
.
MIN_REPEAT
:
if
i
[
1
]
[
1
]
+
1
i
[
1
]
[
0
]
>=
limit
:
min
,
max
=
i
[
1
]
[
0
]
,
i
[
1
]
[
0
]
+
limit
1
else
:
min
,
max
=
i
[
1
]
[
0
]
,
i
[
1
]
[
1
]
for
_
in
range
(
randint
(
min
,
max
)
)
:
ret
+=
_randone
(
list
(
i
[
1
]
[
2
]
)
,
limit
,
grouprefs
)
elif
i
[
0
]
==
sre_parse
.
BRANCH
:
ret
+=
_randone
(
choice
(
i
[
1
]
[
1
]
)
,
limit
,
grouprefs
)
elif
i
[
0
]
==
sre_parse
.
SUBPATTERN
or
i
[
0
]
==
sre_parse
.
ASSERT
:
subexpr
=
i
[
1
]
[
1
]
if
IS_PY36_OR_GREATER
and
i
[
0
]
==
sre_parse
.
SUBPATTERN
:
subexpr
=
i
[
1
]
[
3
]
subp
=
_randone
(
subexpr
,
limit
,
grouprefs
)
if
i
[
1
]
[
0
]
:
grouprefs
[
i
[
1
]
[
0
]
]
=
subp
ret
+=
subp
elif
i
[
0
]
==
sre_parse
.
AT
:
continue
elif
i
[
0
]
==
sre_parse
.
NOT_LITERAL
:
c
=
list
(
CATEGORIES
[
'category_any'
]
)
if
unichr
(
i
[
1
]
)
in
c
:
c
.
remove
(
unichr
(
i
[
1
]
)
)
ret
+=
choice
(
c
)
elif
i
[
0
]
==
sre_parse
.
GROUPREF
:
ret
+=
grouprefs
[
i
[
1
]
]
elif
i
[
0
]
==
sre_parse
.
ASSERT_NOT
:
pass
else
:
print
(
'[!] cannot handle expression "%s"'
%
str
(
i
)
)
return
ret


def parse(s):
    """Regular expression parser

    :param s: Regular expression
    :type s: str
    :rtype: list
    """
    if IS_PY3:
        r = sre_parse.parse(s, flags=U)
    else:
        r = sre_parse.parse(s.decode('utf-8'), flags=U)
    return list(r)

def parse ( s ) : if IS_PY3 : r = sre_parse . parse ( s , flags = U ) else : r = sre_parse . parse ( s . decode ( 'utf-8' ) , flags = U ) return list ( r )


def ib64_patched(self, attrsD, contentparams):
    """ Patch isBase64 to prevent Base64 encoding of JSON content
    """
    if attrsD.get("mode", "") == "base64":
        return 0
    if self.contentparams["type"].startswith("text/"):
        return 0
    if self.contentparams["type"].endswith("+xml"):
        return 0
    if self.contentparams["type"].endswith("/xml"):
        return 0
    if self.contentparams["type"].endswith("/json"):
        return 0
    return 0

def ib64_patched ( self , attrsD , contentparams ) : if attrsD . get ( "mode" , "" ) == "base64" : return 0 if self . contentparams [ "type" ] . startswith ( "text/" ) : return 0 if self . contentparams [ "type" ] . endswith ( "+xml" ) : return 0 if self . contentparams [ "type" ] . endswith ( "/xml" ) : return 0 if self . contentparams [ "type" ] . endswith ( "/json" ) : return 0 return 0


def cleanwrap(func):
    """ Wrapper for Zotero._cleanup
    """

    def enc(self, *args, **kwargs):
        """ Send each item to _cleanup() """
        return (func(self, item, **kwargs) for item in args)

    return enc

def cleanwrap ( func ) : def enc ( self , * args , ** kwargs ) : return ( func ( self , item , ** kwargs ) for item in args ) return enc


def ss_wrap(func):
    """ ensure that a SavedSearch object exists """

    def wrapper(self, *args, **kwargs):
        if not self.savedsearch:
            self.savedsearch = SavedSearch(self)
        return func(self, *args, **kwargs)

    return wrapper

def ss_wrap ( func ) : def wrapper ( self , * args , ** kwargs ) : if not self . savedsearch : self . savedsearch = SavedSearch ( self ) return func ( self , * args , ** kwargs ) return wrapper


def error_handler(req):
    """ Error handler for HTTP requests
    """
    error_codes = {
        400: ze.UnsupportedParams,
        401: ze.UserNotAuthorised,
        403: ze.UserNotAuthorised,
        404: ze.ResourceNotFound,
        409: ze.Conflict,
        412: ze.PreConditionFailed,
        413: ze.RequestEntityTooLarge,
        428: ze.PreConditionRequired,
        429: ze.TooManyRequests,
    }

    def err_msg(req):
        """ Return a nicely-formatted error message
        """
        return "\nCode: %s\nURL: %s\nMethod: %s\nResponse: %s" % (
            req.status_code,
            # error.msg,
            req.url,
            req.request.method,
            req.text,
        )

    if error_codes.get(req.status_code):
        # check to see whether its 429
        if req.status_code == 429:
            # call our back-off function
            delay = backoff.delay
            if delay > 32:
                # we've waited a total of 62 seconds (2 + 4 … + 32), so give up
                backoff.reset()
                raise ze.TooManyRetries(
                    "Continuing to receive HTTP 429 \
responses after 62 seconds. You are being rate-limited, try again later"
                )
            time.sleep(delay)
            sess = requests.Session()
            new_req = sess.send(req.request)
            try:
                new_req.raise_for_status()
            except requests.exceptions.HTTPError:
                error_handler(new_req)
        else:
            raise error_codes.get(req.status_code)(err_msg(req))
    else:
        raise ze.HTTPError(err_msg(req))

"Continuing to receive HTTP 429 \responses after 62 seconds. You are being rate-limited, try again later"


def default_headers(self):
        """
        It's always OK to include these headers
        """
        _headers = {
            "User-Agent": "Pyzotero/%s" % __version__,
            "Zotero-API-Version": "%s" % __api_version__,
        }
        if self.api_key:
            _headers["Authorization"] = "Bearer %s" % self.api_key
        return _headers

def
default_headers
(
self
)
:
_headers
=
{
"User-Agent"
:
"Pyzotero/%s"
%
__version__
,
"Zotero-API-Version"
:
"%s"
%
__api_version__
,
}
if
self
.
api_key
:
_headers
[
"Authorization"
]
=
"Bearer %s"
%
self
.
api_key
return
_headers


def _cache(self, response, key):
        """
        Add a retrieved template to the cache for 304 checking
        accepts a dict and key name, adds the retrieval time, and adds both
        to self.templates as a new dict using the specified key
        """
        # cache template and retrieval time for subsequent calls
        thetime = datetime.datetime.utcnow().replace(tzinfo=pytz.timezone("GMT"))
        self.templates[key] = {"tmplt": response.json(), "updated": thetime}
        return copy.deepcopy(response.json())

def _cache ( self , response , key ) : thetime = datetime . datetime . utcnow ( ) . replace ( tzinfo = pytz . timezone ( "GMT" ) ) self . templates [ key ] = { "tmplt" : response . json ( ) , "updated" : thetime } return copy . deepcopy ( response . json ( ) )

Add a retrieved template to the cache for 304 checking accepts a dict and key name adds the retrieval time and adds both to self . templates as a new dict using the specified key


def _cleanup(self, to_clean, allow=()):
        """ Remove keys we added for internal use
        """
        # this item's been retrieved from the API, we only need the 'data'
        # entry
        if to_clean.keys() == ["links", "library", "version", "meta", "key", "data"]:
            to_clean = to_clean["data"]
        return dict(
            [
                [k, v]
                for k, v in list(to_clean.items())
                if (k in allow or k not in self.temp_keys)
            ]
        )

def _cleanup ( self , to_clean , allow = ( ) ) : if to_clean . keys ( ) == [ "links" , "library" , "version" , "meta" , "key" , "data" ] : to_clean = to_clean [ "data" ] return dict ( [ [ k , v ] for k , v in list ( to_clean . items ( ) ) if ( k in allow or k not in self . temp_keys ) ] )


def _retrieve_data(self, request=None):
        """
        Retrieve Zotero items via the API
        Combine endpoint and request to access the specific resource
        Returns a JSON document
        """
        full_url = "%s%s" % (self.endpoint, request)
        # The API doesn't return this any more, so we have to cheat
        self.self_link = request
        self.request = requests.get(url=full_url, headers=self.default_headers())
        self.request.encoding = "utf-8"
        try:
            self.request.raise_for_status()
        except requests.exceptions.HTTPError:
            error_handler(self.request)
        return self.request

def _retrieve_data ( self , request = None ) : full_url = "%s%s" % ( self . endpoint , request ) self . self_link = request self . request = requests . get ( url = full_url , headers = self . default_headers ( ) ) self . request . encoding = "utf-8" try : self . request . raise_for_status ( ) except requests . exceptions . HTTPError : error_handler ( self . request ) return self . request


def _extract_links(self):
        """
        Extract self, first, next, last links from a request response
        """
        extracted = dict()
        try:
            for key, value in self.request.links.items():
                parsed = urlparse(value["url"])
                fragment = "{path}?{query}".format(path=parsed[2], query=parsed[4])
                extracted[key] = fragment
            # add a 'self' link
            parsed = list(urlparse(self.self_link))
            # strip 'format' query parameter
            stripped = "&".join(
                [
                    "%s=%s" % (p[0], p[1])
                    for p in parse_qsl(parsed[4])
                    if p[0] != "format"
                ]
            )
            # rebuild url fragment
            # this is a death march
            extracted["self"] = urlunparse(
                [parsed[0], parsed[1], parsed[2], parsed[3], stripped, parsed[5]]
            )
            return extracted
        except KeyError:
            # No links present, because it's a single item
            return None

def _extract_links ( self ) : extracted = dict ( ) try : for key , value in self . request . links . items ( ) : parsed = urlparse ( value [ "url" ] ) fragment = "{path}?{query}" . format ( path = parsed [ 2 ] , query = parsed [ 4 ] ) extracted [ key ] = fragment parsed = list ( urlparse ( self . self_link ) ) stripped = "&" . join ( [ "%s=%s" % ( p [ 0 ] , p [ 1 ] ) for p in parse_qsl ( parsed [ 4 ] ) if p [ 0 ] != "format" ] ) extracted [ "self" ] = urlunparse ( [ parsed [ 0 ] , parsed [ 1 ] , parsed [ 2 ] , parsed [ 3 ] , stripped , parsed [ 5 ] ] ) return extracted except KeyError : return None

Extract self first next last links from a request response


def _build_query(self, query_string, no_params=False):
        """
        Set request parameters. Will always add the user ID if it hasn't
        been specifically set by an API method
        """
        try:
            query = quote(query_string.format(u=self.library_id, t=self.library_type))
        except KeyError as err:
            raise ze.ParamNotPassed("There's a request parameter missing: %s" % err)
        # Add the URL parameters and the user key, if necessary
        if no_params is False:
            if not self.url_params:
                self.add_parameters()
            query = "%s?%s" % (query, self.url_params)
        return query

Set request parameters . Will always add the user ID if it hasn t been specifically set by an API method


def publications(self):
        """ Return the contents of My Publications
        """
        if self.library_type != "users":
            raise ze.CallDoesNotExist(
                "This API call does not exist for group libraries"
            )
        query_string = "/{t}/{u}/publications/items"
        return self._build_query(query_string)


def num_collectionitems(self, collection):
        """ Return the total number of items in the specified collection
        """
        query = "/{t}/{u}/collections/{c}/items".format(
            u=self.library_id, t=self.library_type, c=collection.upper()
        )
        return self._totals(query)

def num_collectionitems ( self , collection ) : query = "/{t}/{u}/collections/{c}/items" . format ( u = self . library_id , t = self . library_type , c = collection . upper ( ) ) return self . _totals ( query )


def num_tagitems(self, tag):
        """ Return the total number of items for the specified tag
        """
        query = "/{t}/{u}/tags/{ta}/items".format(
            u=self.library_id, t=self.library_type, ta=tag
        )
        return self._totals(query)

def num_tagitems ( self , tag ) : query = "/{t}/{u}/tags/{ta}/items" . format ( u = self . library_id , t = self . library_type , ta = tag ) return self . _totals ( query )


def _totals(self, query):
        """ General method for returning total counts
        """
        self.add_parameters(limit=1)
        query = self._build_query(query)
        self._retrieve_data(query)
        self.url_params = None
        # extract the 'total items' figure
        return int(self.request.headers["Total-Results"])

def _totals ( self , query ) : self . add_parameters ( limit = 1 ) query = self . _build_query ( query ) self . _retrieve_data ( query ) self . url_params = None return int ( self . request . headers [ "Total-Results" ] )


def key_info(self, **kwargs):
        """
        Retrieve info about the permissions associated with the
        key associated to the given Zotero instance
        """
        query_string = "/keys/{k}".format(k=self.api_key)
        return self._build_query(query_string)

def key_info ( self , ** kwargs ) : query_string = "/keys/{k}" . format ( k = self . api_key ) return self . _build_query ( query_string )


def fulltext_item(self, itemkey, **kwargs):
        """ Get full-text content for an item"""
        query_string = "/{t}/{u}/items/{itemkey}/fulltext".format(
            t=self.library_type, u=self.library_id, itemkey=itemkey
        )
        return self._build_query(query_string)

def fulltext_item ( self , itemkey , ** kwargs ) : query_string = "/{t}/{u}/items/{itemkey}/fulltext" . format ( t = self . library_type , u = self . library_id , itemkey = itemkey ) return self . _build_query ( query_string )


def last_modified_version(self, **kwargs):
        """ Get the last modified version
        """
        self.items(**kwargs)
        return int(self.request.headers.get("last-modified-version", 0))

def last_modified_version ( self , ** kwargs ) : self . items ( ** kwargs ) return int ( self . request . headers . get ( "last-modified-version" , 0 ) )


def file(self, item, **kwargs):
        """ Get the file from an specific item
        """
        query_string = "/{t}/{u}/items/{i}/file".format(
            u=self.library_id, t=self.library_type, i=item.upper()
        )
        return self._build_query(query_string, no_params=True)

def file ( self , item , ** kwargs ) : query_string = "/{t}/{u}/items/{i}/file" . format ( u = self . library_id , t = self . library_type , i = item . upper ( ) ) return self . _build_query ( query_string , no_params = True )


def dump(self, itemkey, filename=None, path=None):
        """
        Dump a file attachment to disk, with optional filename and path
        """
        if not filename:
            filename = self.item(itemkey)["data"]["filename"]
        if path:
            pth = os.path.join(path, filename)
        else:
            pth = filename
        file = self.file(itemkey)
        if self.snapshot:
            self.snapshot = False
            pth = pth + ".zip"
        with open(pth, "wb") as f:
            f.write(file)

def dump ( self , itemkey , filename = None , path = None ) : if not filename : filename = self . item ( itemkey ) [ "data" ] [ "filename" ] if path : pth = os . path . join ( path , filename ) else : pth = filename file = self . file ( itemkey ) if self . snapshot : self . snapshot = False pth = pth + ".zip" with open ( pth , "wb" ) as f : f . write ( file )

Dump a file attachment to disk with optional filename and path