def read_state_file(self, state_file):
        """Read the state file, if it exists.

        If this script has been run previously, resource IDs will have been written to a
        state file. On starting a run, a state file will be looked for before creating new
        infrastructure. Information on VPCs, security groups, and subnets are saved, as
        well as running instances and their states.

        AWS has a maximum number of VPCs per region per account, so we do not want to
        clutter users' AWS accounts with security groups and VPCs that will be used only
        once.
        """
        try:
            fh = open(state_file, 'r')
            state = json.load(fh)
            self.vpc_id = state['vpcID']
            self.sg_id = state['sgID']
            self.sn_ids = state['snIDs']
            self.instances = state['instances']
        except Exception as e:
            logger.debug("Caught exception while reading state file: {0}".format(e))
            raise e
        logger.debug("Done reading state from the local state file.")


def write_state_file(self):
        """Save information that must persist to a file.

        We do not want to create a new VPC and new identical security groups, so we save
        information about them in a file between runs.
        """
        fh = open('awsproviderstate.json', 'w')
        state = {}
        state['vpcID'] = self.vpc_id
        state['sgID'] = self.sg_id
        state['snIDs'] = self.sn_ids
        state['instances'] = self.instances
        state["instanceState"] = self.instance_states
        fh.write(json.dumps(state, indent=4))

def write_state_file ( self ) : fh = open ( 'awsproviderstate.json' , 'w' ) state = { } state [ 'vpcID' ] = self . vpc_id state [ 'sgID' ] = self . sg_id state [ 'snIDs' ] = self . sn_ids state [ 'instances' ] = self . instances state [ "instanceState" ] = self . instance_states fh . write ( json . dumps ( state , indent = 4 ) )

Save information that must persist to a file .


def create_session(self):
        """Create a session.

        First we look in self.key_file for a path to a json file with the
        credentials. The key file should have 'AWSAccessKeyId' and 'AWSSecretKey'.

        Next we look at self.profile for a profile name and try
        to use the Session call to automatically pick up the keys for the profile from
        the user default keys file ~/.aws/config.

        Finally, boto3 will look for the keys in environment variables:
        AWS_ACCESS_KEY_ID: The access key for your AWS account.
        AWS_SECRET_ACCESS_KEY: The secret key for your AWS account.
        AWS_SESSION_TOKEN: The session key for your AWS account.
        This is only needed when you are using temporary credentials.
        The AWS_SECURITY_TOKEN environment variable can also be used,
        but is only supported for backwards compatibility purposes.
        AWS_SESSION_TOKEN is supported by multiple AWS SDKs besides python.
        """

        session = None

        if self.key_file is not None:
            credfile = os.path.expandvars(os.path.expanduser(self.key_file))

            try:
                with open(credfile, 'r') as f:
                    creds = json.load(f)
            except json.JSONDecodeError as e:
                logger.error(
                    "EC2Provider '{}': json decode error in credential file {}".format(self.label, credfile)
                )
                raise e

            except Exception as e:
                logger.debug(
                    "EC2Provider '{0}' caught exception while reading credential file: {1}".format(
                        self.label, credfile
                    )
                )
                raise e

            logger.debug("EC2Provider '{}': Using credential file to create session".format(self.label))
            session = boto3.session.Session(region_name=self.region, **creds)
        elif self.profile is not None:
            logger.debug("EC2Provider '{}': Using profile name to create session".format(self.label))
            session = boto3.session.Session(
                profile_name=self.profile, region_name=self.region
            )
        else:
            logger.debug("EC2Provider '{}': Using environment variables to create session".format(self.label))
            session = boto3.session.Session(region_name=self.region)

        return session


def create_vpc(self):
        """Create and configure VPC

        We create a VPC with CIDR 10.0.0.0/16, which provides up to 64,000 instances.

        We attach a subnet for each availability zone within the region specified in the
        config. We give each subnet an ip range like 10.0.X.0/20, which is large enough
        for approx. 4000 instances.

        Security groups are configured in function security_group.
        """

        try:
            # We use a large VPC so that the cluster can get large
            vpc = self.ec2.create_vpc(
                CidrBlock='10.0.0.0/16',
                AmazonProvidedIpv6CidrBlock=False,
            )
        except Exception as e:
            # This failure will cause a full abort
            logger.error("{}\n".format(e))
            raise e

        # Attach internet gateway so that our cluster can
        # talk to the outside internet
        internet_gateway = self.ec2.create_internet_gateway()
        internet_gateway.attach_to_vpc(VpcId=vpc.vpc_id)  # Returns None
        self.internet_gateway = internet_gateway.id

        # Create and configure route table to allow proper traffic
        route_table = self.config_route_table(vpc, internet_gateway)
        self.route_table = route_table.id

        # Get all avaliability zones
        availability_zones = self.client.describe_availability_zones()

        # go through AZs and set up a subnet per
        for num, zone in enumerate(availability_zones['AvailabilityZones']):
            if zone['State'] == "available":

                # Create a large subnet (4000 max nodes)
                subnet = vpc.create_subnet(
                    CidrBlock='10.0.{}.0/20'.format(16 * num), AvailabilityZone=zone['ZoneName']
                )

                # Make subnet accessible
                subnet.meta.client.modify_subnet_attribute(
                    SubnetId=subnet.id, MapPublicIpOnLaunch={"Value": True}
                )

                route_table.associate_with_subnet(SubnetId=subnet.id)
                self.sn_ids.append(subnet.id)
            else:
                logger.info("{} unavailable".format(zone['ZoneName']))
        # Security groups
        self.security_group(vpc)
        self.vpc_id = vpc.id
        return vpc

def
create_vpc
(
self
)
:
try
:
vpc
=
self
.
ec2
.
create_vpc
(
CidrBlock
=
'10.0.0.0/16'
,
AmazonProvidedIpv6CidrBlock
=
False
,
)
except
Exception
as
e
:
logger
.
error
(
"{}\n"
.
format
(
e
)
)
raise
e
internet_gateway
=
self
.
ec2
.
create_internet_gateway
(
)
internet_gateway
.
attach_to_vpc
(
VpcId
=
vpc
.
vpc_id
)
self
.
internet_gateway
=
internet_gateway
.
id
route_table
=
self
.
config_route_table
(
vpc
,
internet_gateway
)
self
.
route_table
=
route_table
.
id
availability_zones
=
self
.
client
.
describe_availability_zones
(
)
for
num
,
zone
in
enumerate
(
availability_zones
[
'AvailabilityZones'
]
)
:
if
zone
[
'State'
]
==
"available"
:
subnet
=
vpc
.
create_subnet
(
CidrBlock
=
'10.0.{}.0/20'
.
format
(
16
*
num
)
,
AvailabilityZone
=
zone
[
'ZoneName'
]
)
subnet
.
meta
.
client
.
modify_subnet_attribute
(
SubnetId
=
subnet
.
id
,
MapPublicIpOnLaunch
=
{
"Value"
:
True
}
)
route_table
.
associate_with_subnet
(
SubnetId
=
subnet
.
id
)
self
.
sn_ids
.
append
(
subnet
.
id
)
else
:
logger
.
info
(
"{} unavailable"
.
format
(
zone
[
'ZoneName'
]
)
)
self
.
security_group
(
vpc
)
self
.
vpc_id
=
vpc
.
id
return
vpc


def security_group(self, vpc):
        """Create and configure a new security group.

        Allows all ICMP in, all TCP and UDP in within VPC.

        This security group is very open. It allows all incoming ping requests on all
        ports. It also allows all outgoing traffic on all ports. This can be limited by
        changing the allowed port ranges.

        Parameters
        ----------
        vpc : VPC instance
            VPC in which to set up security group.
        """

        sg = vpc.create_security_group(
            GroupName="private-subnet", Description="security group for remote executors"
        )

        ip_ranges = [{'CidrIp': '10.0.0.0/16'}]

        # Allows all ICMP in, all TCP and UDP in within VPC
        in_permissions = [
            {
                'IpProtocol': 'TCP',
                'FromPort': 0,
                'ToPort': 65535,
                'IpRanges': ip_ranges,
            }, {
                'IpProtocol': 'UDP',
                'FromPort': 0,
                'ToPort': 65535,
                'IpRanges': ip_ranges,
            }, {
                'IpProtocol': 'ICMP',
                'FromPort': -1,
                'ToPort': -1,
                'IpRanges': [{
                    'CidrIp': '0.0.0.0/0'
                }],
            }, {
                'IpProtocol': 'TCP',
                'FromPort': 22,
                'ToPort': 22,
                'IpRanges': [{
                    'CidrIp': '0.0.0.0/0'
                }],
            }
        ]

        # Allows all TCP out, all TCP and UDP out within VPC
        out_permissions = [
            {
                'IpProtocol': 'TCP',
                'FromPort': 0,
                'ToPort': 65535,
                'IpRanges': [{
                    'CidrIp': '0.0.0.0/0'
                }],
            },
            {
                'IpProtocol': 'TCP',
                'FromPort': 0,
                'ToPort': 65535,
                'IpRanges': ip_ranges,
            },
            {
                'IpProtocol': 'UDP',
                'FromPort': 0,
                'ToPort': 65535,
                'IpRanges': ip_ranges,
            },
        ]

        sg.authorize_ingress(IpPermissions=in_permissions)
        sg.authorize_egress(IpPermissions=out_permissions)
        self.sg_id = sg.id
        return sg

def
security_group
(
self
,
vpc
)
:
sg
=
vpc
.
create_security_group
(
GroupName
=
"private-subnet"
,
Description
=
"security group for remote executors"
)
ip_ranges
=
[
{
'CidrIp'
:
'10.0.0.0/16'
}
]
in_permissions
=
[
{
'IpProtocol'
:
'TCP'
,
'FromPort'
:
0
,
'ToPort'
:
65535
,
'IpRanges'
:
ip_ranges
,
}
,
{
'IpProtocol'
:
'UDP'
,
'FromPort'
:
0
,
'ToPort'
:
65535
,
'IpRanges'
:
ip_ranges
,
}
,
{
'IpProtocol'
:
'ICMP'
,
'FromPort'
:
1
,
'ToPort'
:
1
,
'IpRanges'
:
[
{
'CidrIp'
:
'0.0.0.0/0'
}
]
,
}
,
{
'IpProtocol'
:
'TCP'
,
'FromPort'
:
22
,
'ToPort'
:
22
,
'IpRanges'
:
[
{
'CidrIp'
:
'0.0.0.0/0'
}
]
,
}
]
out_permissions
=
[
{
'IpProtocol'
:
'TCP'
,
'FromPort'
:
0
,
'ToPort'
:
65535
,
'IpRanges'
:
[
{
'CidrIp'
:
'0.0.0.0/0'
}
]
,
}
,
{
'IpProtocol'
:
'TCP'
,
'FromPort'
:
0
,
'ToPort'
:
65535
,
'IpRanges'
:
ip_ranges
,
}
,
{
'IpProtocol'
:
'UDP'
,
'FromPort'
:
0
,
'ToPort'
:
65535
,
'IpRanges'
:
ip_ranges
,
}
,
]
sg
.
authorize_ingress
(
IpPermissions
=
in_permissions
)
sg
.
authorize_egress
(
IpPermissions
=
out_permissions
)
self
.
sg_id
=
sg
.
id
return
sg

Create and configure a new security group .


def spin_up_instance(self, command, job_name):
        """Start an instance in the VPC in the first available subnet.

        N instances will be started if nodes_per_block > 1.
        Not supported. We only do 1 node per block.

        Parameters
        ----------
        command : str
            Command string to execute on the node.
        job_name : str
            Name associated with the instances.
        """

        command = Template(template_string).substitute(jobname=job_name,
                                                       user_script=command,
                                                       linger=str(self.linger).lower(),
                                                       worker_init=self.worker_init)
        instance_type = self.instance_type
        subnet = self.sn_ids[0]
        ami_id = self.image_id
        total_instances = len(self.instances)

        if float(self.spot_max_bid) > 0:
            spot_options = {
                'MarketType': 'spot',
                'SpotOptions': {
                    'MaxPrice': str(self.spot_max_bid),
                    'SpotInstanceType': 'one-time',
                    'InstanceInterruptionBehavior': 'terminate'
                }
            }
        else:
            spot_options = {}

        if total_instances > self.max_nodes:
            logger.warn("Exceeded instance limit ({}). Cannot continue\n".format(self.max_nodes))
            return [None]
        try:
            tag_spec = [{"ResourceType": "instance", "Tags": [{'Key': 'Name', 'Value': job_name}]}]

            instance = self.ec2.create_instances(
                MinCount=1,
                MaxCount=1,
                InstanceType=instance_type,
                ImageId=ami_id,
                KeyName=self.key_name,
                SubnetId=subnet,
                SecurityGroupIds=[self.sg_id],
                TagSpecifications=tag_spec,
                InstanceMarketOptions=spot_options,
                InstanceInitiatedShutdownBehavior='terminate',
                IamInstanceProfile={'Arn': self.iam_instance_profile_arn},
                UserData=command
            )
        except ClientError as e:
            print(e)
            logger.error(e.response)
            return [None]

        except Exception as e:
            logger.error("Request for EC2 resources failed : {0}".format(e))
            return [None]

        self.instances.append(instance[0].id)
        logger.info(
            "Started up 1 instance {} . Instance type:{}".format(instance[0].id, instance_type)
        )
        return instance

Start an instance in the VPC in the first available subnet .


def shut_down_instance(self, instances=None):
        """Shut down a list of instances, if provided.

        If no instance is provided, the last instance started up will be shut down.
        """

        if instances and len(self.instances) > 0:
            print(instances)
            try:
                print([i.id for i in instances])
            except Exception as e:
                print(e)
            term = self.client.terminate_instances(InstanceIds=instances)
            logger.info("Shut down {} instances (ids:{}".format(len(instances), str(instances)))
        elif len(self.instances) > 0:
            instance = self.instances.pop()
            term = self.client.terminate_instances(InstanceIds=[instance])
            logger.info("Shut down 1 instance (id:{})".format(instance))
        else:
            logger.warn("No Instances to shut down.\n")
            return -1
        self.get_instance_state()
        return term

def
shut_down_instance
(
self
,
instances
=
)
:
if
instances
and
len
(
self
.
instances
)
>
0
:
print
(
instances
)
try
:
print
(
[
i
.
id
for
i
in
instances
]
)
except
Exception
as
e
:
print
(
e
)
term
=
self
.
client
.
terminate_instances
(
InstanceIds
=
instances
)
logger
.
info
(
"Shut down {} instances (ids:{}"
.
format
(
len
(
instances
)
,
str
(
instances
)
)
)
elif
len
(
self
.
instances
)
>
0
:
instance
=
self
.
instances
.
pop
(
)
term
=
self
.
client
.
terminate_instances
(
InstanceIds
=
[
instance
]
)
logger
.
info
(
"Shut down 1 instance (id:{})"
.
format
(
instance
)
)
else
:
logger
.
warn
(
"No Instances to shut down.\n"
)
return
1
self
.
get_instance_state
(
)
return
term

Shut down a list of instances if provided .


def get_instance_state(self, instances=None):
        """Get states of all instances on EC2 which were started by this file."""
        if instances:
            desc = self.client.describe_instances(InstanceIds=instances)
        else:
            desc = self.client.describe_instances(InstanceIds=self.instances)
        # pprint.pprint(desc['Reservations'],indent=4)
        for i in range(len(desc['Reservations'])):
            instance = desc['Reservations'][i]['Instances'][0]
            self.instance_states[instance['InstanceId']] = instance['State']['Name']
        return self.instance_states

def get_instance_state ( self , instances = None ) : if instances : desc = self . client . describe_instances ( InstanceIds = instances ) else : desc = self . client . describe_instances ( InstanceIds = self . instances ) for i in range ( len ( desc [ 'Reservations' ] ) ) : instance = desc [ 'Reservations' ] [ i ] [ 'Instances' ] [ 0 ] self . instance_states [ instance [ 'InstanceId' ] ] = instance [ 'State' ] [ 'Name' ] return self . instance_states

Get states of all instances on EC2 which were started by this file .


def submit(self, command='sleep 1', blocksize=1, tasks_per_node=1, job_name="parsl.auto"):
        """Submit the command onto a freshly instantiated AWS EC2 instance.

        Submit returns an ID that corresponds to the task that was just submitted.

        Parameters
        ----------
        command : str
            Command to be invoked on the remote side.
        blocksize : int
            Number of blocks requested.
        tasks_per_node : int (default=1)
            Number of command invocations to be launched per node
        job_name : str
            Prefix for the job name.

        Returns
        -------
        None or str
            If at capacity, None will be returned. Otherwise, the job identifier will be returned.
        """

        job_name = "parsl.auto.{0}".format(time.time())
        wrapped_cmd = self.launcher(command,
                                    tasks_per_node,
                                    self.nodes_per_block)
        [instance, *rest] = self.spin_up_instance(command=wrapped_cmd, job_name=job_name)

        if not instance:
            logger.error("Failed to submit request to EC2")
            return None

        logger.debug("Started instance_id: {0}".format(instance.instance_id))

        state = translate_table.get(instance.state['Name'], "PENDING")

        self.resources[instance.instance_id] = {
            "job_id": instance.instance_id,
            "instance": instance,
            "status": state
        }

        return instance.instance_id

def
submit
(
self
,
command
=
'sleep 1'
,
blocksize
=
1
,
tasks_per_node
=
1
,
job_name
=
"parsl.auto"
)
:
job_name
=
"parsl.auto.{0}"
.
format
(
time
.
time
(
)
)
wrapped_cmd
=
self
.
launcher
(
command
,
tasks_per_node
,
self
.
nodes_per_block
)
[
instance
,
*
rest
]
=
self
.
spin_up_instance
(
command
=
wrapped_cmd
,
job_name
=
job_name
)
if
not
instance
:
logger
.
error
(
"Failed to submit request to EC2"
)
return
logger
.
debug
(
"Started instance_id: {0}"
.
format
(
instance
.
instance_id
)
)
state
=
translate_table
.
get
(
instance
.
state
[
'Name'
]
,
"PENDING"
)
self
.
resources
[
instance
.
instance_id
]
=
{
"job_id"
:
instance
.
instance_id
,
"instance"
:
instance
,
"status"
:
state
}
return
instance
.
instance_id

Submit the command onto a freshly instantiated AWS EC2 instance .


def cancel(self, job_ids):
        """Cancel the jobs specified by a list of job ids.

        Parameters
        ----------
        job_ids : list of str
            List of of job identifiers

        Returns
        -------
        list of bool
            Each entry in the list will contain False if the operation fails. Otherwise, the entry will be True.
        """

        if self.linger is True:
            logger.debug("Ignoring cancel requests due to linger mode")
            return [False for x in job_ids]

        try:
            self.client.terminate_instances(InstanceIds=list(job_ids))
        except Exception as e:
            logger.error("Caught error while attempting to remove instances: {0}".format(job_ids))
            raise e
        else:
            logger.debug("Removed the instances: {0}".format(job_ids))

        for job_id in job_ids:
            self.resources[job_id]["status"] = "COMPLETED"

        for job_id in job_ids:
            self.instances.remove(job_id)

        return [True for x in job_ids]

Cancel the jobs specified by a list of job ids .


def show_summary(self):
        """Print human readable summary of current AWS state to log and to console."""
        self.get_instance_state()
        status_string = "EC2 Summary:\n\tVPC IDs: {}\n\tSubnet IDs: \
{}\n\tSecurity Group ID: {}\n\tRunning Instance IDs: {}\n".format(
            self.vpc_id, self.sn_ids, self.sg_id, self.instances
        )
        status_string += "\tInstance States:\n\t\t"
        self.get_instance_state()
        for state in self.instance_states.keys():
            status_string += "Instance ID: {}  State: {}\n\t\t".format(
                state, self.instance_states[state]
            )
        status_string += "\n"
        logger.info(status_string)
        return status_string

"EC2 Summary:\n\tVPC IDs: {}\n\tSubnet IDs: \{}\n\tSecurity Group ID: {}\n\tRunning Instance IDs: {}\n"

Print human readable summary of current AWS state to log and to console .


def teardown(self):
        """Teardown the EC2 infastructure.

        Terminate all EC2 instances, delete all subnets, delete security group, delete VPC,
        and reset all instance variables.
        """

        self.shut_down_instance(self.instances)
        self.instances = []
        try:
            self.client.delete_internet_gateway(InternetGatewayId=self.internet_gateway)
            self.internet_gateway = None
            self.client.delete_route_table(RouteTableId=self.route_table)
            self.route_table = None
            for subnet in list(self.sn_ids):
                # Cast to list ensures that this is a copy
                # Which is important because it means that
                # the length of the list won't change during iteration
                self.client.delete_subnet(SubnetId=subnet)
                self.sn_ids.remove(subnet)
            self.client.delete_security_group(GroupId=self.sg_id)
            self.sg_id = None
            self.client.delete_vpc(VpcId=self.vpc_id)
            self.vpc_id = None
        except Exception as e:
            logger.error("{}".format(e))
            raise e
        self.show_summary()
        os.remove(self.config['state_file_path'])

def teardown ( self ) : self . shut_down_instance ( self . instances ) self . instances = [ ] try : self . client . delete_internet_gateway ( InternetGatewayId = self . internet_gateway ) self . internet_gateway = None self . client . delete_route_table ( RouteTableId = self . route_table ) self . route_table = None for subnet in list ( self . sn_ids ) : self . client . delete_subnet ( SubnetId = subnet ) self . sn_ids . remove ( subnet ) self . client . delete_security_group ( GroupId = self . sg_id ) self . sg_id = None self . client . delete_vpc ( VpcId = self . vpc_id ) self . vpc_id = None except Exception as e : logger . error ( "{}" . format ( e ) ) raise e self . show_summary ( ) os . remove ( self . config [ 'state_file_path' ] )


def scale_out(self, blocks=1, block_size=1):
        ''' Scale out the existing resources.
        '''
        self.config['sites.jetstream.{0}'.format(self.pool)]['flavor']
        count = 0
        if blocks == 1:
            block_id = len(self.blocks)
            self.blocks[block_id] = []
            for instance_id in range(0, block_size):
                instances = self.server_manager.create(
                    'parsl-{0}-{1}'.format(block_id, instance_id),  # Name
                    self.client.images.get('87e08a17-eae2-4ce4-9051-c561d9a54bde'),  # Image_id
                    self.client.flavors.list()[0],
                    min_count=1,
                    max_count=1,
                    userdata=setup_script.format(engine_config=self.engine_config),
                    key_name='TG-MCB090174-api-key',
                    security_groups=['global-ssh'],
                    nics=[{
                        "net-id": '724a50cf-7f11-4b3b-a884-cd7e6850e39e',
                        "net-name": 'PARSL-priv-net',
                        "v4-fixed-ip": ''
                    }])
                self.blocks[block_id].extend([instances])
                count += 1

        return count

def scale_out ( self , blocks = 1 , block_size = 1 ) : self . config [ 'sites.jetstream.{0}' . format ( self . pool ) ] [ 'flavor' ] count = 0 if blocks == 1 : block_id = len ( self . blocks ) self . blocks [ block_id ] = [ ] for instance_id in range ( 0 , block_size ) : instances = self . server_manager . create ( 'parsl-{0}-{1}' . format ( block_id , instance_id ) , self . client . images . get ( '87e08a17-eae2-4ce4-9051-c561d9a54bde' ) , self . client . flavors . list ( ) [ 0 ] , min_count = 1 , max_count = 1 , userdata = setup_script . format ( engine_config = self . engine_config ) , key_name = 'TG-MCB090174-api-key' , security_groups = [ 'global-ssh' ] , nics = [ { "net-id" : '724a50cf-7f11-4b3b-a884-cd7e6850e39e' , "net-name" : 'PARSL-priv-net' , "v4-fixed-ip" : '' } ] ) self . blocks [ block_id ] . extend ( [ instances ] ) count += 1 return count


def scale_in(self, blocks=0, machines=0, strategy=None):
        ''' Scale in resources
        '''
        count = 0
        instances = self.client.servers.list()
        for instance in instances[0:machines]:
            print("Deleting : ", instance)
            instance.delete()
            count += 1

        return count

def
scale_in
(
self
,
blocks
=
0
,
machines
=
0
,
strategy
=
)
:
count
=
0
instances
=
self
.
client
.
servers
.
list
(
)
for
instance
in
instances
[
0
:
machines
]
:
print
(
"Deleting : "
,
instance
)
instance
.
delete
(
)
count
+=
1
return
count


def _status(self):
        """Update the resource dictionary with job statuses."""

        job_id_list = ' '.join(self.resources.keys())
        cmd = "condor_q {0} -af:jr JobStatus".format(job_id_list)
        retcode, stdout, stderr = super().execute_wait(cmd)
        """
        Example output:

        $ condor_q 34524642.0 34524643.0 -af:jr JobStatus
        34524642.0 2
        34524643.0 1
        """

        for line in stdout.strip().split('\n'):
            parts = line.split()
            job_id = parts[0]
            status = translate_table.get(parts[1], 'UNKNOWN')
            self.resources[job_id]['status'] = status

def
_status
(
self
)
:
job_id_list
=
' '
.
join
(
self
.
resources
.
keys
(
)
)
cmd
=
"condor_q {0} -af:jr JobStatus"
.
format
(
job_id_list
)
retcode
,
stdout
,
stderr
=
super
(
)
.
execute_wait
(
cmd
)
for
line
in
stdout
.
strip
(
)
.
split
(
'\n'
)
:
parts
=
line
.
split
(
)
job_id
=
parts
[
0
]
status
=
translate_table
.
get
(
parts
[
1
]
,
'UNKNOWN'
)
self
.
resources
[
job_id
]
[
'status'
]
=
status

Update the resource dictionary with job statuses .


def scale_out(self, blocks=1):
        """Scales out the number of active workers by 1.

        This method is notImplemented for threads and will raise the error if called.

        Parameters:
            blocks : int
               Number of blocks to be provisioned.
        """
        r = []
        for i in range(blocks):
            if self.provider:
                block = self.provider.submit(self.launch_cmd, 1, self.workers_per_node)
                logger.debug("Launched block {}:{}".format(i, block))
                if not block:
                    raise(ScalingFailed(self.provider.label,
                                        "Attempts to provision nodes via provider has failed"))
                self.engines.extend([block])
                r.extend([block])
        else:
            logger.error("No execution provider available")
            r = None

        return r

Scales out the number of active workers by 1 .


def scale_in(self, blocks):
        """Scale in the number of active blocks by the specified number.

        """
        status = dict(zip(self.engines, self.provider.status(self.engines)))

        # This works for blocks=0
        to_kill = [engine for engine in status if status[engine] == "RUNNING"][:blocks]

        if self.provider:
            r = self.provider.cancel(to_kill)
        else:
            logger.error("No execution provider available")
            r = None

        return r

def
scale_in
(
self
,
blocks
)
:
status
=
dict
(
zip
(
self
.
engines
,
self
.
provider
.
status
(
self
.
engines
)
)
)
to_kill
=
[
engine
for
engine
in
status
if
status
[
engine
]
==
"RUNNING"
]
[
:
blocks
]
if
self
.
provider
:
r
=
self
.
provider
.
cancel
(
to_kill
)
else
:
logger
.
error
(
"No execution provider available"
)
r
=
return
r

Scale in the number of active blocks by the specified number .


def status(self):
        """Returns the status of the executor via probing the execution providers."""
        if self.provider:
            status = self.provider.status(self.engines)

        else:
            status = []

        return status

def status ( self ) : if self . provider : status = self . provider . status ( self . engines ) else : status = [ ] return status

Returns the status of the executor via probing the execution providers .


def parent_callback(self, executor_fu):
        """Callback from a parent future to update the AppFuture.

        Used internally by AppFuture, and should not be called by code using AppFuture.

        Args:
            - executor_fu (Future): Future returned by the executor along with callback.
              This may not be the current parent future, as the parent future may have
              already been updated to point to a retrying execution, and in that case,
              this is logged.

              In the case that a new parent has been attached, we must immediately discard
              this result no matter what it contains (although it might be interesting
              to log if it was successful...)

        Returns:
            - None

        Updates the super() with the result() or exception()
        """
        with self._update_lock:

            if not executor_fu.done():
                raise ValueError("done callback called, despite future not reporting itself as done")

            # this is for consistency checking
            if executor_fu != self.parent:
                if executor_fu.exception() is None and not isinstance(executor_fu.result(), RemoteExceptionWrapper):
                    # ... then we completed with a value, not an exception or wrapped exception,
                    # but we've got an updated executor future.
                    # This is bad - for example, we've started a retry even though we have a result

                    raise ValueError("internal consistency error: AppFuture done callback called without an exception, but parent has been changed since then")

            try:
                res = executor_fu.result()
                if isinstance(res, RemoteExceptionWrapper):
                    res.reraise()
                super().set_result(executor_fu.result())

            except Exception as e:
                if executor_fu.retries_left > 0:
                    # ignore this exception, because assume some later
                    # parent executor, started external to this class,
                    # will provide the answer
                    pass
                else:
                    super().set_exception(e)

Callback from a parent future to update the AppFuture .


def update_parent(self, fut):
        """Add a callback to the parent to update the state.

        This handles the case where the user has called result on the AppFuture
        before the parent exists.
        """
        self.parent = fut

        try:
            fut.add_done_callback(self.parent_callback)
        except Exception as e:
            logger.error("add_done_callback got an exception {} which will be ignored".format(e))

Add a callback to the parent to update the state .


def parent_callback(self, parent_fu):
        """Callback from executor future to update the parent.

        Args:
            - parent_fu (Future): Future returned by the executor along with callback

        Returns:
            - None

        Updates the super() with the result() or exception()
        """
        if parent_fu.done() is True:
            e = parent_fu._exception
            if e:
                super().set_exception(e)
            else:
                super().set_result(self.file_obj)
        return

def parent_callback ( self , parent_fu ) : if parent_fu . done ( ) is True : e = parent_fu . _exception if e : super ( ) . set_exception ( e ) else : super ( ) . set_result ( self . file_obj ) return

Callback from executor future to update the parent .


def submit(self, command, blocksize, tasks_per_node, job_name="parsl.auto"):
        ''' The submit method takes the command string to be executed upon
        instantiation of a resource most often to start a pilot.

        Args :
             - command (str) : The bash command string to be executed.
             - blocksize (int) : Blocksize to be requested
             - tasks_per_node (int) : command invocations to be launched per node

        KWargs:
             - job_name (str) : Human friendly name to be assigned to the job request

        Returns:
             - A job identifier, this could be an integer, string etc

        Raises:
             - ExecutionProviderException or its subclasses
        '''
        wrapped_cmd = self.launcher(command,
                                    tasks_per_node,
                                    1)

        instance, name = self.create_instance(command=wrapped_cmd)
        self.provisioned_blocks += 1
        self.resources[name] = {"job_id": name, "status": translate_table[instance['status']]}
        return name

def submit ( self , command , blocksize , tasks_per_node , job_name = "parsl.auto" ) : wrapped_cmd = self . launcher ( command , tasks_per_node , 1 ) instance , name = self . create_instance ( command = wrapped_cmd ) self . provisioned_blocks += 1 self . resources [ name ] = { "job_id" : name , "status" : translate_table [ instance [ 'status' ] ] } return name

The submit method takes the command string to be executed upon instantiation of a resource most often to start a pilot .


def cancel(self, job_ids):
        ''' Cancels the resources identified by the job_ids provided by the user.

        Args:
             - job_ids (list): A list of job identifiers

        Returns:
             - A list of status from cancelling the job which can be True, False

        Raises:
             - ExecutionProviderException or its subclasses
        '''
        statuses = []
        for job_id in job_ids:
            try:
                self.delete_instance(job_id)
                statuses.append(True)
                self.provisioned_blocks -= 1
            except Exception:
                statuses.append(False)
        return statuses

def cancel ( self , job_ids ) : statuses = [ ] for job_id in job_ids : try : self . delete_instance ( job_id ) statuses . append ( True ) self . provisioned_blocks -= 1 except Exception : statuses . append ( False ) return statuses

Cancels the resources identified by the job_ids provided by the user .


def runner(incoming_q, outgoing_q):
    """This is a function that mocks the Swift-T side.

    It listens on the the incoming_q for tasks and posts returns on the outgoing_q.

    Args:
         - incoming_q (Queue object) : The queue to listen on
         - outgoing_q (Queue object) : Queue to post results on

    The messages posted on the incoming_q will be of the form :

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "buffer"  : serialized buffer containing the fn, args and kwargs
       }

    If ``None`` is received, the runner will exit.

    Response messages should be of the form:

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "result"  : serialized buffer containing result
          "exception" : serialized exception object
       }

    On exiting the runner will post ``None`` to the outgoing_q

    """
    logger.debug("[RUNNER] Starting")

    def execute_task(bufs):
        """Deserialize the buffer and execute the task.

        Returns the serialized result or exception.
        """
        user_ns = locals()
        user_ns.update({'__builtins__': __builtins__})

        f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)

        fname = getattr(f, '__name__', 'f')
        prefix = "parsl_"
        fname = prefix + "f"
        argname = prefix + "args"
        kwargname = prefix + "kwargs"
        resultname = prefix + "result"

        user_ns.update({fname: f,
                        argname: args,
                        kwargname: kwargs,
                        resultname: resultname})

        code = "{0} = {1}(*{2}, **{3})".format(resultname, fname,
                                               argname, kwargname)

        try:
            logger.debug("[RUNNER] Executing: {0}".format(code))
            exec(code, user_ns, user_ns)

        except Exception as e:
            logger.warning("Caught exception; will raise it: {}".format(e))
            raise e

        else:
            logger.debug("[RUNNER] Result: {0}".format(user_ns.get(resultname)))
            return user_ns.get(resultname)

    while True:
        try:
            # Blocking wait on the queue
            msg = incoming_q.get(block=True, timeout=10)

        except queue.Empty:
            # Handle case where no items were in the queue
            logger.debug("[RUNNER] Queue is empty")

        except IOError as e:
            logger.debug("[RUNNER] Broken pipe: {}".format(e))
            try:
                # Attempt to send a stop notification to the management thread
                outgoing_q.put(None)

            except Exception:
                pass

            break

        except Exception as e:
            logger.debug("[RUNNER] Caught unknown exception: {}".format(e))

        else:
            # Handle received message
            if not msg:
                # Empty message is a die request
                logger.debug("[RUNNER] Received exit request")
                outgoing_q.put(None)
                break
            else:
                # Received a valid message, handle it
                logger.debug("[RUNNER] Got a valid task with ID {}".format(msg["task_id"]))
                try:
                    response_obj = execute_task(msg['buffer'])
                    response = {"task_id": msg["task_id"],
                                "result": serialize_object(response_obj)}

                    logger.debug("[RUNNER] Returing result: {}".format(
                                   deserialize_object(response["result"])))

                except Exception as e:
                    logger.debug("[RUNNER] Caught task exception: {}".format(e))
                    response = {"task_id": msg["task_id"],
                                "exception": serialize_object(e)}

                outgoing_q.put(response)

    logger.debug("[RUNNER] Terminating")

def
runner
(
incoming_q
,
outgoing_q
)
:
logger
.
debug
(
"[RUNNER] Starting"
)
def
execute_task
(
bufs
)
:
user_ns
=
locals
(
)
user_ns
.
update
(
{
'__builtins__'
:
__builtins__
}
)
f
,
args
,
kwargs
=
unpack_apply_message
(
bufs
,
user_ns
,
copy
=
False
)
fname
=
getattr
(
f
,
'__name__'
,
'f'
)
prefix
=
"parsl_"
fname
=
prefix
+
"f"
argname
=
prefix
+
"args"
kwargname
=
prefix
+
"kwargs"
resultname
=
prefix
+
"result"
user_ns
.
update
(
{
fname
:
f
,
argname
:
args
,
kwargname
:
kwargs
,
resultname
:
resultname
}
)
code
=
"{0} = {1}(*{2}, **{3})"
.
format
(
resultname
,
fname
,
argname
,
kwargname
)
try
:
logger
.
debug
(
"[RUNNER] Executing: {0}"
.
format
(
code
)
)
exec
(
code
,
user_ns
,
user_ns
)
except
Exception
as
e
:
logger
.
warning
(
"Caught exception; will raise it: {}"
.
format
(
e
)
)
raise
e
else
:
logger
.
debug
(
"[RUNNER] Result: {0}"
.
format
(
user_ns
.
get
(
resultname
)
)
)
return
user_ns
.
get
(
resultname
)
while
True
:
try
:
msg
=
incoming_q
.
get
(
block
=
True
,
timeout
=
10
)
except
queue
.
Empty
:
logger
.
debug
(
"[RUNNER] Queue is empty"
)
except
IOError
as
e
:
logger
.
debug
(
"[RUNNER] Broken pipe: {}"
.
format
(
e
)
)
try
:
outgoing_q
.
put
(
)
except
Exception
:
pass
break
except
Exception
as
e
:
logger
.
debug
(
"[RUNNER] Caught unknown exception: {}"
.
format
(
e
)
)
else
:
if
not
msg
:
logger
.
debug
(
"[RUNNER] Received exit request"
)
outgoing_q
.
put
(
)
break
else
:
logger
.
debug
(
"[RUNNER] Got a valid task with ID {}"
.
format
(
msg
[
"task_id"
]
)
)
try
:
response_obj
=
execute_task
(
msg
[
'buffer'
]
)
response
=
{
"task_id"
:
msg
[
"task_id"
]
,
"result"
:
serialize_object
(
response_obj
)
}
logger
.
debug
(
"[RUNNER] Returing result: {}"
.
format
(
deserialize_object
(
response
[
"result"
]
)
)
)
except
Exception
as
e
:
logger
.
debug
(
"[RUNNER] Caught task exception: {}"
.
format
(
e
)
)
response
=
{
"task_id"
:
msg
[
"task_id"
]
,
"exception"
:
serialize_object
(
e
)
}
outgoing_q
.
put
(
response
)
logger
.
debug
(
"[RUNNER] Terminating"
)

This is a function that mocks the Swift - T side .


def shutdown(self):
        """Shutdown method, to kill the threads and workers."""
        self.is_alive = False
        logging.debug("Waking management thread")
        self.incoming_q.put(None)  # Wake up the thread
        self._queue_management_thread.join()  # Force join
        logging.debug("Exiting thread")
        self.worker.join()
        return True

def
shutdown
(
self
)
:
self
.
is_alive
=
False
logging
.
debug
(
"Waking management thread"
)
self
.
incoming_q
.
put
(
)
self
.
_queue_management_thread
.
join
(
)
logging
.
debug
(
"Exiting thread"
)
self
.
worker
.
join
(
)
return
True

Shutdown method to kill the threads and workers .


def submit(self, func, *args, **kwargs):
        """Submits work to the the outgoing_q.

        The outgoing_q is an external process listens on this
        queue for new work. This method is simply pass through and behaves like a
        submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_

        Args:
            - func (callable) : Callable function
            - *args (list) : List of arbitrary positional arguments.

        Kwargs:
            - **kwargs (dict) : A dictionary of arbitrary keyword args for func.

        Returns:
              Future
        """
        task_id = uuid.uuid4()

        logger.debug("Pushing function {} to queue with args {}".format(func, args))

        self.tasks[task_id] = Future()

        fn_buf = pack_apply_message(func, args, kwargs,
                                    buffer_threshold=1024 * 1024,
                                    item_threshold=1024)

        msg = {"task_id": task_id,
               "buffer": fn_buf}

        # Post task to the the outgoing queue
        self.outgoing_q.put(msg)

        # Return the future
        return self.tasks[task_id]


def filepath(self):
        """Return the resolved filepath on the side where it is called from.

        The appropriate filepath will be returned when called from within
        an app running remotely as well as regular python on the client side.

        Args:
            - self
        Returns:
             - filepath (string)
        """
        if hasattr(self, 'local_path'):
            return self.local_path

        if self.scheme in ['ftp', 'http', 'https', 'globus']:
            return self.filename
        elif self.scheme in ['file']:
            return self.path
        else:
            raise Exception('Cannot return filepath for unknown scheme {}'.format(self.scheme))

Return the resolved filepath on the side where it is called from .


def push_file(self, source, dest_dir):
        ''' If the source files dirpath is the same as dest_dir, a copy
        is not necessary, and nothing is done. Else a copy is made.

        Args:
            - source (string) : Path to the source file
            - dest_dir (string) : Path to the directory to which the files is to be copied

        Returns:
            - destination_path (String) : Absolute path of the destination file

        Raises:
            - FileCopyException : If file copy failed.
        '''

        local_dest = dest_dir + '/' + os.path.basename(source)

        # Only attempt to copy if the target dir and source dir are different
        if os.path.dirname(source) != dest_dir:
            try:
                shutil.copyfile(source, local_dest)
                os.chmod(local_dest, 0o777)

            except OSError as e:
                raise FileCopyException(e, self.hostname)

        return local_dest

def push_file ( self , source , dest_dir ) : local_dest = dest_dir + '/' + os . path . basename ( source ) if os . path . dirname ( source ) != dest_dir : try : shutil . copyfile ( source , local_dest ) os . chmod ( local_dest , 0o777 ) except OSError as e : raise FileCopyException ( e , self . hostname ) return local_dest

If the source files dirpath is the same as dest_dir a copy is not necessary and nothing is done . Else a copy is made .


def App(apptype, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """The App decorator function.

    Args:
        - apptype (string) : Apptype can be bash|python

    Kwargs:
        - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
          managing this app. This can be omitted only
          after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
        - walltime (int) : Walltime for app in seconds,
             default=60
        - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
        - cache (Bool) : Enable caching of the app call
             default=False

    Returns:
         A PythonApp or BashApp object, which when called runs the apps through the executor.
    """

    from parsl.app.python import PythonApp
    from parsl.app.bash import BashApp

    logger.warning("The 'App' decorator will be deprecated in Parsl 0.8. Please use 'python_app' or 'bash_app' instead.")

    if apptype == 'python':
        app_class = PythonApp
    elif apptype == 'bash':
        app_class = BashApp
    else:
        raise InvalidAppTypeError("Invalid apptype requested {}; must be 'python' or 'bash'".format(apptype))

    def wrapper(f):
        return app_class(f,
                         data_flow_kernel=data_flow_kernel,
                         walltime=walltime,
                         cache=cache,
                         executors=executors)
    return wrapper


def python_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making python apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@python_app` if using all defaults or `@python_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """
    from parsl.app.python import PythonApp

    def decorator(func):
        def wrapper(f):
            return PythonApp(f,
                             data_flow_kernel=data_flow_kernel,
                             walltime=walltime,
                             cache=cache,
                             executors=executors)
        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator

def python_app ( function = None , data_flow_kernel = None , walltime = 60 , cache = False , executors = 'all' ) : from parsl . app . python import PythonApp def decorator ( func ) : def wrapper ( f ) : return PythonApp ( f , data_flow_kernel = data_flow_kernel , walltime = walltime , cache = cache , executors = executors ) return wrapper ( func ) if function is not None : return decorator ( function ) return decorator

Decorator function for making python apps .


def bash_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making bash apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@bash_app` if using all defaults or `@bash_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """
    from parsl.app.bash import BashApp

    def decorator(func):
        def wrapper(f):
            return BashApp(f,
                           data_flow_kernel=data_flow_kernel,
                           walltime=walltime,
                           cache=cache,
                           executors=executors)
        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator

def bash_app ( function = None , data_flow_kernel = None , walltime = 60 , cache = False , executors = 'all' ) : from parsl . app . bash import BashApp def decorator ( func ) : def wrapper ( f ) : return BashApp ( f , data_flow_kernel = data_flow_kernel , walltime = walltime , cache = cache , executors = executors ) return wrapper ( func ) if function is not None : return decorator ( function ) return decorator

Decorator function for making bash apps .


def make_rundir(path):
    """When a path has not been specified, make the run directory.

    Creates a rundir with the following hierarchy:
        ./runinfo <- Home of all run directories
          |----000
          |----001 <- Directories for each run
          | ....
          |----NNN

    Kwargs:
        - path (str): String path to a specific run dir
               Default : None.
    """
    try:
        if not os.path.exists(path):
            os.makedirs(path)

        prev_rundirs = glob(os.path.join(path, "[0-9]*"))

        current_rundir = os.path.join(path, '000')

        if prev_rundirs:
            # Since we globbed on files named as 0-9
            x = sorted([int(os.path.basename(x)) for x in prev_rundirs])[-1]
            current_rundir = os.path.join(path, '{0:03}'.format(x + 1))

        os.makedirs(current_rundir)
        logger.debug("Parsl run initializing in rundir: {0}".format(current_rundir))
        return os.path.abspath(current_rundir)

    except Exception as e:
        logger.error("Failed to create a run directory")
        logger.error("Error: {0}".format(e))
        raise

def
make_rundir
(
path
)
:
try
:
if
not
os
.
path
.
exists
(
path
)
:
os
.
makedirs
(
path
)
prev_rundirs
=
glob
(
os
.
path
.
join
(
path
,
"[0-9]*"
)
)
current_rundir
=
os
.
path
.
join
(
path
,
'000'
)
if
prev_rundirs
:
x
=
sorted
(
[
int
(
os
.
path
.
basename
(
x
)
)
for
x
in
prev_rundirs
]
)
[
1
]
current_rundir
=
os
.
path
.
join
(
path
,
'{0:03}'
.
format
(
x
+
1
)
)
os
.
makedirs
(
current_rundir
)
logger
.
debug
(
"Parsl run initializing in rundir: {0}"
.
format
(
current_rundir
)
)
return
os
.
path
.
abspath
(
current_rundir
)
except
Exception
as
e
:
logger
.
error
(
"Failed to create a run directory"
)
logger
.
error
(
"Error: {0}"
.
format
(
e
)
)
raise

When a path has not been specified make the run directory .


def monitor(pid, task_id, monitoring_hub_url, run_id, sleep_dur=10):
    """Internal
    Monitors the Parsl task's resources by pointing psutil to the task's pid and watching it and its children.
    """
    import psutil

    radio = UDPRadio(monitoring_hub_url,
                     source_id=task_id)

    # these values are simple to log. Other information is available in special formats such as memory below.
    simple = ["cpu_num", 'cpu_percent', 'create_time', 'cwd', 'exe', 'memory_percent', 'nice', 'name', 'num_threads', 'pid', 'ppid', 'status', 'username']
    # values that can be summed up to see total resources used by task process and its children
    summable_values = ['cpu_percent', 'memory_percent', 'num_threads']

    pm = psutil.Process(pid)
    pm.cpu_percent()

    first_msg = True

    while True:
        try:
            d = {"psutil_process_" + str(k): v for k, v in pm.as_dict().items() if k in simple}
            d["run_id"] = run_id
            d["task_id"] = task_id
            d['resource_monitoring_interval'] = sleep_dur
            d['first_msg'] = first_msg
            d['timestamp'] = datetime.datetime.now()
            children = pm.children(recursive=True)
            d["psutil_cpu_count"] = psutil.cpu_count()
            d['psutil_process_memory_virtual'] = pm.memory_info().vms
            d['psutil_process_memory_resident'] = pm.memory_info().rss
            d['psutil_process_time_user'] = pm.cpu_times().user
            d['psutil_process_time_system'] = pm.cpu_times().system
            d['psutil_process_children_count'] = len(children)
            try:
                d['psutil_process_disk_write'] = pm.io_counters().write_bytes
                d['psutil_process_disk_read'] = pm.io_counters().read_bytes
            except psutil._exceptions.AccessDenied:
                # occassionally pid temp files that hold this information are unvailable to be read so set to zero
                d['psutil_process_disk_write'] = 0
                d['psutil_process_disk_read'] = 0
            for child in children:
                for k, v in child.as_dict(attrs=summable_values).items():
                    d['psutil_process_' + str(k)] += v
                d['psutil_process_time_user'] += child.cpu_times().user
                d['psutil_process_time_system'] += child.cpu_times().system
                d['psutil_process_memory_virtual'] += child.memory_info().vms
                d['psutil_process_memory_resident'] += child.memory_info().rss
                try:
                    d['psutil_process_disk_write'] += child.io_counters().write_bytes
                    d['psutil_process_disk_read'] += child.io_counters().read_bytes
                except psutil._exceptions.AccessDenied:
                    # occassionally pid temp files that hold this information are unvailable to be read so add zero
                    d['psutil_process_disk_write'] += 0
                    d['psutil_process_disk_read'] += 0

        finally:
            radio.send(MessageType.TASK_INFO, task_id, d)
            time.sleep(sleep_dur)
            first_msg = False

def monitor ( pid , task_id , monitoring_hub_url , run_id , sleep_dur = 10 ) : import psutil radio = UDPRadio ( monitoring_hub_url , source_id = task_id ) simple = [ "cpu_num" , 'cpu_percent' , 'create_time' , 'cwd' , 'exe' , 'memory_percent' , 'nice' , 'name' , 'num_threads' , 'pid' , 'ppid' , 'status' , 'username' ] summable_values = [ 'cpu_percent' , 'memory_percent' , 'num_threads' ] pm = psutil . Process ( pid ) pm . cpu_percent ( ) first_msg = True while True : try : d = { "psutil_process_" + str ( k ) : v for k , v in pm . as_dict ( ) . items ( ) if k in simple } d [ "run_id" ] = run_id d [ "task_id" ] = task_id d [ 'resource_monitoring_interval' ] = sleep_dur d [ 'first_msg' ] = first_msg d [ 'timestamp' ] = datetime . datetime . now ( ) children = pm . children ( recursive = True ) d [ "psutil_cpu_count" ] = psutil . cpu_count ( ) d [ 'psutil_process_memory_virtual' ] = pm . memory_info ( ) . vms d [ 'psutil_process_memory_resident' ] = pm . memory_info ( ) . rss d [ 'psutil_process_time_user' ] = pm . cpu_times ( ) . user d [ 'psutil_process_time_system' ] = pm . cpu_times ( ) . system d [ 'psutil_process_children_count' ] = len ( children ) try : d [ 'psutil_process_disk_write' ] = pm . io_counters ( ) . write_bytes d [ 'psutil_process_disk_read' ] = pm . io_counters ( ) . read_bytes except psutil . _exceptions . AccessDenied : d [ 'psutil_process_disk_write' ] = 0 d [ 'psutil_process_disk_read' ] = 0 for child in children : for k , v in child . as_dict ( attrs = summable_values ) . items ( ) : d [ 'psutil_process_' + str ( k ) ] += v d [ 'psutil_process_time_user' ] += child . cpu_times ( ) . user d [ 'psutil_process_time_system' ] += child . cpu_times ( ) . system d [ 'psutil_process_memory_virtual' ] += child . memory_info ( ) . vms d [ 'psutil_process_memory_resident' ] += child . memory_info ( ) . rss try : d [ 'psutil_process_disk_write' ] += child . io_counters ( ) . write_bytes d [ 'psutil_process_disk_read' ] += child . io_counters ( ) . read_bytes except psutil . _exceptions . AccessDenied : d [ 'psutil_process_disk_write' ] += 0 d [ 'psutil_process_disk_read' ] += 0 finally : radio . send ( MessageType . TASK_INFO , task_id , d ) time . sleep ( sleep_dur ) first_msg = False

Internal Monitors the Parsl task s resources by pointing psutil to the task s pid and watching it and its children .


def send(self, message_type, task_id, message):
        """ Sends a message to the UDP receiver

        Parameter
        ---------

        message_type: monitoring.MessageType (enum)
            In this case message type is RESOURCE_INFO most often
        task_id: int
            Task identifier of the task for which resource monitoring is being reported
        message: object
            Arbitrary pickle-able object that is to be sent

        Returns:
            # bytes sent
        """
        x = 0
        try:
            buffer = pickle.dumps((self.source_id,   # Identifier for manager
                                   int(time.time()),  # epoch timestamp
                                   message_type,
                                   message))
        except Exception as e:
            print("Exception during pickling {}".format(e))
            return

        try:
            x = self.sock.sendto(buffer, (self.ip, self.port))
        except socket.timeout:
            print("Could not send message within timeout limit")
            return False
        return x


def monitor_wrapper(f, task_id, monitoring_hub_url, run_id, sleep_dur):
        """ Internal
        Wrap the Parsl app with a function that will call the monitor function and point it at the correct pid when the task begins.
        """
        def wrapped(*args, **kwargs):
            p = Process(target=monitor, args=(os.getpid(), task_id, monitoring_hub_url, run_id, sleep_dur))
            p.start()
            try:
                return f(*args, **kwargs)
            finally:
                # There's a chance of zombification if the workers are killed by some signals
                p.terminate()
                p.join()
        return wrapped

def monitor_wrapper ( f , task_id , monitoring_hub_url , run_id , sleep_dur ) : def wrapped ( * args , ** kwargs ) : p = Process ( target = monitor , args = ( os . getpid ( ) , task_id , monitoring_hub_url , run_id , sleep_dur ) ) p . start ( ) try : return f ( * args , ** kwargs ) finally : p . terminate ( ) p . join ( ) return wrapped

Internal Wrap the Parsl app with a function that will call the monitor function and point it at the correct pid when the task begins .


def execute_no_wait(self, cmd, walltime=2, envs={}):
        ''' Execute asynchronousely without waiting for exitcode

        Args:
            - cmd (string): Commandline string to be executed on the remote side
            - walltime (int): timeout to exec_command

        KWargs:
            - envs (dict): A dictionary of env variables

        Returns:
            - None, stdout (readable stream), stderr (readable stream)

        Raises:
            - ChannelExecFailed (reason)
        '''

        # Execute the command
        stdin, stdout, stderr = self.ssh_client.exec_command(
            self.prepend_envs(cmd, envs), bufsize=-1, timeout=walltime
        )
        return None, stdout, stderr

def execute_no_wait ( self , cmd , walltime = 2 , envs = { } ) : stdin , stdout , stderr = self . ssh_client . exec_command ( self . prepend_envs ( cmd , envs ) , bufsize = - 1 , timeout = walltime ) return None , stdout , stderr


def push_file(self, local_source, remote_dir):
        ''' Transport a local file to a directory on a remote machine

        Args:
            - local_source (string): Path
            - remote_dir (string): Remote path

        Returns:
            - str: Path to copied file on remote machine

        Raises:
            - BadScriptPath : if script path on the remote side is bad
            - BadPermsScriptPath : You do not have perms to make the channel script dir
            - FileCopyException : FileCopy failed.

        '''
        remote_dest = remote_dir + '/' + os.path.basename(local_source)

        try:
            self.makedirs(remote_dir, exist_ok=True)
        except IOError as e:
            logger.exception("Pushing {0} to {1} failed".format(local_source, remote_dir))
            if e.errno == 2:
                raise BadScriptPath(e, self.hostname)
            elif e.errno == 13:
                raise BadPermsScriptPath(e, self.hostname)
            else:
                logger.exception("File push failed due to SFTP client failure")
                raise FileCopyException(e, self.hostname)
        try:
            self.sftp_client.put(local_source, remote_dest, confirm=True)
            # Set perm because some systems require the script to be executable
            self.sftp_client.chmod(remote_dest, 0o777)
        except Exception as e:
            logger.exception("File push from local source {} to remote destination {} failed".format(
                local_source, remote_dest))
            raise FileCopyException(e, self.hostname)

        return remote_dest


def pull_file(self, remote_source, local_dir):
        ''' Transport file on the remote side to a local directory

        Args:
            - remote_source (string): remote_source
            - local_dir (string): Local directory to copy to

        Returns:
            - str: Local path to file

        Raises:
            - FileExists : Name collision at local directory.
            - FileCopyException : FileCopy failed.
        '''

        local_dest = local_dir + '/' + os.path.basename(remote_source)

        try:
            os.makedirs(local_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                logger.exception("Failed to create script_dir: {0}".format(script_dir))
                raise BadScriptPath(e, self.hostname)

        # Easier to check this than to waste time trying to pull file and
        # realize there's a problem.
        if os.path.exists(local_dest):
            logger.exception("Remote file copy will overwrite a local file:{0}".format(local_dest))
            raise FileExists(None, self.hostname, filename=local_dest)

        try:
            self.sftp_client.get(remote_source, local_dest)
        except Exception as e:
            logger.exception("File pull failed")
            raise FileCopyException(e, self.hostname)

        return local_dest


def isdir(self, path):
        """Return true if the path refers to an existing directory.

        Parameters
        ----------
        path : str
            Path of directory on the remote side to check.
        """
        result = True
        try:
            self.sftp_client.lstat(path)
        except FileNotFoundError:
            result = False

        return result

def isdir ( self , path ) : result = True try : self . sftp_client . lstat ( path ) except FileNotFoundError : result = False return result

Return true if the path refers to an existing directory .


def makedirs(self, path, mode=511, exist_ok=False):
        """Create a directory on the remote side.

        If intermediate directories do not exist, they will be created.

        Parameters
        ----------
        path : str
            Path of directory on the remote side to create.
        mode : int
            Permissions (posix-style) for the newly-created directory.
        exist_ok : bool
            If False, raise an OSError if the target directory already exists.
        """
        if exist_ok is False and self.isdir(path):
            raise OSError('Target directory {} already exists'.format(path))

        self.execute_wait('mkdir -p {}'.format(path))
        self.sftp_client.chmod(path, mode)

def
makedirs
(
self
,
path
,
mode
=
511
,
exist_ok
=
False
)
:
if
exist_ok
is
False
and
self
.
isdir
(
path
)
:
raise
OSError
(
'Target directory {} already exists'
.
format
(
path
)
)
self
.
execute_wait
(
'mkdir -p {}'
.
format
(
path
)
)
self
.
sftp_client
.
chmod
(
path
,
mode
)


def notify(self, event_id):
        """Let the FlowControl system know that there is an event."""
        self._event_buffer.extend([event_id])
        self._event_count += 1
        if self._event_count >= self.threshold:
            logger.debug("Eventcount >= threshold")
            self.make_callback(kind="event")

def
notify
(
self
,
event_id
)
:
self
.
_event_buffer
.
extend
(
[
event_id
]
)
self
.
_event_count
+=
1
if
self
.
_event_count
>=
self
.
threshold
:
logger
.
debug
(
"Eventcount >= threshold"
)
self
.
make_callback
(
kind
=
"event"
)

Let the FlowControl system know that there is an event .


def _create_deployment(self, deployment):
        """ Create the kubernetes deployment """

        api_response = self.kube_client.create_namespaced_deployment(
            body=deployment,
            namespace=self.namespace)

        logger.debug("Deployment created. status='{0}'".format(str(api_response.status)))

def
_create_deployment
(
self
,
deployment
)
:
api_response
=
self
.
kube_client
.
create_namespaced_deployment
(
body
=
deployment
,
namespace
=
self
.
namespace
)
logger
.
debug
(
"Deployment created. status='{0}'"
.
format
(
str
(
api_response
.
status
)
)
)


def initialize_scaling(self):
        """ Compose the launch command and call the scale_out

        This should be implemented in the child classes to take care of
        executor specific oddities.
        """
        debug_opts = "--debug" if self.worker_debug else ""
        max_workers = "" if self.max_workers == float('inf') else "--max_workers={}".format(self.max_workers)

        worker_logdir = "{}/{}".format(self.run_dir, self.label)
        if self.worker_logdir_root is not None:
            worker_logdir = "{}/{}".format(self.worker_logdir_root, self.label)

        l_cmd = self.launch_cmd.format(debug=debug_opts,
                                       prefetch_capacity=self.prefetch_capacity,
                                       task_url=self.worker_task_url,
                                       result_url=self.worker_result_url,
                                       cores_per_worker=self.cores_per_worker,
                                       max_workers=max_workers,
                                       nodes_per_block=self.provider.nodes_per_block,
                                       heartbeat_period=self.heartbeat_period,
                                       heartbeat_threshold=self.heartbeat_threshold,
                                       poll_period=self.poll_period,
                                       logdir=worker_logdir)
        self.launch_cmd = l_cmd
        logger.debug("Launch command: {}".format(self.launch_cmd))

        self._scaling_enabled = self.provider.scaling_enabled
        logger.debug("Starting HighThroughputExecutor with provider:\n%s", self.provider)
        if hasattr(self.provider, 'init_blocks'):
            try:
                self.scale_out(blocks=self.provider.init_blocks)
            except Exception as e:
                logger.error("Scaling out failed: {}".format(e))
                raise e


def _start_local_queue_process(self):
        """ Starts the interchange process locally

        Starts the interchange process locally and uses an internal command queue to
        get the worker task and result ports that the interchange has bound to.
        """
        comm_q = Queue(maxsize=10)
        self.queue_proc = Process(target=interchange.starter,
                                  args=(comm_q,),
                                  kwargs={"client_ports": (self.outgoing_q.port,
                                                           self.incoming_q.port,
                                                           self.command_client.port),
                                          "worker_ports": self.worker_ports,
                                          "worker_port_range": self.worker_port_range,
                                          "logdir": "{}/{}".format(self.run_dir, self.label),
                                          "suppress_failure": self.suppress_failure,
                                          "heartbeat_threshold": self.heartbeat_threshold,
                                          "poll_period": self.poll_period,
                                          "logging_level": logging.DEBUG if self.worker_debug else logging.INFO
                                  },
        )
        self.queue_proc.start()
        try:
            (worker_task_port, worker_result_port) = comm_q.get(block=True, timeout=120)
        except queue.Empty:
            logger.error("Interchange has not completed initialization in 120s. Aborting")
            raise Exception("Interchange failed to start")

        self.worker_task_url = "tcp://{}:{}".format(self.address, worker_task_port)
        self.worker_result_url = "tcp://{}:{}".format(self.address, worker_result_port)


def hold_worker(self, worker_id):
        """Puts a worker on hold, preventing scheduling of additional tasks to it.

        This is called "hold" mostly because this only stops scheduling of tasks,
        and does not actually kill the worker.

        Parameters
        ----------

        worker_id : str
            Worker id to be put on hold
        """
        c = self.command_client.run("HOLD_WORKER;{}".format(worker_id))
        logger.debug("Sent hold request to worker: {}".format(worker_id))
        return c

def
hold_worker
(
self
,
worker_id
)
:
c
=
self
.
command_client
.
run
(
"HOLD_WORKER;{}"
.
format
(
worker_id
)
)
logger
.
debug
(
"Sent hold request to worker: {}"
.
format
(
worker_id
)
)
return
c

Puts a worker on hold preventing scheduling of additional tasks to it .


def _hold_block(self, block_id):
        """ Sends hold command to all managers which are in a specific block

        Parameters
        ----------
        block_id : str
             Block identifier of the block to be put on hold
        """

        managers = self.connected_managers

        for manager in managers:
            if manager['block_id'] == block_id:
                logger.debug("[HOLD_BLOCK]: Sending hold to manager:{}".format(manager['manager']))
                self.hold_worker(manager['manager'])


def scale_out(self, blocks=1):
        """Scales out the number of blocks by "blocks"

        Raises:
             NotImplementedError
        """
        r = []
        for i in range(blocks):
            if self.provider:
                external_block_id = str(len(self.blocks))
                launch_cmd = self.launch_cmd.format(block_id=external_block_id)
                internal_block = self.provider.submit(launch_cmd, 1, 1)
                logger.debug("Launched block {}->{}".format(external_block_id, internal_block))
                if not internal_block:
                    raise(ScalingFailed(self.provider.label,
                                        "Attempts to provision nodes via provider has failed"))
                r.extend([external_block_id])
                self.blocks[external_block_id] = internal_block
            else:
                logger.error("No execution provider available")
                r = None
        return r

Scales out the number of blocks by blocks


def status(self):
        """Return status of all blocks."""

        status = []
        if self.provider:
            status = self.provider.status(self.blocks.values())

        return status

def status ( self ) : status = [ ] if self . provider : status = self . provider . status ( self . blocks . values ( ) ) return status


def readinto(self, buf, **kwargs):
        """
        Read into ``buf`` from the device. The number of bytes read will be the
        length of ``buf``.

        If ``start`` or ``end`` is provided, then the buffer will be sliced
        as if ``buf[start:end]``. This will not cause an allocation like
        ``buf[start:end]`` will so it saves memory.

        :param bytearray buffer: buffer to write into
        :param int start: Index to start writing at
        :param int end: Index to write up to but not include
        """
        self.i2c.readfrom_into(self.device_address, buf, **kwargs)
        if self._debug:
            print("i2c_device.readinto:", [hex(i) for i in buf])

def readinto ( self , buf , ** kwargs ) : self . i2c . readfrom_into ( self . device_address , buf , ** kwargs ) if self . _debug : print ( "i2c_device.readinto:" , [ hex ( i ) for i in buf ] )

Read into buf from the device . The number of bytes read will be the length of buf .


def write(self, buf, **kwargs):
        """
        Write the bytes from ``buffer`` to the device. Transmits a stop bit if
        ``stop`` is set.

        If ``start`` or ``end`` is provided, then the buffer will be sliced
        as if ``buffer[start:end]``. This will not cause an allocation like
        ``buffer[start:end]`` will so it saves memory.

        :param bytearray buffer: buffer containing the bytes to write
        :param int start: Index to start writing from
        :param int end: Index to read up to but not include
        :param bool stop: If true, output an I2C stop condition after the buffer is written
        """
        self.i2c.writeto(self.device_address, buf, **kwargs)
        if self._debug:
            print("i2c_device.write:", [hex(i) for i in buf])

def write ( self , buf , ** kwargs ) : self . i2c . writeto ( self . device_address , buf , ** kwargs ) if self . _debug : print ( "i2c_device.write:" , [ hex ( i ) for i in buf ] )

Write the bytes from buffer to the device . Transmits a stop bit if stop is set .


def write_then_readinto(self, out_buffer, in_buffer, *,
                            out_start=0, out_end=None, in_start=0, in_end=None, stop=True):
        """
        Write the bytes from ``out_buffer`` to the device, then immediately
        reads into ``in_buffer`` from the device. The number of bytes read
        will be the length of ``in_buffer``.
        Transmits a stop bit after the write, if ``stop`` is set.

        If ``out_start`` or ``out_end`` is provided, then the output buffer
        will be sliced as if ``out_buffer[out_start:out_end]``. This will
        not cause an allocation like ``buffer[out_start:out_end]`` will so
        it saves memory.

        If ``in_start`` or ``in_end`` is provided, then the input buffer
        will be sliced as if ``in_buffer[in_start:in_end]``. This will not
        cause an allocation like ``in_buffer[in_start:in_end]`` will so
        it saves memory.

        :param bytearray out_buffer: buffer containing the bytes to write
        :param bytearray in_buffer: buffer containing the bytes to read into
        :param int out_start: Index to start writing from
        :param int out_end: Index to read up to but not include
        :param int in_start: Index to start writing at
        :param int in_end: Index to write up to but not include
        :param bool stop: If true, output an I2C stop condition after the buffer is written
        """
        if out_end is None:
            out_end = len(out_buffer)
        if in_end is None:
            in_end = len(in_buffer)
        if hasattr(self.i2c, 'writeto_then_readfrom'):
            if self._debug:
                print("i2c_device.writeto_then_readfrom.out_buffer:",
                      [hex(i) for i in out_buffer[out_start:out_end]])
            # In linux, at least, this is a special kernel function call
            self.i2c.writeto_then_readfrom(self.device_address, out_buffer, in_buffer,
                                           out_start=out_start, out_end=out_end,
                                           in_start=in_start, in_end=in_end, stop=stop)
            if self._debug:
                print("i2c_device.writeto_then_readfrom.in_buffer:",
                      [hex(i) for i in in_buffer[in_start:in_end]])
        else:
            # If we don't have a special implementation, we can fake it with two calls
            self.write(out_buffer, start=out_start, end=out_end, stop=stop)
            if self._debug:
                print("i2c_device.write_then_readinto.write.out_buffer:",
                      [hex(i) for i in out_buffer[out_start:out_end]])
            self.readinto(in_buffer, start=in_start, end=in_end)
            if self._debug:
                print("i2c_device.write_then_readinto.readinto.in_buffer:",
                      [hex(i) for i in in_buffer[in_start:in_end]])

Write the bytes from out_buffer to the device then immediately reads into in_buffer from the device . The number of bytes read will be the length of in_buffer . Transmits a stop bit after the write if stop is set .


def sixteen_oscillator_two_stimulated_ensembles_grid():
    "Not accurate false due to spikes are observed"
    parameters = legion_parameters();
    parameters.teta_x = -1.1;
    template_dynamic_legion(16, 2000, 1500, conn_type = conn_type.GRID_FOUR, params = parameters, stimulus = [1, 1, 1, 0,
                                                                                1, 1, 1, 0,
                                                                                0, 0, 0, 1,
                                                                                0, 0, 1, 1]);


def cleanup_old_versions(
    src, keep_last_versions,
    config_file='config.yaml', profile_name=None,
):
    """Deletes old deployed versions of the function in AWS Lambda.

    Won't delete $Latest and any aliased version

    :param str src:
        The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param int keep_last_versions:
        The number of recent versions to keep and not delete
    """
    if keep_last_versions <= 0:
        print("Won't delete all versions. Please do this manually")
    else:
        path_to_config_file = os.path.join(src, config_file)
        cfg = read_cfg(path_to_config_file, profile_name)

        profile_name = cfg.get('profile')
        aws_access_key_id = cfg.get('aws_access_key_id')
        aws_secret_access_key = cfg.get('aws_secret_access_key')

        client = get_client(
            'lambda', profile_name, aws_access_key_id, aws_secret_access_key,
            cfg.get('region'),
        )

        response = client.list_versions_by_function(
            FunctionName=cfg.get('function_name'),
        )
        versions = response.get('Versions')
        if len(response.get('Versions')) < keep_last_versions:
            print('Nothing to delete. (Too few versions published)')
        else:
            version_numbers = [elem.get('Version') for elem in
                               versions[1:-keep_last_versions]]
            for version_number in version_numbers:
                try:
                    client.delete_function(
                        FunctionName=cfg.get('function_name'),
                        Qualifier=version_number,
                    )
                except botocore.exceptions.ClientError as e:
                    print('Skipping Version {}: {}'
                          .format(version_number, e.message))

Deletes old deployed versions of the function in AWS Lambda .


def deploy(
        src, requirements=None, local_package=None,
        config_file='config.yaml', profile_name=None,
        preserve_vpc=False
):
    """Deploys a new function to AWS Lambda.

    :param str src:
        The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param str local_package:
        The path to a local package with should be included in the deploy as
        well (and/or is not available on PyPi)
    """
    # Load and parse the config file.
    path_to_config_file = os.path.join(src, config_file)
    cfg = read_cfg(path_to_config_file, profile_name)

    # Copy all the pip dependencies required to run your code into a temporary
    # folder then add the handler file in the root of this directory.
    # Zip the contents of this folder into a single file and output to the dist
    # directory.
    path_to_zip_file = build(
        src, config_file=config_file,
        requirements=requirements,
        local_package=local_package,
    )

    existing_config = get_function_config(cfg)
    if existing_config:
        update_function(cfg, path_to_zip_file, existing_config, preserve_vpc=preserve_vpc)
    else:
        create_function(cfg, path_to_zip_file)

def deploy ( src , requirements = None , local_package = None , config_file = 'config.yaml' , profile_name = None , preserve_vpc = False ) : path_to_config_file = os . path . join ( src , config_file ) cfg = read_cfg ( path_to_config_file , profile_name ) path_to_zip_file = build ( src , config_file = config_file , requirements = requirements , local_package = local_package , ) existing_config = get_function_config ( cfg ) if existing_config : update_function ( cfg , path_to_zip_file , existing_config , preserve_vpc = preserve_vpc ) else : create_function ( cfg , path_to_zip_file )


def deploy_s3(
    src, requirements=None, local_package=None,
    config_file='config.yaml', profile_name=None,
    preserve_vpc=False
):
    """Deploys a new function via AWS S3.

    :param str src:
        The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param str local_package:
        The path to a local package with should be included in the deploy as
        well (and/or is not available on PyPi)
    """
    # Load and parse the config file.
    path_to_config_file = os.path.join(src, config_file)
    cfg = read_cfg(path_to_config_file, profile_name)

    # Copy all the pip dependencies required to run your code into a temporary
    # folder then add the handler file in the root of this directory.
    # Zip the contents of this folder into a single file and output to the dist
    # directory.
    path_to_zip_file = build(
        src, config_file=config_file, requirements=requirements,
        local_package=local_package,
    )

    use_s3 = True
    s3_file = upload_s3(cfg, path_to_zip_file, use_s3)
    existing_config = get_function_config(cfg)
    if existing_config:
        update_function(cfg, path_to_zip_file, existing_config, use_s3=use_s3,
                        s3_file=s3_file, preserve_vpc=preserve_vpc)
    else:
        create_function(cfg, path_to_zip_file, use_s3=use_s3, s3_file=s3_file)

def deploy_s3 ( src , requirements = None , local_package = None , config_file = 'config.yaml' , profile_name = None , preserve_vpc = False ) : path_to_config_file = os . path . join ( src , config_file ) cfg = read_cfg ( path_to_config_file , profile_name ) path_to_zip_file = build ( src , config_file = config_file , requirements = requirements , local_package = local_package , ) use_s3 = True s3_file = upload_s3 ( cfg , path_to_zip_file , use_s3 ) existing_config = get_function_config ( cfg ) if existing_config : update_function ( cfg , path_to_zip_file , existing_config , use_s3 = use_s3 , s3_file = s3_file , preserve_vpc = preserve_vpc ) else : create_function ( cfg , path_to_zip_file , use_s3 = use_s3 , s3_file = s3_file )


def upload(
        src, requirements=None, local_package=None,
        config_file='config.yaml', profile_name=None,
):
    """Uploads a new function to AWS S3.

    :param str src:
        The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param str local_package:
        The path to a local package with should be included in the deploy as
        well (and/or is not available on PyPi)
    """
    # Load and parse the config file.
    path_to_config_file = os.path.join(src, config_file)
    cfg = read_cfg(path_to_config_file, profile_name)

    # Copy all the pip dependencies required to run your code into a temporary
    # folder then add the handler file in the root of this directory.
    # Zip the contents of this folder into a single file and output to the dist
    # directory.
    path_to_zip_file = build(
        src, config_file=config_file, requirements=requirements,
        local_package=local_package,
    )

    upload_s3(cfg, path_to_zip_file)

def upload ( src , requirements = None , local_package = None , config_file = 'config.yaml' , profile_name = None , ) : path_to_config_file = os . path . join ( src , config_file ) cfg = read_cfg ( path_to_config_file , profile_name ) path_to_zip_file = build ( src , config_file = config_file , requirements = requirements , local_package = local_package , ) upload_s3 ( cfg , path_to_zip_file )


def invoke(
    src, event_file='event.json',
    config_file='config.yaml', profile_name=None,
    verbose=False,
):
    """Simulates a call to your function.

    :param str src:
        The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param str alt_event:
        An optional argument to override which event file to use.
    :param bool verbose:
        Whether to print out verbose details.
    """
    # Load and parse the config file.
    path_to_config_file = os.path.join(src, config_file)
    cfg = read_cfg(path_to_config_file, profile_name)

    # Set AWS_PROFILE environment variable based on `--profile` option.
    if profile_name:
        os.environ['AWS_PROFILE'] = profile_name

    # Load environment variables from the config file into the actual
    # environment.
    env_vars = cfg.get('environment_variables')
    if env_vars:
        for key, value in env_vars.items():
            os.environ[key] = get_environment_variable_value(value)

    # Load and parse event file.
    path_to_event_file = os.path.join(src, event_file)
    event = read(path_to_event_file, loader=json.loads)

    # Tweak to allow module to import local modules
    try:
        sys.path.index(src)
    except ValueError:
        sys.path.append(src)

    handler = cfg.get('handler')
    # Inspect the handler string (<module>.<function name>) and translate it
    # into a function we can execute.
    fn = get_callable_handler_function(src, handler)

    timeout = cfg.get('timeout')
    if timeout:
        context = LambdaContext(cfg.get('function_name'),timeout)
    else:
        context = LambdaContext(cfg.get('function_name'))

    start = time.time()
    results = fn(event, context)
    end = time.time()

    print('{0}'.format(results))
    if verbose:
        print('\nexecution time: {:.8f}s\nfunction execution '
              'timeout: {:2}s'.format(end - start, cfg.get('timeout', 15)))


def init(src, minimal=False):
    """Copies template files to a given directory.

    :param str src:
        The path to output the template lambda project files.
    :param bool minimal:
        Minimal possible template files (excludes event.json).
    """

    templates_path = os.path.join(
        os.path.dirname(os.path.abspath(__file__)), 'project_templates',
    )
    for filename in os.listdir(templates_path):
        if (minimal and filename == 'event.json') or filename.endswith('.pyc'):
            continue
        dest_path = os.path.join(templates_path, filename)

        if not os.path.isdir(dest_path):
            copy(dest_path, src)

def init ( src , minimal = False ) : templates_path = os . path . join ( os . path . dirname ( os . path . abspath ( __file__ ) ) , 'project_templates' , ) for filename in os . listdir ( templates_path ) : if ( minimal and filename == 'event.json' ) or filename . endswith ( '.pyc' ) : continue dest_path = os . path . join ( templates_path , filename ) if not os . path . isdir ( dest_path ) : copy ( dest_path , src )

Copies template files to a given directory .


def build(
    src, requirements=None, local_package=None,
    config_file='config.yaml', profile_name=None,
):
    """Builds the file bundle.

    :param str src:
       The path to your Lambda ready project (folder must contain a valid
        config.yaml and handler module (e.g.: service.py).
    :param str local_package:
        The path to a local package with should be included in the deploy as
        well (and/or is not available on PyPi)
    """
    # Load and parse the config file.
    path_to_config_file = os.path.join(src, config_file)
    cfg = read_cfg(path_to_config_file, profile_name)

    # Get the absolute path to the output directory and create it if it doesn't
    # already exist.
    dist_directory = cfg.get('dist_directory', 'dist')
    path_to_dist = os.path.join(src, dist_directory)
    mkdir(path_to_dist)

    # Combine the name of the Lambda function with the current timestamp to use
    # for the output filename.
    function_name = cfg.get('function_name')
    output_filename = '{0}-{1}.zip'.format(timestamp(), function_name)

    path_to_temp = mkdtemp(prefix='aws-lambda')
    pip_install_to_target(
        path_to_temp,
        requirements=requirements,
        local_package=local_package,
    )

    # Hack for Zope.
    if 'zope' in os.listdir(path_to_temp):
        print(
            'Zope packages detected; fixing Zope package paths to '
            'make them importable.',
        )
        # Touch.
        with open(os.path.join(path_to_temp, 'zope/__init__.py'), 'wb'):
            pass

    # Gracefully handle whether ".zip" was included in the filename or not.
    output_filename = (
        '{0}.zip'.format(output_filename)
        if not output_filename.endswith('.zip')
        else output_filename
    )

    # Allow definition of source code directories we want to build into our
    # zipped package.
    build_config = defaultdict(**cfg.get('build', {}))
    build_source_directories = build_config.get('source_directories', '')
    build_source_directories = (
        build_source_directories
        if build_source_directories is not None
        else ''
    )
    source_directories = [
        d.strip() for d in build_source_directories.split(',')
    ]

    files = []
    for filename in os.listdir(src):
        if os.path.isfile(filename):
            if filename == '.DS_Store':
                continue
            if filename == config_file:
                continue
            print('Bundling: %r' % filename)
            files.append(os.path.join(src, filename))
        elif os.path.isdir(filename) and filename in source_directories:
            print('Bundling directory: %r' % filename)
            files.append(os.path.join(src, filename))

    # "cd" into `temp_path` directory.
    os.chdir(path_to_temp)
    for f in files:
        if os.path.isfile(f):
            _, filename = os.path.split(f)

            # Copy handler file into root of the packages folder.
            copyfile(f, os.path.join(path_to_temp, filename))
            copystat(f, os.path.join(path_to_temp, filename))
        elif os.path.isdir(f):
            destination_folder = os.path.join(path_to_temp, f[len(src) + 1:])
            copytree(f, destination_folder)

    # Zip them together into a single file.
    # TODO: Delete temp directory created once the archive has been compiled.
    path_to_zip_file = archive('./', path_to_dist, output_filename)
    return path_to_zip_file


def get_callable_handler_function(src, handler):
    """Tranlate a string of the form "module.function" into a callable
    function.

    :param str src:
      The path to your Lambda project containing a valid handler file.
    :param str handler:
      A dot delimited string representing the `<module>.<function name>`.
    """

    # "cd" into `src` directory.
    os.chdir(src)

    module_name, function_name = handler.split('.')
    filename = get_handler_filename(handler)

    path_to_module_file = os.path.join(src, filename)
    module = load_source(module_name, path_to_module_file)
    return getattr(module, function_name)

def get_callable_handler_function ( src , handler ) : os . chdir ( src ) module_name , function_name = handler . split ( '.' ) filename = get_handler_filename ( handler ) path_to_module_file = os . path . join ( src , filename ) module = load_source ( module_name , path_to_module_file ) return getattr ( module , function_name )

Tranlate a string of the form module . function into a callable function .


def _install_packages(path, packages):
    """Install all packages listed to the target directory.

    Ignores any package that includes Python itself and python-lambda as well
    since its only needed for deploying and not running the code

    :param str path:
        Path to copy installed pip packages to.
    :param list packages:
        A list of packages to be installed via pip.
    """
    def _filter_blacklist(package):
        blacklist = ['-i', '#', 'Python==', 'python-lambda==']
        return all(package.startswith(entry) is False for entry in blacklist)
    filtered_packages = filter(_filter_blacklist, packages)
    for package in filtered_packages:
        if package.startswith('-e '):
            package = package.replace('-e ', '')

        print('Installing {package}'.format(package=package))
        subprocess.check_call([sys.executable, '-m', 'pip', 'install', package, '-t', path, '--ignore-installed'])
    print ('Install directory contents are now: {directory}'.format(directory=os.listdir(path)))

Install all packages listed to the target directory .


def get_role_name(region, account_id, role):
    """Shortcut to insert the `account_id` and `role` into the iam string."""
    prefix = ARN_PREFIXES.get(region, 'aws')
    return 'arn:{0}:iam::{1}:role/{2}'.format(prefix, account_id, role)

def get_role_name ( region , account_id , role ) : prefix = ARN_PREFIXES . get ( region , 'aws' ) return 'arn:{0}:iam::{1}:role/{2}' . format ( prefix , account_id , role )

Shortcut to insert the account_id and role into the iam string .


def get_account_id(
    profile_name, aws_access_key_id, aws_secret_access_key,
    region=None,
):
    """Query STS for a users' account_id"""
    client = get_client(
        'sts', profile_name, aws_access_key_id, aws_secret_access_key,
        region,
    )
    return client.get_caller_identity().get('Account')

def get_account_id ( profile_name , aws_access_key_id , aws_secret_access_key , region = None , ) : client = get_client ( 'sts' , profile_name , aws_access_key_id , aws_secret_access_key , region , ) return client . get_caller_identity ( ) . get ( 'Account' )


def get_client(
    client, profile_name, aws_access_key_id, aws_secret_access_key,
    region=None,
):
    """Shortcut for getting an initialized instance of the boto3 client."""

    boto3.setup_default_session(
        profile_name=profile_name,
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        region_name=region,
    )
    return boto3.client(client)

def get_client ( client , profile_name , aws_access_key_id , aws_secret_access_key , region = None , ) : boto3 . setup_default_session ( profile_name = profile_name , aws_access_key_id = aws_access_key_id , aws_secret_access_key = aws_secret_access_key , region_name = region , ) return boto3 . client ( client )

Shortcut for getting an initialized instance of the boto3 client .