def write_object_to_file(self,
                             query_results,
                             filename,
                             fmt="csv",
                             coerce_to_timestamp=False,
                             record_time_added=False):
        """
        Write query results to file.

        Acceptable formats are:
            - csv:
                comma-separated-values file. This is the default format.
            - json:
                JSON array. Each element in the array is a different row.
            - ndjson:
                JSON array but each element is new-line delimited instead of comma delimited like in `json`

        This requires a significant amount of cleanup.
        Pandas doesn't handle output to CSV and json in a uniform way.
        This is especially painful for datetime types.
        Pandas wants to write them as strings in CSV, but as millisecond Unix timestamps.

        By default, this function will try and leave all values as they are represented in Salesforce.
        You use the `coerce_to_timestamp` flag to force all datetimes to become Unix timestamps (UTC).
        This is can be greatly beneficial as it will make all of your datetime fields look the same,
        and makes it easier to work with in other database environments

        :param query_results: the results from a SQL query
        :type query_results: list of dict
        :param filename: the name of the file where the data should be dumped to
        :type filename: str
        :param fmt: the format you want the output in. Default:  'csv'
        :type fmt: str
        :param coerce_to_timestamp: True if you want all datetime fields to be converted into Unix timestamps.
            False if you want them to be left in the same format as they were in Salesforce.
            Leaving the value as False will result in datetimes being strings. Default: False
        :type coerce_to_timestamp: bool
        :param record_time_added: True if you want to add a Unix timestamp field
            to the resulting data that marks when the data was fetched from Salesforce. Default: False
        :type record_time_added: bool
        :return: the dataframe that gets written to the file.
        :rtype: pd.Dataframe
        """
        fmt = fmt.lower()
        if fmt not in ['csv', 'json', 'ndjson']:
            raise ValueError("Format value is not recognized: {}".format(fmt))

        # this line right here will convert all integers to floats
        # if there are any None/np.nan values in the column
        # that's because None/np.nan cannot exist in an integer column
        # we should write all of our timestamps as FLOATS in our final schema
        df = pd.DataFrame.from_records(query_results, exclude=["attributes"])

        df.columns = [column.lower() for column in df.columns]

        # convert columns with datetime strings to datetimes
        # not all strings will be datetimes, so we ignore any errors that occur
        # we get the object's definition at this point and only consider
        # features that are DATE or DATETIME
        if coerce_to_timestamp and df.shape[0] > 0:
            # get the object name out of the query results
            # it's stored in the "attributes" dictionary
            # for each returned record
            object_name = query_results[0]['attributes']['type']

            self.log.info("Coercing timestamps for: %s", object_name)

            schema = self.describe_object(object_name)

            # possible columns that can be converted to timestamps
            # are the ones that are either date or datetime types
            # strings are too general and we risk unintentional conversion
            possible_timestamp_cols = [
                field['name'].lower()
                for field in schema['fields']
                if field['type'] in ["date", "datetime"] and field['name'].lower() in df.columns
            ]
            df[possible_timestamp_cols] = df[possible_timestamp_cols].apply(self._to_timestamp)

        if record_time_added:
            fetched_time = time.time()
            df["time_fetched_from_salesforce"] = fetched_time

        # write the CSV or JSON file depending on the option
        # NOTE:
        #   datetimes here are an issue.
        #   There is no good way to manage the difference
        #   for to_json, the options are an epoch or a ISO string
        #   but for to_csv, it will be a string output by datetime
        #   For JSON we decided to output the epoch timestamp in seconds
        #   (as is fairly standard for JavaScript)
        #   And for csv, we do a string
        if fmt == "csv":
            # there are also a ton of newline objects that mess up our ability to write to csv
            # we remove these newlines so that the output is a valid CSV format
            self.log.info("Cleaning data and writing to CSV")
            possible_strings = df.columns[df.dtypes == "object"]
            df[possible_strings] = df[possible_strings].apply(
                lambda x: x.str.replace("\r\n", "").str.replace("\n", "")
            )
            # write the dataframe
            df.to_csv(filename, index=False)
        elif fmt == "json":
            df.to_json(filename, "records", date_unit="s")
        elif fmt == "ndjson":
            df.to_json(filename, "records", lines=True, date_unit="s")

        return df

def
write_object_to_file
(
self
,
query_results
,
filename
,
fmt
=
"csv"
,
coerce_to_timestamp
=
False
,
record_time_added
=
False
)
:
fmt
=
fmt
.
lower
(
)
if
fmt
not
in
[
'csv'
,
'json'
,
'ndjson'
]
:
raise
ValueError
(
"Format value is not recognized: {}"
.
format
(
fmt
)
)
df
=
pd
.
DataFrame
.
from_records
(
query_results
,
exclude
=
[
"attributes"
]
)
df
.
columns
=
[
column
.
lower
(
)
for
column
in
df
.
columns
]
if
coerce_to_timestamp
and
df
.
shape
[
0
]
>
0
:
object_name
=
query_results
[
0
]
[
'attributes'
]
[
'type'
]
self
.
log
.
info
(
"Coercing timestamps for: %s"
,
object_name
)
schema
=
self
.
describe_object
(
object_name
)
possible_timestamp_cols
=
[
field
[
'name'
]
.
lower
(
)
for
field
in
schema
[
'fields'
]
if
field
[
'type'
]
in
[
"date"
,
"datetime"
]
and
field
[
'name'
]
.
lower
(
)
in
df
.
columns
]
df
[
possible_timestamp_cols
]
=
df
[
possible_timestamp_cols
]
.
apply
(
self
.
_to_timestamp
)
if
record_time_added
:
fetched_time
=
time
.
time
(
)
df
[
"time_fetched_from_salesforce"
]
=
fetched_time
if
fmt
==
"csv"
:
self
.
log
.
info
(
"Cleaning data and writing to CSV"
)
possible_strings
=
df
.
columns
[
df
.
dtypes
==
"object"
]
df
[
possible_strings
]
=
df
[
possible_strings
]
.
apply
(
lambda
x
:
x
.
str
.
replace
(
"\r\n"
,
""
)
.
str
.
replace
(
"\n"
,
""
)
)
df
.
to_csv
(
filename
,
index
=
False
)
elif
fmt
==
"json"
:
df
.
to_json
(
filename
,
"records"
,
date_unit
=
"s"
)
elif
fmt
==
"ndjson"
:
df
.
to_json
(
filename
,
"records"
,
lines
=
True
,
date_unit
=
"s"
)
return
df


def get_conn(self):
        """
        Fetches PyMongo Client
        """
        if self.client is not None:
            return self.client

        # Mongo Connection Options dict that is unpacked when passed to MongoClient
        options = self.extras

        # If we are using SSL disable requiring certs from specific hostname
        if options.get('ssl', False):
            options.update({'ssl_cert_reqs': CERT_NONE})

        self.client = MongoClient(self.uri, **options)

        return self.client

def get_conn ( self ) : if self . client is not None : return self . client options = self . extras if options . get ( 'ssl' , False ) : options . update ( { 'ssl_cert_reqs' : CERT_NONE } ) self . client = MongoClient ( self . uri , ** options ) return self . client


def get_collection(self, mongo_collection, mongo_db=None):
        """
        Fetches a mongo collection object for querying.

        Uses connection schema as DB unless specified.
        """
        mongo_db = mongo_db if mongo_db is not None else self.connection.schema
        mongo_conn = self.get_conn()

        return mongo_conn.get_database(mongo_db).get_collection(mongo_collection)

def get_collection ( self , mongo_collection , mongo_db = None ) : mongo_db = mongo_db if mongo_db is not None else self . connection . schema mongo_conn = self . get_conn ( ) return mongo_conn . get_database ( mongo_db ) . get_collection ( mongo_collection )

Fetches a mongo collection object for querying .


def replace_many(self, mongo_collection, docs,
                     filter_docs=None, mongo_db=None, upsert=False, collation=None,
                     **kwargs):
        """
        Replaces many documents in a mongo collection.

        Uses bulk_write with multiple ReplaceOne operations
        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.bulk_write

        .. note::
            If no ``filter_docs``are given, it is assumed that all
            replacement documents contain the ``_id`` field which are then
            used as filters.

        :param mongo_collection: The name of the collection to update.
        :type mongo_collection: str
        :param docs: The new documents.
        :type docs: list[dict]
        :param filter_docs: A list of queries that match the documents to replace.
            Can be omitted; then the _id fields from docs will be used.
        :type filter_docs: list[dict]
        :param mongo_db: The name of the database to use.
            Can be omitted; then the database from the connection string is used.
        :type mongo_db: str
        :param upsert: If ``True``, perform an insert if no documents
            match the filters for the replace operation.
        :type upsert: bool
        :param collation: An instance of
            :class:`~pymongo.collation.Collation`. This option is only
            supported on MongoDB 3.4 and above.
        :type collation: pymongo.collation.Collation

        """
        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)

        if not filter_docs:
            filter_docs = [{'_id': doc['_id']} for doc in docs]

        requests = [
            ReplaceOne(
                filter_docs[i],
                docs[i],
                upsert=upsert,
                collation=collation)
            for i in range(len(docs))
        ]

        return collection.bulk_write(requests, **kwargs)

def replace_many ( self , mongo_collection , docs , filter_docs = None , mongo_db = None , upsert = False , collation = None , ** kwargs ) : collection = self . get_collection ( mongo_collection , mongo_db = mongo_db ) if not filter_docs : filter_docs = [ { '_id' : doc [ '_id' ] } for doc in docs ] requests = [ ReplaceOne ( filter_docs [ i ] , docs [ i ] , upsert = upsert , collation = collation ) for i in range ( len ( docs ) ) ] return collection . bulk_write ( requests , ** kwargs )

Replaces many documents in a mongo collection .


def has_mail_attachment(self, name, mail_folder='INBOX', check_regex=False):
        """
        Checks the mail folder for mails containing attachments with the given name.

        :param name: The name of the attachment that will be searched for.
        :type name: str
        :param mail_folder: The mail folder where to look at.
        :type mail_folder: str
        :param check_regex: Checks the name for a regular expression.
        :type check_regex: bool
        :returns: True if there is an attachment with the given name and False if not.
        :rtype: bool
        """
        mail_attachments = self._retrieve_mails_attachments_by_name(name,
                                                                    mail_folder,
                                                                    check_regex,
                                                                    latest_only=True)
        return len(mail_attachments) > 0

def has_mail_attachment ( self , name , mail_folder = 'INBOX' , check_regex = False ) : mail_attachments = self . _retrieve_mails_attachments_by_name ( name , mail_folder , check_regex , latest_only = True ) return len ( mail_attachments ) > 0

Checks the mail folder for mails containing attachments with the given name .


def retrieve_mail_attachments(self,
                                  name,
                                  mail_folder='INBOX',
                                  check_regex=False,
                                  latest_only=False,
                                  not_found_mode='raise'):
        """
        Retrieves mail's attachments in the mail folder by its name.

        :param name: The name of the attachment that will be downloaded.
        :type name: str
        :param mail_folder: The mail folder where to look at.
        :type mail_folder: str
        :param check_regex: Checks the name for a regular expression.
        :type check_regex: bool
        :param latest_only: If set to True it will only retrieve
                            the first matched attachment.
        :type latest_only: bool
        :param not_found_mode: Specify what should happen if no attachment has been found.
                               Supported values are 'raise', 'warn' and 'ignore'.
                               If it is set to 'raise' it will raise an exception,
                               if set to 'warn' it will only print a warning and
                               if set to 'ignore' it won't notify you at all.
        :type not_found_mode: str
        :returns: a list of tuple each containing the attachment filename and its payload.
        :rtype: a list of tuple
        """
        mail_attachments = self._retrieve_mails_attachments_by_name(name,
                                                                    mail_folder,
                                                                    check_regex,
                                                                    latest_only)
        if not mail_attachments:
            self._handle_not_found_mode(not_found_mode)

        return mail_attachments

def retrieve_mail_attachments ( self , name , mail_folder = 'INBOX' , check_regex = False , latest_only = False , not_found_mode = 'raise' ) : mail_attachments = self . _retrieve_mails_attachments_by_name ( name , mail_folder , check_regex , latest_only ) if not mail_attachments : self . _handle_not_found_mode ( not_found_mode ) return mail_attachments

Retrieves mail s attachments in the mail folder by its name .


def download_mail_attachments(self,
                                  name,
                                  local_output_directory,
                                  mail_folder='INBOX',
                                  check_regex=False,
                                  latest_only=False,
                                  not_found_mode='raise'):
        """
        Downloads mail's attachments in the mail folder by its name to the local directory.

        :param name: The name of the attachment that will be downloaded.
        :type name: str
        :param local_output_directory: The output directory on the local machine
                                       where the files will be downloaded to.
        :type local_output_directory: str
        :param mail_folder: The mail folder where to look at.
        :type mail_folder: str
        :param check_regex: Checks the name for a regular expression.
        :type check_regex: bool
        :param latest_only: If set to True it will only download
                            the first matched attachment.
        :type latest_only: bool
        :param not_found_mode: Specify what should happen if no attachment has been found.
                               Supported values are 'raise', 'warn' and 'ignore'.
                               If it is set to 'raise' it will raise an exception,
                               if set to 'warn' it will only print a warning and
                               if set to 'ignore' it won't notify you at all.
        :type not_found_mode: str
        """
        mail_attachments = self._retrieve_mails_attachments_by_name(name,
                                                                    mail_folder,
                                                                    check_regex,
                                                                    latest_only)

        if not mail_attachments:
            self._handle_not_found_mode(not_found_mode)

        self._create_files(mail_attachments, local_output_directory)

def download_mail_attachments ( self , name , local_output_directory , mail_folder = 'INBOX' , check_regex = False , latest_only = False , not_found_mode = 'raise' ) : mail_attachments = self . _retrieve_mails_attachments_by_name ( name , mail_folder , check_regex , latest_only ) if not mail_attachments : self . _handle_not_found_mode ( not_found_mode ) self . _create_files ( mail_attachments , local_output_directory )

Downloads mail s attachments in the mail folder by its name to the local directory .


def get_attachments_by_name(self, name, check_regex, find_first=False):
        """
        Gets all attachments by name for the mail.

        :param name: The name of the attachment to look for.
        :type name: str
        :param check_regex: Checks the name for a regular expression.
        :type check_regex: bool
        :param find_first: If set to True it will only find the first match and then quit.
        :type find_first: bool
        :returns: a list of tuples each containing name and payload
                  where the attachments name matches the given name.
        :rtype: list of tuple
        """
        attachments = []

        for part in self.mail.walk():
            mail_part = MailPart(part)
            if mail_part.is_attachment():
                found_attachment = mail_part.has_matching_name(name) if check_regex \
                    else mail_part.has_equal_name(name)
                if found_attachment:
                    file_name, file_payload = mail_part.get_file()
                    self.log.info('Found attachment: {}'.format(file_name))
                    attachments.append((file_name, file_payload))
                    if find_first:
                        break

        return attachments

def
get_attachments_by_name
(
self
,
name
,
check_regex
,
find_first
=
False
)
:
attachments
=
[
]
for
part
in
self
.
mail
.
walk
(
)
:
mail_part
=
MailPart
(
part
)
if
mail_part
.
is_attachment
(
)
:
found_attachment
=
mail_part
.
has_matching_name
(
name
)
if
check_regex
else
mail_part
.
has_equal_name
(
name
)
if
found_attachment
:
file_name
,
file_payload
=
mail_part
.
get_file
(
)
self
.
log
.
info
(
'Found attachment: {}'
.
format
(
file_name
)
)
attachments
.
append
(
(
file_name
,
file_payload
)
)
if
find_first
:
break
return
attachments

Gets all attachments by name for the mail .


def get_file(self):
        """
        Gets the file including name and payload.

        :returns: the part's name and payload.
        :rtype: tuple
        """
        return self.part.get_filename(), self.part.get_payload(decode=True)

def get_file ( self ) : return self . part . get_filename ( ) , self . part . get_payload ( decode = True )

Gets the file including name and payload .


def put_records(self, records):
        """
        Write batch records to Kinesis Firehose
        """

        firehose_conn = self.get_conn()

        response = firehose_conn.put_record_batch(
            DeliveryStreamName=self.delivery_stream,
            Records=records
        )

        return response

def put_records ( self , records ) : firehose_conn = self . get_conn ( ) response = firehose_conn . put_record_batch ( DeliveryStreamName = self . delivery_stream , Records = records ) return response


def _get_dep_statuses(self, ti, session, dep_context):
        """
        Determines whether a task is ready to be rescheduled. Only tasks in
        NONE state with at least one row in task_reschedule table are
        handled by this dependency class, otherwise this dependency is
        considered as passed. This dependency fails if the latest reschedule
        request's reschedule date is still in future.
        """
        if dep_context.ignore_in_reschedule_period:
            yield self._passing_status(
                reason="The context specified that being in a reschedule period was "
                       "permitted.")
            return

        if ti.state not in self.RESCHEDULEABLE_STATES:
            yield self._passing_status(
                reason="The task instance is not in State_UP_FOR_RESCHEDULE or NONE state.")
            return

        task_reschedules = TaskReschedule.find_for_task_instance(task_instance=ti)
        if not task_reschedules:
            yield self._passing_status(
                reason="There is no reschedule request for this task instance.")
            return

        now = timezone.utcnow()
        next_reschedule_date = task_reschedules[-1].reschedule_date
        if now >= next_reschedule_date:
            yield self._passing_status(
                reason="Task instance id ready for reschedule.")
            return

        yield self._failing_status(
            reason="Task is not ready for reschedule yet but will be rescheduled "
                   "automatically. Current date is {0} and task will be rescheduled "
                   "at {1}.".format(now.isoformat(), next_reschedule_date.isoformat()))

Determines whether a task is ready to be rescheduled . Only tasks in NONE state with at least one row in task_reschedule table are handled by this dependency class otherwise this dependency is considered as passed . This dependency fails if the latest reschedule request s reschedule date is still in future .


def send_email(to, subject, html_content,
               files=None, dryrun=False, cc=None, bcc=None,
               mime_subtype='mixed', mime_charset='utf-8', **kwargs):
    """
    Send email using backend specified in EMAIL_BACKEND.
    """
    path, attr = configuration.conf.get('email', 'EMAIL_BACKEND').rsplit('.', 1)
    module = importlib.import_module(path)
    backend = getattr(module, attr)
    to = get_email_address_list(to)
    to = ", ".join(to)

    return backend(to, subject, html_content, files=files,
                   dryrun=dryrun, cc=cc, bcc=bcc,
                   mime_subtype=mime_subtype, mime_charset=mime_charset, **kwargs)

def
send_email
(
to
,
subject
,
html_content
,
files
=
,
dryrun
=
False
,
cc
=
,
bcc
=
,
mime_subtype
=
'mixed'
,
mime_charset
=
'utf-8'
,
**
kwargs
)
:
path
,
attr
=
configuration
.
conf
.
get
(
'email'
,
'EMAIL_BACKEND'
)
.
rsplit
(
'.'
,
1
)
module
=
importlib
.
import_module
(
path
)
backend
=
getattr
(
module
,
attr
)
to
=
get_email_address_list
(
to
)
to
=
", "
.
join
(
to
)
return
backend
(
to
,
subject
,
html_content
,
files
=
files
,
dryrun
=
dryrun
,
cc
=
cc
,
bcc
=
bcc
,
mime_subtype
=
mime_subtype
,
mime_charset
=
mime_charset
,
**
kwargs
)

Send email using backend specified in EMAIL_BACKEND .


def send_email_smtp(to, subject, html_content, files=None,
                    dryrun=False, cc=None, bcc=None,
                    mime_subtype='mixed', mime_charset='utf-8',
                    **kwargs):
    """
    Send an email with html content

    >>> send_email('test@example.com', 'foo', '<b>Foo</b> bar', ['/dev/null'], dryrun=True)
    """
    smtp_mail_from = configuration.conf.get('smtp', 'SMTP_MAIL_FROM')

    to = get_email_address_list(to)

    msg = MIMEMultipart(mime_subtype)
    msg['Subject'] = subject
    msg['From'] = smtp_mail_from
    msg['To'] = ", ".join(to)
    recipients = to
    if cc:
        cc = get_email_address_list(cc)
        msg['CC'] = ", ".join(cc)
        recipients = recipients + cc

    if bcc:
        # don't add bcc in header
        bcc = get_email_address_list(bcc)
        recipients = recipients + bcc

    msg['Date'] = formatdate(localtime=True)
    mime_text = MIMEText(html_content, 'html', mime_charset)
    msg.attach(mime_text)

    for fname in files or []:
        basename = os.path.basename(fname)
        with open(fname, "rb") as f:
            part = MIMEApplication(
                f.read(),
                Name=basename
            )
            part['Content-Disposition'] = 'attachment; filename="%s"' % basename
            part['Content-ID'] = '<%s>' % basename
            msg.attach(part)

    send_MIME_email(smtp_mail_from, recipients, msg, dryrun)

def
send_email_smtp
(
to
,
subject
,
html_content
,
files
=
,
dryrun
=
False
,
cc
=
,
bcc
=
,
mime_subtype
=
'mixed'
,
mime_charset
=
'utf-8'
,
**
kwargs
)
:
smtp_mail_from
=
configuration
.
conf
.
get
(
'smtp'
,
'SMTP_MAIL_FROM'
)
to
=
get_email_address_list
(
to
)
msg
=
MIMEMultipart
(
mime_subtype
)
msg
[
'Subject'
]
=
subject
msg
[
'From'
]
=
smtp_mail_from
msg
[
'To'
]
=
", "
.
join
(
to
)
recipients
=
to
if
cc
:
cc
=
get_email_address_list
(
cc
)
msg
[
'CC'
]
=
", "
.
join
(
cc
)
recipients
=
recipients
+
cc
if
bcc
:
bcc
=
get_email_address_list
(
bcc
)
recipients
=
recipients
+
bcc
msg
[
'Date'
]
=
formatdate
(
localtime
=
True
)
mime_text
=
MIMEText
(
html_content
,
'html'
,
mime_charset
)
msg
.
attach
(
mime_text
)
for
fname
in
files
or
[
]
:
basename
=
os
.
path
.
basename
(
fname
)
with
open
(
fname
,
"rb"
)
as
f
:
part
=
MIMEApplication
(
f
.
read
(
)
,
Name
=
basename
)
part
[
'Content-Disposition'
]
=
'attachment; filename="%s"'
%
basename
part
[
'Content-ID'
]
=
'<%s>'
%
basename
msg
.
attach
(
part
)
send_MIME_email
(
smtp_mail_from
,
recipients
,
msg
,
dryrun
)


def process_result_value(self, value, dialect):
        """
        Processes DateTimes from the DB making sure it is always
        returning UTC. Not using timezone.convert_to_utc as that
        converts to configured TIMEZONE while the DB might be
        running with some other setting. We assume UTC datetimes
        in the database.
        """
        if value is not None:
            if value.tzinfo is None:
                value = value.replace(tzinfo=utc)
            else:
                value = value.astimezone(utc)

        return value

def process_result_value ( self , value , dialect ) : if value is not None : if value . tzinfo is None : value = value . replace ( tzinfo = utc ) else : value = value . astimezone ( utc ) return value

Processes DateTimes from the DB making sure it is always returning UTC . Not using timezone . convert_to_utc as that converts to configured TIMEZONE while the DB might be running with some other setting . We assume UTC datetimes in the database .


def check_for_blob(self, container_name, blob_name, **kwargs):
        """
        Check if a blob exists on Azure Blob Storage.

        :param container_name: Name of the container.
        :type container_name: str
        :param blob_name: Name of the blob.
        :type blob_name: str
        :param kwargs: Optional keyword arguments that
            `BlockBlobService.exists()` takes.
        :type kwargs: object
        :return: True if the blob exists, False otherwise.
        :rtype: bool
        """
        return self.connection.exists(container_name, blob_name, **kwargs)

def check_for_blob ( self , container_name , blob_name , ** kwargs ) : return self . connection . exists ( container_name , blob_name , ** kwargs )

Check if a blob exists on Azure Blob Storage .


def check_for_prefix(self, container_name, prefix, **kwargs):
        """
        Check if a prefix exists on Azure Blob storage.

        :param container_name: Name of the container.
        :type container_name: str
        :param prefix: Prefix of the blob.
        :type prefix: str
        :param kwargs: Optional keyword arguments that
            `BlockBlobService.list_blobs()` takes.
        :type kwargs: object
        :return: True if blobs matching the prefix exist, False otherwise.
        :rtype: bool
        """
        matches = self.connection.list_blobs(container_name, prefix,
                                             num_results=1, **kwargs)
        return len(list(matches)) > 0

def check_for_prefix ( self , container_name , prefix , ** kwargs ) : matches = self . connection . list_blobs ( container_name , prefix , num_results = 1 , ** kwargs ) return len ( list ( matches ) ) > 0

Check if a prefix exists on Azure Blob storage .


def load_string(self, string_data, container_name, blob_name, **kwargs):
        """
        Upload a string to Azure Blob Storage.

        :param string_data: String to load.
        :type string_data: str
        :param container_name: Name of the container.
        :type container_name: str
        :param blob_name: Name of the blob.
        :type blob_name: str
        :param kwargs: Optional keyword arguments that
            `BlockBlobService.create_blob_from_text()` takes.
        :type kwargs: object
        """
        # Reorder the argument order from airflow.hooks.S3_hook.load_string.
        self.connection.create_blob_from_text(container_name, blob_name,
                                              string_data, **kwargs)

def load_string ( self , string_data , container_name , blob_name , ** kwargs ) : self . connection . create_blob_from_text ( container_name , blob_name , string_data , ** kwargs )


def read_file(self, container_name, blob_name, **kwargs):
        """
        Read a file from Azure Blob Storage and return as a string.

        :param container_name: Name of the container.
        :type container_name: str
        :param blob_name: Name of the blob.
        :type blob_name: str
        :param kwargs: Optional keyword arguments that
            `BlockBlobService.create_blob_from_path()` takes.
        :type kwargs: object
        """
        return self.connection.get_blob_to_text(container_name,
                                                blob_name,
                                                **kwargs).content

def read_file ( self , container_name , blob_name , ** kwargs ) : return self . connection . get_blob_to_text ( container_name , blob_name , ** kwargs ) . content

Read a file from Azure Blob Storage and return as a string .


def delete_file(self, container_name, blob_name, is_prefix=False,
                    ignore_if_missing=False, **kwargs):
        """
        Delete a file from Azure Blob Storage.

        :param container_name: Name of the container.
        :type container_name: str
        :param blob_name: Name of the blob.
        :type blob_name: str
        :param is_prefix: If blob_name is a prefix, delete all matching files
        :type is_prefix: bool
        :param ignore_if_missing: if True, then return success even if the
            blob does not exist.
        :type ignore_if_missing: bool
        :param kwargs: Optional keyword arguments that
            `BlockBlobService.create_blob_from_path()` takes.
        :type kwargs: object
        """

        if is_prefix:
            blobs_to_delete = [
                blob.name for blob in self.connection.list_blobs(
                    container_name, prefix=blob_name, **kwargs
                )
            ]
        elif self.check_for_blob(container_name, blob_name):
            blobs_to_delete = [blob_name]
        else:
            blobs_to_delete = []

        if not ignore_if_missing and len(blobs_to_delete) == 0:
            raise AirflowException('Blob(s) not found: {}'.format(blob_name))

        for blob_uri in blobs_to_delete:
            self.log.info("Deleting blob: " + blob_uri)
            self.connection.delete_blob(container_name,
                                        blob_uri,
                                        delete_snapshots='include',
                                        **kwargs)

def
delete_file
(
self
,
container_name
,
blob_name
,
is_prefix
=
False
,
ignore_if_missing
=
False
,
**
kwargs
)
:
if
is_prefix
:
blobs_to_delete
=
[
blob
.
name
for
blob
in
self
.
connection
.
list_blobs
(
container_name
,
prefix
=
blob_name
,
**
kwargs
)
]
elif
self
.
check_for_blob
(
container_name
,
blob_name
)
:
blobs_to_delete
=
[
blob_name
]
else
:
blobs_to_delete
=
[
]
if
not
ignore_if_missing
and
len
(
blobs_to_delete
)
==
0
:
raise
AirflowException
(
'Blob(s) not found: {}'
.
format
(
blob_name
)
)
for
blob_uri
in
blobs_to_delete
:
self
.
log
.
info
(
"Deleting blob: "
+
blob_uri
)
self
.
connection
.
delete_blob
(
container_name
,
blob_uri
,
delete_snapshots
=
'include'
,
**
kwargs
)


def mlsd(conn, path="", facts=None):
    """
    BACKPORT FROM PYTHON3 FTPLIB.

    List a directory in a standardized format by using MLSD
    command (RFC-3659). If path is omitted the current directory
    is assumed. "facts" is a list of strings representing the type
    of information desired (e.g. ["type", "size", "perm"]).

    Return a generator object yielding a tuple of two elements
    for every file found in path.
    First element is the file name, the second one is a dictionary
    including a variable number of "facts" depending on the server
    and whether "facts" argument has been provided.
    """
    facts = facts or []
    if facts:
        conn.sendcmd("OPTS MLST " + ";".join(facts) + ";")
    if path:
        cmd = "MLSD %s" % path
    else:
        cmd = "MLSD"
    lines = []
    conn.retrlines(cmd, lines.append)
    for line in lines:
        facts_found, _, name = line.rstrip(ftplib.CRLF).partition(' ')
        entry = {}
        for fact in facts_found[:-1].split(";"):
            key, _, value = fact.partition("=")
            entry[key.lower()] = value
        yield (name, entry)

def
mlsd
(
conn
,
path
=
""
,
facts
=
)
:
facts
=
facts
or
[
]
if
facts
:
conn
.
sendcmd
(
"OPTS MLST "
+
";"
.
join
(
facts
)
+
";"
)
if
path
:
cmd
=
"MLSD %s"
%
path
else
:
cmd
=
"MLSD"
lines
=
[
]
conn
.
retrlines
(
cmd
,
lines
.
append
)
for
line
in
lines
:
facts_found
,
_
,
name
=
line
.
rstrip
(
ftplib
.
CRLF
)
.
partition
(
' '
)
entry
=
{
}
for
fact
in
facts_found
[
:
1
]
.
split
(
";"
)
:
key
,
_
,
value
=
fact
.
partition
(
"="
)
entry
[
key
.
lower
(
)
]
=
value
yield
(
name
,
entry
)


def get_conn(self):
        """
        Returns a FTP connection object
        """
        if self.conn is None:
            params = self.get_connection(self.ftp_conn_id)
            pasv = params.extra_dejson.get("passive", True)
            self.conn = ftplib.FTP(params.host, params.login, params.password)
            self.conn.set_pasv(pasv)

        return self.conn

def get_conn ( self ) : if self . conn is None : params = self . get_connection ( self . ftp_conn_id ) pasv = params . extra_dejson . get ( "passive" , True ) self . conn = ftplib . FTP ( params . host , params . login , params . password ) self . conn . set_pasv ( pasv ) return self . conn


def list_directory(self, path, nlst=False):
        """
        Returns a list of files on the remote system.

        :param path: full path to the remote directory to list
        :type path: str
        """
        conn = self.get_conn()
        conn.cwd(path)

        files = conn.nlst()
        return files

def list_directory ( self , path , nlst = False ) : conn = self . get_conn ( ) conn . cwd ( path ) files = conn . nlst ( ) return files

Returns a list of files on the remote system .


def retrieve_file(
            self,
            remote_full_path,
            local_full_path_or_buffer,
            callback=None):
        """
        Transfers the remote file to a local location.

        If local_full_path_or_buffer is a string path, the file will be put
        at that location; if it is a file-like buffer, the file will
        be written to the buffer but not closed.

        :param remote_full_path: full path to the remote file
        :type remote_full_path: str
        :param local_full_path_or_buffer: full path to the local file or a
            file-like buffer
        :type local_full_path_or_buffer: str or file-like buffer
        :param callback: callback which is called each time a block of data
            is read. if you do not use a callback, these blocks will be written
            to the file or buffer passed in. if you do pass in a callback, note
            that writing to a file or buffer will need to be handled inside the
            callback.
            [default: output_handle.write()]
        :type callback: callable

        :Example::

            hook = FTPHook(ftp_conn_id='my_conn')

            remote_path = '/path/to/remote/file'
            local_path = '/path/to/local/file'

            # with a custom callback (in this case displaying progress on each read)
            def print_progress(percent_progress):
                self.log.info('Percent Downloaded: %s%%' % percent_progress)

            total_downloaded = 0
            total_file_size = hook.get_size(remote_path)
            output_handle = open(local_path, 'wb')
            def write_to_file_with_progress(data):
                total_downloaded += len(data)
                output_handle.write(data)
                percent_progress = (total_downloaded / total_file_size) * 100
                print_progress(percent_progress)
            hook.retrieve_file(remote_path, None, callback=write_to_file_with_progress)

            # without a custom callback data is written to the local_path
            hook.retrieve_file(remote_path, local_path)
        """
        conn = self.get_conn()

        is_path = isinstance(local_full_path_or_buffer, basestring)

        # without a callback, default to writing to a user-provided file or
        # file-like buffer
        if not callback:
            if is_path:
                output_handle = open(local_full_path_or_buffer, 'wb')
            else:
                output_handle = local_full_path_or_buffer
            callback = output_handle.write
        else:
            output_handle = None

        remote_path, remote_file_name = os.path.split(remote_full_path)
        conn.cwd(remote_path)
        self.log.info('Retrieving file from FTP: %s', remote_full_path)
        conn.retrbinary('RETR %s' % remote_file_name, callback)
        self.log.info('Finished retrieving file from FTP: %s', remote_full_path)

        if is_path and output_handle:
            output_handle.close()

def
retrieve_file
(
self
,
remote_full_path
,
local_full_path_or_buffer
,
callback
=
)
:
conn
=
self
.
get_conn
(
)
is_path
=
isinstance
(
local_full_path_or_buffer
,
basestring
)
if
not
callback
:
if
is_path
:
output_handle
=
open
(
local_full_path_or_buffer
,
'wb'
)
else
:
output_handle
=
local_full_path_or_buffer
callback
=
output_handle
.
write
else
:
output_handle
=
remote_path
,
remote_file_name
=
os
.
path
.
split
(
remote_full_path
)
conn
.
cwd
(
remote_path
)
self
.
log
.
info
(
'Retrieving file from FTP: %s'
,
remote_full_path
)
conn
.
retrbinary
(
'RETR %s'
%
remote_file_name
,
callback
)
self
.
log
.
info
(
'Finished retrieving file from FTP: %s'
,
remote_full_path
)
if
is_path
and
output_handle
:
output_handle
.
close
(
)

Transfers the remote file to a local location .


def store_file(self, remote_full_path, local_full_path_or_buffer):
        """
        Transfers a local file to the remote location.

        If local_full_path_or_buffer is a string path, the file will be read
        from that location; if it is a file-like buffer, the file will
        be read from the buffer but not closed.

        :param remote_full_path: full path to the remote file
        :type remote_full_path: str
        :param local_full_path_or_buffer: full path to the local file or a
            file-like buffer
        :type local_full_path_or_buffer: str or file-like buffer
        """
        conn = self.get_conn()

        is_path = isinstance(local_full_path_or_buffer, basestring)

        if is_path:
            input_handle = open(local_full_path_or_buffer, 'rb')
        else:
            input_handle = local_full_path_or_buffer
        remote_path, remote_file_name = os.path.split(remote_full_path)
        conn.cwd(remote_path)
        conn.storbinary('STOR %s' % remote_file_name, input_handle)

        if is_path:
            input_handle.close()

def
store_file
(
self
,
remote_full_path
,
local_full_path_or_buffer
)
:
conn
=
self
.
get_conn
(
)
is_path
=
isinstance
(
local_full_path_or_buffer
,
basestring
)
if
is_path
:
input_handle
=
open
(
local_full_path_or_buffer
,
'rb'
)
else
:
input_handle
=
local_full_path_or_buffer
remote_path
,
remote_file_name
=
os
.
path
.
split
(
remote_full_path
)
conn
.
cwd
(
remote_path
)
conn
.
storbinary
(
'STOR %s'
%
remote_file_name
,
input_handle
)
if
is_path
:
input_handle
.
close
(
)

Transfers a local file to the remote location .


def get_mod_time(self, path):
        """
        Returns a datetime object representing the last time the file was modified

        :param path: remote file path
        :type path: string
        """
        conn = self.get_conn()
        ftp_mdtm = conn.sendcmd('MDTM ' + path)
        time_val = ftp_mdtm[4:]
        # time_val optionally has microseconds
        try:
            return datetime.datetime.strptime(time_val, "%Y%m%d%H%M%S.%f")
        except ValueError:
            return datetime.datetime.strptime(time_val, '%Y%m%d%H%M%S')

def
get_mod_time
(
self
,
path
)
:
conn
=
self
.
get_conn
(
)
ftp_mdtm
=
conn
.
sendcmd
(
'MDTM '
+
path
)
time_val
=
ftp_mdtm
[
4
:
]
try
:
return
datetime
.
datetime
.
strptime
(
time_val
,
"%Y%m%d%H%M%S.%f"
)
except
ValueError
:
return
datetime
.
datetime
.
strptime
(
time_val
,
'%Y%m%d%H%M%S'
)


def execute(self, context):
        """
        Call the DiscordWebhookHook to post message
        """
        self.hook = DiscordWebhookHook(
            self.http_conn_id,
            self.webhook_endpoint,
            self.message,
            self.username,
            self.avatar_url,
            self.tts,
            self.proxy
        )
        self.hook.execute()

def execute ( self , context ) : self . hook = DiscordWebhookHook ( self . http_conn_id , self . webhook_endpoint , self . message , self . username , self . avatar_url , self . tts , self . proxy ) self . hook . execute ( )


def get_conn(self):
        """Return the FileService object."""
        conn = self.get_connection(self.conn_id)
        service_options = conn.extra_dejson
        return FileService(account_name=conn.login,
                           account_key=conn.password, **service_options)

def get_conn ( self ) : conn = self . get_connection ( self . conn_id ) service_options = conn . extra_dejson return FileService ( account_name = conn . login , account_key = conn . password , ** service_options )


def check_for_directory(self, share_name, directory_name, **kwargs):
        """
        Check if a directory exists on Azure File Share.

        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.exists()` takes.
        :type kwargs: object
        :return: True if the file exists, False otherwise.
        :rtype: bool
        """
        return self.connection.exists(share_name, directory_name,
                                      **kwargs)

def check_for_directory ( self , share_name , directory_name , ** kwargs ) : return self . connection . exists ( share_name , directory_name , ** kwargs )

Check if a directory exists on Azure File Share .


def check_for_file(self, share_name, directory_name, file_name, **kwargs):
        """
        Check if a file exists on Azure File Share.

        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param file_name: Name of the file.
        :type file_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.exists()` takes.
        :type kwargs: object
        :return: True if the file exists, False otherwise.
        :rtype: bool
        """
        return self.connection.exists(share_name, directory_name,
                                      file_name, **kwargs)

def check_for_file ( self , share_name , directory_name , file_name , ** kwargs ) : return self . connection . exists ( share_name , directory_name , file_name , ** kwargs )

Check if a file exists on Azure File Share .


def list_directories_and_files(self, share_name, directory_name=None, **kwargs):
        """
        Return the list of directories and files stored on a Azure File Share.

        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.list_directories_and_files()` takes.
        :type kwargs: object
        :return: A list of files and directories
        :rtype: list
        """
        return self.connection.list_directories_and_files(share_name,
                                                          directory_name,
                                                          **kwargs)

def list_directories_and_files ( self , share_name , directory_name = None , ** kwargs ) : return self . connection . list_directories_and_files ( share_name , directory_name , ** kwargs )

Return the list of directories and files stored on a Azure File Share .


def create_directory(self, share_name, directory_name, **kwargs):
        """
        Create a new directory on a Azure File Share.

        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.create_directory()` takes.
        :type kwargs: object
        :return: A list of files and directories
        :rtype: list
        """
        return self.connection.create_directory(share_name, directory_name, **kwargs)

def create_directory ( self , share_name , directory_name , ** kwargs ) : return self . connection . create_directory ( share_name , directory_name , ** kwargs )

Create a new directory on a Azure File Share .


def load_file(self, file_path, share_name, directory_name, file_name, **kwargs):
        """
        Upload a file to Azure File Share.

        :param file_path: Path to the file to load.
        :type file_path: str
        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param file_name: Name of the file.
        :type file_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.create_file_from_path()` takes.
        :type kwargs: object
        """
        self.connection.create_file_from_path(share_name, directory_name,
                                              file_name, file_path, **kwargs)

def load_file ( self , file_path , share_name , directory_name , file_name , ** kwargs ) : self . connection . create_file_from_path ( share_name , directory_name , file_name , file_path , ** kwargs )


def load_string(self, string_data, share_name, directory_name, file_name, **kwargs):
        """
        Upload a string to Azure File Share.

        :param string_data: String to load.
        :type string_data: str
        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param file_name: Name of the file.
        :type file_name: str
        :param kwargs: Optional keyword arguments that
            `FileService.create_file_from_text()` takes.
        :type kwargs: object
        """
        self.connection.create_file_from_text(share_name, directory_name,
                                              file_name, string_data, **kwargs)

def load_string ( self , string_data , share_name , directory_name , file_name , ** kwargs ) : self . connection . create_file_from_text ( share_name , directory_name , file_name , string_data , ** kwargs )


def load_stream(self, stream, share_name, directory_name, file_name, count, **kwargs):
        """
        Upload a stream to Azure File Share.

        :param stream: Opened file/stream to upload as the file content.
        :type stream: file-like
        :param share_name: Name of the share.
        :type share_name: str
        :param directory_name: Name of the directory.
        :type directory_name: str
        :param file_name: Name of the file.
        :type file_name: str
        :param count: Size of the stream in bytes
        :type count: int
        :param kwargs: Optional keyword arguments that
            `FileService.create_file_from_stream()` takes.
        :type kwargs: object
        """
        self.connection.create_file_from_stream(share_name, directory_name,
                                                file_name, stream, count, **kwargs)

def load_stream ( self , stream , share_name , directory_name , file_name , count , ** kwargs ) : self . connection . create_file_from_stream ( share_name , directory_name , file_name , stream , count , ** kwargs )


def get_conn(self):
        """
        Returns a Google Cloud Storage service object.
        """
        if not self._conn:
            self._conn = storage.Client(credentials=self._get_credentials())

        return self._conn

def get_conn ( self ) : if not self . _conn : self . _conn = storage . Client ( credentials = self . _get_credentials ( ) ) return self . _conn

Returns a Google Cloud Storage service object .


def copy(self, source_bucket, source_object, destination_bucket=None,
             destination_object=None):
        """
        Copies an object from a bucket to another, with renaming if requested.

        destination_bucket or destination_object can be omitted, in which case
        source bucket/object is used, but not both.

        :param source_bucket: The bucket of the object to copy from.
        :type source_bucket: str
        :param source_object: The object to copy.
        :type source_object: str
        :param destination_bucket: The destination of the object to copied to.
            Can be omitted; then the same bucket is used.
        :type destination_bucket: str
        :param destination_object: The (renamed) path of the object if given.
            Can be omitted; then the same name is used.
        :type destination_object: str
        """
        destination_bucket = destination_bucket or source_bucket
        destination_object = destination_object or source_object
        if source_bucket == destination_bucket and \
                source_object == destination_object:

            raise ValueError(
                'Either source/destination bucket or source/destination object '
                'must be different, not both the same: bucket=%s, object=%s' %
                (source_bucket, source_object))
        if not source_bucket or not source_object:
            raise ValueError('source_bucket and source_object cannot be empty.')

        client = self.get_conn()
        source_bucket = client.get_bucket(source_bucket)
        source_object = source_bucket.blob(source_object)
        destination_bucket = client.get_bucket(destination_bucket)
        destination_object = source_bucket.copy_blob(
            blob=source_object,
            destination_bucket=destination_bucket,
            new_name=destination_object)

        self.log.info('Object %s in bucket %s copied to object %s in bucket %s',
                      source_object.name, source_bucket.name,
                      destination_object.name, destination_bucket.name)

Copies an object from a bucket to another with renaming if requested .


def download(self, bucket_name, object_name, filename=None):
        """
        Get a file from Google Cloud Storage.

        :param bucket_name: The bucket to fetch from.
        :type bucket_name: str
        :param object_name: The object to fetch.
        :type object_name: str
        :param filename: If set, a local file path where the file should be written to.
        :type filename: str
        """
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name)
        blob = bucket.blob(blob_name=object_name)

        if filename:
            blob.download_to_filename(filename)
            self.log.info('File downloaded to %s', filename)

        return blob.download_as_string()

def
download
(
self
,
bucket_name
,
object_name
,
filename
=
)
:
client
=
self
.
get_conn
(
)
bucket
=
client
.
get_bucket
(
bucket_name
)
blob
=
bucket
.
blob
(
blob_name
=
object_name
)
if
filename
:
blob
.
download_to_filename
(
filename
)
self
.
log
.
info
(
'File downloaded to %s'
,
filename
)
return
blob
.
download_as_string
(
)


def upload(self, bucket_name, object_name, filename,
               mime_type='application/octet-stream', gzip=False):
        """
        Uploads a local file to Google Cloud Storage.

        :param bucket_name: The bucket to upload to.
        :type bucket_name: str
        :param object_name: The object name to set when uploading the local file.
        :type object_name: str
        :param filename: The local file path to the file to be uploaded.
        :type filename: str
        :param mime_type: The MIME type to set when uploading the file.
        :type mime_type: str
        :param gzip: Option to compress file for upload
        :type gzip: bool
        """

        if gzip:
            filename_gz = filename + '.gz'

            with open(filename, 'rb') as f_in:
                with gz.open(filename_gz, 'wb') as f_out:
                    shutil.copyfileobj(f_in, f_out)
                    filename = filename_gz

        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.blob(blob_name=object_name)
        blob.upload_from_filename(filename=filename,
                                  content_type=mime_type)

        if gzip:
            os.remove(filename)
        self.log.info('File %s uploaded to %s in %s bucket', filename, object_name, bucket_name)

def
upload
(
self
,
bucket_name
,
object_name
,
filename
,
mime_type
=
'application/octet-stream'
,
gzip
=
False
)
:
if
gzip
:
filename_gz
=
filename
+
'.gz'
with
open
(
filename
,
'rb'
)
as
f_in
:
with
gz
.
open
(
filename_gz
,
'wb'
)
as
f_out
:
shutil
.
copyfileobj
(
f_in
,
f_out
)
filename
=
filename_gz
client
=
self
.
get_conn
(
)
bucket
=
client
.
get_bucket
(
bucket_name
=
bucket_name
)
blob
=
bucket
.
blob
(
blob_name
=
object_name
)
blob
.
upload_from_filename
(
filename
=
filename
,
content_type
=
mime_type
)
if
gzip
:
os
.
remove
(
filename
)
self
.
log
.
info
(
'File %s uploaded to %s in %s bucket'
,
filename
,
object_name
,
bucket_name
)

Uploads a local file to Google Cloud Storage .


def exists(self, bucket_name, object_name):
        """
        Checks for the existence of a file in Google Cloud Storage.

        :param bucket_name: The Google cloud storage bucket where the object is.
        :type bucket_name: str
        :param object_name: The name of the blob_name to check in the Google cloud
            storage bucket.
        :type object_name: str
        """
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.blob(blob_name=object_name)
        return blob.exists()

def exists ( self , bucket_name , object_name ) : client = self . get_conn ( ) bucket = client . get_bucket ( bucket_name = bucket_name ) blob = bucket . blob ( blob_name = object_name ) return blob . exists ( )

Checks for the existence of a file in Google Cloud Storage .


def is_updated_after(self, bucket_name, object_name, ts):
        """
        Checks if an blob_name is updated in Google Cloud Storage.

        :param bucket_name: The Google cloud storage bucket where the object is.
        :type bucket_name: str
        :param object_name: The name of the object to check in the Google cloud
            storage bucket.
        :type object_name: str
        :param ts: The timestamp to check against.
        :type ts: datetime.datetime
        """
        client = self.get_conn()
        bucket = storage.Bucket(client=client, name=bucket_name)
        blob = bucket.get_blob(blob_name=object_name)
        blob.reload()

        blob_update_time = blob.updated

        if blob_update_time is not None:
            import dateutil.tz

            if not ts.tzinfo:
                ts = ts.replace(tzinfo=dateutil.tz.tzutc())

            self.log.info("Verify object date: %s > %s", blob_update_time, ts)

            if blob_update_time > ts:
                return True

        return False

def
is_updated_after
(
self
,
bucket_name
,
object_name
,
ts
)
:
client
=
self
.
get_conn
(
)
bucket
=
storage
.
Bucket
(
client
=
client
,
name
=
bucket_name
)
blob
=
bucket
.
get_blob
(
blob_name
=
object_name
)
blob
.
reload
(
)
blob_update_time
=
blob
.
updated
if
blob_update_time
is
not
:
import
dateutil
.
tz
if
not
ts
.
tzinfo
:
ts
=
ts
.
replace
(
tzinfo
=
dateutil
.
tz
.
tzutc
(
)
)
self
.
log
.
info
(
"Verify object date: %s > %s"
,
blob_update_time
,
ts
)
if
blob_update_time
>
ts
:
return
True
return
False

Checks if an blob_name is updated in Google Cloud Storage .


def delete(self, bucket_name, object_name):
        """
        Deletes an object from the bucket.

        :param bucket_name: name of the bucket, where the object resides
        :type bucket_name: str
        :param object_name: name of the object to delete
        :type object_name: str
        """
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.blob(blob_name=object_name)
        blob.delete()

        self.log.info('Blob %s deleted.', object_name)

def
delete
(
self
,
bucket_name
,
object_name
)
:
client
=
self
.
get_conn
(
)
bucket
=
client
.
get_bucket
(
bucket_name
=
bucket_name
)
blob
=
bucket
.
blob
(
blob_name
=
object_name
)
blob
.
delete
(
)
self
.
log
.
info
(
'Blob %s deleted.'
,
object_name
)


def list(self, bucket_name, versions=None, max_results=None, prefix=None, delimiter=None):
        """
        List all objects from the bucket with the give string prefix in name

        :param bucket_name: bucket name
        :type bucket_name: str
        :param versions: if true, list all versions of the objects
        :type versions: bool
        :param max_results: max count of items to return in a single page of responses
        :type max_results: int
        :param prefix: prefix string which filters objects whose name begin with
            this prefix
        :type prefix: str
        :param delimiter: filters objects based on the delimiter (for e.g '.csv')
        :type delimiter: str
        :return: a stream of object names matching the filtering criteria
        """
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)

        ids = []
        pageToken = None
        while True:
            blobs = bucket.list_blobs(
                max_results=max_results,
                page_token=pageToken,
                prefix=prefix,
                delimiter=delimiter,
                versions=versions
            )

            blob_names = []
            for blob in blobs:
                blob_names.append(blob.name)

            prefixes = blobs.prefixes
            if prefixes:
                ids += list(prefixes)
            else:
                ids += blob_names

            pageToken = blobs.next_page_token
            if pageToken is None:
                # empty next page token
                break
        return ids

def list ( self , bucket_name , versions = None , max_results = None , prefix = None , delimiter = None ) : client = self . get_conn ( ) bucket = client . get_bucket ( bucket_name = bucket_name ) ids = [ ] pageToken = None while True : blobs = bucket . list_blobs ( max_results = max_results , page_token = pageToken , prefix = prefix , delimiter = delimiter , versions = versions ) blob_names = [ ] for blob in blobs : blob_names . append ( blob . name ) prefixes = blobs . prefixes if prefixes : ids += list ( prefixes ) else : ids += blob_names pageToken = blobs . next_page_token if pageToken is None : break return ids


def get_size(self, bucket_name, object_name):
        """
        Gets the size of a file in Google Cloud Storage.

        :param bucket_name: The Google cloud storage bucket where the blob_name is.
        :type bucket_name: str
        :param object_name: The name of the object to check in the Google
            cloud storage bucket_name.
        :type object_name: str

        """
        self.log.info('Checking the file size of object: %s in bucket_name: %s',
                      object_name,
                      bucket_name)
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.get_blob(blob_name=object_name)
        blob.reload()
        blob_size = blob.size
        self.log.info('The file size of %s is %s bytes.', object_name, blob_size)
        return blob_size

Gets the size of a file in Google Cloud Storage .


def get_crc32c(self, bucket_name, object_name):
        """
        Gets the CRC32c checksum of an object in Google Cloud Storage.

        :param bucket_name: The Google cloud storage bucket where the blob_name is.
        :type bucket_name: str
        :param object_name: The name of the object to check in the Google cloud
            storage bucket_name.
        :type object_name: str
        """
        self.log.info('Retrieving the crc32c checksum of '
                      'object_name: %s in bucket_name: %s', object_name, bucket_name)
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.get_blob(blob_name=object_name)
        blob.reload()
        blob_crc32c = blob.crc32c
        self.log.info('The crc32c checksum of %s is %s', object_name, blob_crc32c)
        return blob_crc32c

def
get_crc32c
(
self
,
bucket_name
,
object_name
)
:
self
.
log
.
info
(
'Retrieving the crc32c checksum of '
'object_name: %s in bucket_name: %s'
,
object_name
,
bucket_name
)
client
=
self
.
get_conn
(
)
bucket
=
client
.
get_bucket
(
bucket_name
=
bucket_name
)
blob
=
bucket
.
get_blob
(
blob_name
=
object_name
)
blob
.
reload
(
)
blob_crc32c
=
blob
.
crc32c
self
.
log
.
info
(
'The crc32c checksum of %s is %s'
,
object_name
,
blob_crc32c
)
return
blob_crc32c

Gets the CRC32c checksum of an object in Google Cloud Storage .


def get_md5hash(self, bucket_name, object_name):
        """
        Gets the MD5 hash of an object in Google Cloud Storage.

        :param bucket_name: The Google cloud storage bucket where the blob_name is.
        :type bucket_name: str
        :param object_name: The name of the object to check in the Google cloud
            storage bucket_name.
        :type object_name: str
        """
        self.log.info('Retrieving the MD5 hash of '
                      'object: %s in bucket: %s', object_name, bucket_name)
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name=bucket_name)
        blob = bucket.get_blob(blob_name=object_name)
        blob.reload()
        blob_md5hash = blob.md5_hash
        self.log.info('The md5Hash of %s is %s', object_name, blob_md5hash)
        return blob_md5hash

def
get_md5hash
(
self
,
bucket_name
,
object_name
)
:
self
.
log
.
info
(
'Retrieving the MD5 hash of '
'object: %s in bucket: %s'
,
object_name
,
bucket_name
)
client
=
self
.
get_conn
(
)
bucket
=
client
.
get_bucket
(
bucket_name
=
bucket_name
)
blob
=
bucket
.
get_blob
(
blob_name
=
object_name
)
blob
.
reload
(
)
blob_md5hash
=
blob
.
md5_hash
self
.
log
.
info
(
'The md5Hash of %s is %s'
,
object_name
,
blob_md5hash
)
return
blob_md5hash

Gets the MD5 hash of an object in Google Cloud Storage .


def create_bucket(self,
                      bucket_name,
                      resource=None,
                      storage_class='MULTI_REGIONAL',
                      location='US',
                      project_id=None,
                      labels=None
                      ):
        """
        Creates a new bucket. Google Cloud Storage uses a flat namespace, so
        you can't create a bucket with a name that is already in use.

        .. seealso::
            For more information, see Bucket Naming Guidelines:
            https://cloud.google.com/storage/docs/bucketnaming.html#requirements

        :param bucket_name: The name of the bucket.
        :type bucket_name: str
        :param resource: An optional dict with parameters for creating the bucket.
            For information on available parameters, see Cloud Storage API doc:
            https://cloud.google.com/storage/docs/json_api/v1/buckets/insert
        :type resource: dict
        :param storage_class: This defines how objects in the bucket are stored
            and determines the SLA and the cost of storage. Values include

            - ``MULTI_REGIONAL``
            - ``REGIONAL``
            - ``STANDARD``
            - ``NEARLINE``
            - ``COLDLINE``.

            If this value is not specified when the bucket is
            created, it will default to STANDARD.
        :type storage_class: str
        :param location: The location of the bucket.
            Object data for objects in the bucket resides in physical storage
            within this region. Defaults to US.

            .. seealso::
                https://developers.google.com/storage/docs/bucket-locations

        :type location: str
        :param project_id: The ID of the GCP Project.
        :type project_id: str
        :param labels: User-provided labels, in key/value pairs.
        :type labels: dict
        :return: If successful, it returns the ``id`` of the bucket.
        """

        self.log.info('Creating Bucket: %s; Location: %s; Storage Class: %s',
                      bucket_name, location, storage_class)

        client = self.get_conn()
        bucket = client.bucket(bucket_name=bucket_name)
        bucket_resource = resource or {}

        for item in bucket_resource:
            if item != "name":
                bucket._patch_property(name=item, value=resource[item])

        bucket.storage_class = storage_class
        bucket.labels = labels or {}
        bucket.create(project=project_id, location=location)
        return bucket.id

Creates a new bucket . Google Cloud Storage uses a flat namespace so you can t create a bucket with a name that is already in use .


def compose(self, bucket_name, source_objects, destination_object):
        """
        Composes a list of existing object into a new object in the same storage bucket_name

        Currently it only supports up to 32 objects that can be concatenated
        in a single operation

        https://cloud.google.com/storage/docs/json_api/v1/objects/compose

        :param bucket_name: The name of the bucket containing the source objects.
            This is also the same bucket to store the composed destination object.
        :type bucket_name: str
        :param source_objects: The list of source objects that will be composed
            into a single object.
        :type source_objects: list
        :param destination_object: The path of the object if given.
        :type destination_object: str
        """

        if not source_objects or not len(source_objects):
            raise ValueError('source_objects cannot be empty.')

        if not bucket_name or not destination_object:
            raise ValueError('bucket_name and destination_object cannot be empty.')

        self.log.info("Composing %s to %s in the bucket %s",
                      source_objects, destination_object, bucket_name)
        client = self.get_conn()
        bucket = client.get_bucket(bucket_name)
        destination_blob = bucket.blob(destination_object)
        destination_blob.compose(
            sources=[
                bucket.blob(blob_name=source_object) for source_object in source_objects
            ])

        self.log.info("Completed successfully.")


def secondary_training_status_changed(current_job_description, prev_job_description):
    """
    Returns true if training job's secondary status message has changed.

    :param current_job_description: Current job description, returned from DescribeTrainingJob call.
    :type current_job_description: dict
    :param prev_job_description: Previous job description, returned from DescribeTrainingJob call.
    :type prev_job_description: dict

    :return: Whether the secondary status message of a training job changed or not.
    """
    current_secondary_status_transitions = current_job_description.get('SecondaryStatusTransitions')
    if current_secondary_status_transitions is None or len(current_secondary_status_transitions) == 0:
        return False

    prev_job_secondary_status_transitions = prev_job_description.get('SecondaryStatusTransitions') \
        if prev_job_description is not None else None

    last_message = prev_job_secondary_status_transitions[-1]['StatusMessage'] \
        if prev_job_secondary_status_transitions is not None \
        and len(prev_job_secondary_status_transitions) > 0 else ''

    message = current_job_description['SecondaryStatusTransitions'][-1]['StatusMessage']

    return message != last_message

def secondary_training_status_changed ( current_job_description , prev_job_description ) : current_secondary_status_transitions = current_job_description . get ( 'SecondaryStatusTransitions' ) if current_secondary_status_transitions is None or len ( current_secondary_status_transitions ) == 0 : return False prev_job_secondary_status_transitions = prev_job_description . get ( 'SecondaryStatusTransitions' ) if prev_job_description is not None else None last_message = prev_job_secondary_status_transitions [ - 1 ] [ 'StatusMessage' ] if prev_job_secondary_status_transitions is not None and len ( prev_job_secondary_status_transitions ) > 0 else '' message = current_job_description [ 'SecondaryStatusTransitions' ] [ - 1 ] [ 'StatusMessage' ] return message != last_message

Returns true if training job s secondary status message has changed .


def secondary_training_status_message(job_description, prev_description):
    """
    Returns a string contains start time and the secondary training job status message.

    :param job_description: Returned response from DescribeTrainingJob call
    :type job_description: dict
    :param prev_description: Previous job description from DescribeTrainingJob call
    :type prev_description: dict

    :return: Job status string to be printed.
    """

    if job_description is None or job_description.get('SecondaryStatusTransitions') is None\
            or len(job_description.get('SecondaryStatusTransitions')) == 0:
        return ''

    prev_description_secondary_transitions = prev_description.get('SecondaryStatusTransitions')\
        if prev_description is not None else None
    prev_transitions_num = len(prev_description['SecondaryStatusTransitions'])\
        if prev_description_secondary_transitions is not None else 0
    current_transitions = job_description['SecondaryStatusTransitions']

    transitions_to_print = current_transitions[-1:] if len(current_transitions) == prev_transitions_num else \
        current_transitions[prev_transitions_num - len(current_transitions):]

    status_strs = []
    for transition in transitions_to_print:
        message = transition['StatusMessage']
        time_str = timezone.convert_to_utc(job_description['LastModifiedTime']).strftime('%Y-%m-%d %H:%M:%S')
        status_strs.append('{} {} - {}'.format(time_str, transition['Status'], message))

    return '\n'.join(status_strs)

def
secondary_training_status_message
(
job_description
,
prev_description
)
:
if
job_description
is
or
job_description
.
get
(
'SecondaryStatusTransitions'
)
is
or
len
(
job_description
.
get
(
'SecondaryStatusTransitions'
)
)
==
0
:
return
''
prev_description_secondary_transitions
=
prev_description
.
get
(
'SecondaryStatusTransitions'
)
if
prev_description
is
not
else
prev_transitions_num
=
len
(
prev_description
[
'SecondaryStatusTransitions'
]
)
if
prev_description_secondary_transitions
is
not
else
0
current_transitions
=
job_description
[
'SecondaryStatusTransitions'
]
transitions_to_print
=
current_transitions
[
1
:
]
if
len
(
current_transitions
)
==
prev_transitions_num
else
current_transitions
[
prev_transitions_num
len
(
current_transitions
)
:
]
status_strs
=
[
]
for
transition
in
transitions_to_print
:
message
=
transition
[
'StatusMessage'
]
time_str
=
timezone
.
convert_to_utc
(
job_description
[
'LastModifiedTime'
]
)
.
strftime
(
'%Y-%m-%d %H:%M:%S'
)
status_strs
.
append
(
'{} {} - {}'
.
format
(
time_str
,
transition
[
'Status'
]
,
message
)
)
return
'\n'
.
join
(
status_strs
)

Returns a string contains start time and the secondary training job status message .


def tar_and_s3_upload(self, path, key, bucket):
        """
        Tar the local file or directory and upload to s3

        :param path: local file or directory
        :type path: str
        :param key: s3 key
        :type key: str
        :param bucket: s3 bucket
        :type bucket: str
        :return: None
        """
        with tempfile.TemporaryFile() as temp_file:
            if os.path.isdir(path):
                files = [os.path.join(path, name) for name in os.listdir(path)]
            else:
                files = [path]
            with tarfile.open(mode='w:gz', fileobj=temp_file) as tar_file:
                for f in files:
                    tar_file.add(f, arcname=os.path.basename(f))
            temp_file.seek(0)
            self.s3_hook.load_file_obj(temp_file, key, bucket, replace=True)

def tar_and_s3_upload ( self , path , key , bucket ) : with tempfile . TemporaryFile ( ) as temp_file : if os . path . isdir ( path ) : files = [ os . path . join ( path , name ) for name in os . listdir ( path ) ] else : files = [ path ] with tarfile . open ( mode = 'w:gz' , fileobj = temp_file ) as tar_file : for f in files : tar_file . add ( f , arcname = os . path . basename ( f ) ) temp_file . seek ( 0 ) self . s3_hook . load_file_obj ( temp_file , key , bucket , replace = True )


def configure_s3_resources(self, config):
        """
        Extract the S3 operations from the configuration and execute them.

        :param config: config of SageMaker operation
        :type config: dict
        :rtype: dict
        """
        s3_operations = config.pop('S3Operations', None)

        if s3_operations is not None:
            create_bucket_ops = s3_operations.get('S3CreateBucket', [])
            upload_ops = s3_operations.get('S3Upload', [])
            for op in create_bucket_ops:
                self.s3_hook.create_bucket(bucket_name=op['Bucket'])
            for op in upload_ops:
                if op['Tar']:
                    self.tar_and_s3_upload(op['Path'], op['Key'],
                                           op['Bucket'])
                else:
                    self.s3_hook.load_file(op['Path'], op['Key'],
                                           op['Bucket'])

def configure_s3_resources ( self , config ) : s3_operations = config . pop ( 'S3Operations' , None ) if s3_operations is not None : create_bucket_ops = s3_operations . get ( 'S3CreateBucket' , [ ] ) upload_ops = s3_operations . get ( 'S3Upload' , [ ] ) for op in create_bucket_ops : self . s3_hook . create_bucket ( bucket_name = op [ 'Bucket' ] ) for op in upload_ops : if op [ 'Tar' ] : self . tar_and_s3_upload ( op [ 'Path' ] , op [ 'Key' ] , op [ 'Bucket' ] ) else : self . s3_hook . load_file ( op [ 'Path' ] , op [ 'Key' ] , op [ 'Bucket' ] )

Extract the S3 operations from the configuration and execute them .


def check_s3_url(self, s3url):
        """
        Check if an S3 URL exists

        :param s3url: S3 url
        :type s3url: str
        :rtype: bool
        """
        bucket, key = S3Hook.parse_s3_url(s3url)
        if not self.s3_hook.check_for_bucket(bucket_name=bucket):
            raise AirflowException(
                "The input S3 Bucket {} does not exist ".format(bucket))
        if key and not self.s3_hook.check_for_key(key=key, bucket_name=bucket)\
           and not self.s3_hook.check_for_prefix(
                prefix=key, bucket_name=bucket, delimiter='/'):
            # check if s3 key exists in the case user provides a single file
            # or if s3 prefix exists in the case user provides multiple files in
            # a prefix
            raise AirflowException("The input S3 Key "
                                   "or Prefix {} does not exist in the Bucket {}"
                                   .format(s3url, bucket))
        return True


def get_log_conn(self):
        """
        Establish an AWS connection for retrieving logs during training

        :rtype: CloudWatchLogs.Client
        """
        config = botocore.config.Config(retries={'max_attempts': 15})
        return self.get_client_type('logs', config=config)

def get_log_conn ( self ) : config = botocore . config . Config ( retries = { 'max_attempts' : 15 } ) return self . get_client_type ( 'logs' , config = config )


def create_training_job(self, config, wait_for_completion=True, print_log=True,
                            check_interval=30, max_ingestion_time=None):
        """
        Create a training job

        :param config: the config for training
        :type config: dict
        :param wait_for_completion: if the program should keep running until job finishes
        :type wait_for_completion: bool
        :param check_interval: the time interval in seconds which the operator
            will check the status of any SageMaker job
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :return: A response to training job creation
        """

        self.check_training_config(config)

        response = self.get_conn().create_training_job(**config)
        if print_log:
            self.check_training_status_with_log(config['TrainingJobName'],
                                                self.non_terminal_states,
                                                self.failed_states,
                                                wait_for_completion,
                                                check_interval, max_ingestion_time
                                                )
        elif wait_for_completion:
            describe_response = self.check_status(config['TrainingJobName'],
                                                  'TrainingJobStatus',
                                                  self.describe_training_job,
                                                  check_interval, max_ingestion_time
                                                  )

            billable_time = \
                (describe_response['TrainingEndTime'] - describe_response['TrainingStartTime']) * \
                describe_response['ResourceConfig']['InstanceCount']
            self.log.info('Billable seconds:{}'.format(int(billable_time.total_seconds()) + 1))

        return response

def
create_training_job
(
self
,
config
,
wait_for_completion
=
True
,
print_log
=
True
,
check_interval
=
30
,
max_ingestion_time
=
)
:
self
.
check_training_config
(
config
)
response
=
self
.
get_conn
(
)
.
create_training_job
(
**
config
)
if
print_log
:
self
.
check_training_status_with_log
(
config
[
'TrainingJobName'
]
,
self
.
non_terminal_states
,
self
.
failed_states
,
wait_for_completion
,
check_interval
,
max_ingestion_time
)
elif
wait_for_completion
:
describe_response
=
self
.
check_status
(
config
[
'TrainingJobName'
]
,
'TrainingJobStatus'
,
self
.
describe_training_job
,
check_interval
,
max_ingestion_time
)
billable_time
=
(
describe_response
[
'TrainingEndTime'
]
describe_response
[
'TrainingStartTime'
]
)
*
describe_response
[
'ResourceConfig'
]
[
'InstanceCount'
]
self
.
log
.
info
(
'Billable seconds:{}'
.
format
(
int
(
billable_time
.
total_seconds
(
)
)
+
1
)
)
return
response


def create_tuning_job(self, config, wait_for_completion=True,
                          check_interval=30, max_ingestion_time=None):
        """
        Create a tuning job

        :param config: the config for tuning
        :type config: dict
        :param wait_for_completion: if the program should keep running until job finishes
        :type wait_for_completion: bool
        :param check_interval: the time interval in seconds which the operator
            will check the status of any SageMaker job
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :return: A response to tuning job creation
        """

        self.check_tuning_config(config)

        response = self.get_conn().create_hyper_parameter_tuning_job(**config)
        if wait_for_completion:
            self.check_status(config['HyperParameterTuningJobName'],
                              'HyperParameterTuningJobStatus',
                              self.describe_tuning_job,
                              check_interval, max_ingestion_time
                              )
        return response

def create_tuning_job ( self , config , wait_for_completion = True , check_interval = 30 , max_ingestion_time = None ) : self . check_tuning_config ( config ) response = self . get_conn ( ) . create_hyper_parameter_tuning_job ( ** config ) if wait_for_completion : self . check_status ( config [ 'HyperParameterTuningJobName' ] , 'HyperParameterTuningJobStatus' , self . describe_tuning_job , check_interval , max_ingestion_time ) return response


def create_transform_job(self, config, wait_for_completion=True,
                             check_interval=30, max_ingestion_time=None):
        """
        Create a transform job

        :param config: the config for transform job
        :type config: dict
        :param wait_for_completion: if the program should keep running until job finishes
        :type wait_for_completion: bool
        :param check_interval: the time interval in seconds which the operator
            will check the status of any SageMaker job
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :return: A response to transform job creation
        """

        self.check_s3_url(config['TransformInput']['DataSource']['S3DataSource']['S3Uri'])

        response = self.get_conn().create_transform_job(**config)
        if wait_for_completion:
            self.check_status(config['TransformJobName'],
                              'TransformJobStatus',
                              self.describe_transform_job,
                              check_interval, max_ingestion_time
                              )
        return response

def create_transform_job ( self , config , wait_for_completion = True , check_interval = 30 , max_ingestion_time = None ) : self . check_s3_url ( config [ 'TransformInput' ] [ 'DataSource' ] [ 'S3DataSource' ] [ 'S3Uri' ] ) response = self . get_conn ( ) . create_transform_job ( ** config ) if wait_for_completion : self . check_status ( config [ 'TransformJobName' ] , 'TransformJobStatus' , self . describe_transform_job , check_interval , max_ingestion_time ) return response


def create_endpoint(self, config, wait_for_completion=True,
                        check_interval=30, max_ingestion_time=None):
        """
        Create an endpoint

        :param config: the config for endpoint
        :type config: dict
        :param wait_for_completion: if the program should keep running until job finishes
        :type wait_for_completion: bool
        :param check_interval: the time interval in seconds which the operator
            will check the status of any SageMaker job
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :return: A response to endpoint creation
        """

        response = self.get_conn().create_endpoint(**config)
        if wait_for_completion:
            self.check_status(config['EndpointName'],
                              'EndpointStatus',
                              self.describe_endpoint,
                              check_interval, max_ingestion_time,
                              non_terminal_states=self.endpoint_non_terminal_states
                              )
        return response

def create_endpoint ( self , config , wait_for_completion = True , check_interval = 30 , max_ingestion_time = None ) : response = self . get_conn ( ) . create_endpoint ( ** config ) if wait_for_completion : self . check_status ( config [ 'EndpointName' ] , 'EndpointStatus' , self . describe_endpoint , check_interval , max_ingestion_time , non_terminal_states = self . endpoint_non_terminal_states ) return response


def describe_training_job_with_log(self, job_name, positions, stream_names,
                                       instance_count, state, last_description,
                                       last_describe_job_call):
        """
        Return the training job info associated with job_name and print CloudWatch logs
        """
        log_group = '/aws/sagemaker/TrainingJobs'

        if len(stream_names) < instance_count:
            # Log streams are created whenever a container starts writing to stdout/err, so this list
            # may be dynamic until we have a stream for every instance.
            logs_conn = self.get_log_conn()
            try:
                streams = logs_conn.describe_log_streams(
                    logGroupName=log_group,
                    logStreamNamePrefix=job_name + '/',
                    orderBy='LogStreamName',
                    limit=instance_count
                )
                stream_names = [s['logStreamName'] for s in streams['logStreams']]
                positions.update([(s, Position(timestamp=0, skip=0))
                                  for s in stream_names if s not in positions])
            except logs_conn.exceptions.ResourceNotFoundException:
                # On the very first training job run on an account, there's no log group until
                # the container starts logging, so ignore any errors thrown about that
                pass

        if len(stream_names) > 0:
            for idx, event in self.multi_stream_iter(log_group, stream_names, positions):
                self.log.info(event['message'])
                ts, count = positions[stream_names[idx]]
                if event['timestamp'] == ts:
                    positions[stream_names[idx]] = Position(timestamp=ts, skip=count + 1)
                else:
                    positions[stream_names[idx]] = Position(timestamp=event['timestamp'], skip=1)

        if state == LogState.COMPLETE:
            return state, last_description, last_describe_job_call

        if state == LogState.JOB_COMPLETE:
            state = LogState.COMPLETE
        elif time.time() - last_describe_job_call >= 30:
            description = self.describe_training_job(job_name)
            last_describe_job_call = time.time()

            if secondary_training_status_changed(description, last_description):
                self.log.info(secondary_training_status_message(description, last_description))
                last_description = description

            status = description['TrainingJobStatus']

            if status not in self.non_terminal_states:
                state = LogState.JOB_COMPLETE
        return state, last_description, last_describe_job_call

def describe_training_job_with_log ( self , job_name , positions , stream_names , instance_count , state , last_description , last_describe_job_call ) : log_group = '/aws/sagemaker/TrainingJobs' if len ( stream_names ) < instance_count : logs_conn = self . get_log_conn ( ) try : streams = logs_conn . describe_log_streams ( logGroupName = log_group , logStreamNamePrefix = job_name + '/' , orderBy = 'LogStreamName' , limit = instance_count ) stream_names = [ s [ 'logStreamName' ] for s in streams [ 'logStreams' ] ] positions . update ( [ ( s , Position ( timestamp = 0 , skip = 0 ) ) for s in stream_names if s not in positions ] ) except logs_conn . exceptions . ResourceNotFoundException : pass if len ( stream_names ) > 0 : for idx , event in self . multi_stream_iter ( log_group , stream_names , positions ) : self . log . info ( event [ 'message' ] ) ts , count = positions [ stream_names [ idx ] ] if event [ 'timestamp' ] == ts : positions [ stream_names [ idx ] ] = Position ( timestamp = ts , skip = count + 1 ) else : positions [ stream_names [ idx ] ] = Position ( timestamp = event [ 'timestamp' ] , skip = 1 ) if state == LogState . COMPLETE : return state , last_description , last_describe_job_call if state == LogState . JOB_COMPLETE : state = LogState . COMPLETE elif time . time ( ) - last_describe_job_call >= 30 : description = self . describe_training_job ( job_name ) last_describe_job_call = time . time ( ) if secondary_training_status_changed ( description , last_description ) : self . log . info ( secondary_training_status_message ( description , last_description ) ) last_description = description status = description [ 'TrainingJobStatus' ] if status not in self . non_terminal_states : state = LogState . JOB_COMPLETE return state , last_description , last_describe_job_call


def check_status(self, job_name, key,
                     describe_function, check_interval,
                     max_ingestion_time,
                     non_terminal_states=None):
        """
        Check status of a SageMaker job

        :param job_name: name of the job to check status
        :type job_name: str
        :param key: the key of the response dict
            that points to the state
        :type key: str
        :param describe_function: the function used to retrieve the status
        :type describe_function: python callable
        :param args: the arguments for the function
        :param check_interval: the time interval in seconds which the operator
            will check the status of any SageMaker job
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :param non_terminal_states: the set of nonterminal states
        :type non_terminal_states: set
        :return: response of describe call after job is done
        """
        if not non_terminal_states:
            non_terminal_states = self.non_terminal_states

        sec = 0
        running = True

        while running:
            time.sleep(check_interval)
            sec = sec + check_interval

            try:
                response = describe_function(job_name)
                status = response[key]
                self.log.info('Job still running for %s seconds... '
                              'current status is %s' % (sec, status))
            except KeyError:
                raise AirflowException('Could not get status of the SageMaker job')
            except ClientError:
                raise AirflowException('AWS request failed, check logs for more info')

            if status in non_terminal_states:
                running = True
            elif status in self.failed_states:
                raise AirflowException('SageMaker job failed because %s' % response['FailureReason'])
            else:
                running = False

            if max_ingestion_time and sec > max_ingestion_time:
                # ensure that the job gets killed if the max ingestion time is exceeded
                raise AirflowException('SageMaker job took more than %s seconds', max_ingestion_time)

        self.log.info('SageMaker Job Compeleted')
        response = describe_function(job_name)
        return response


def check_training_status_with_log(self, job_name, non_terminal_states, failed_states,
                                       wait_for_completion, check_interval, max_ingestion_time):
        """
        Display the logs for a given training job, optionally tailing them until the
        job is complete.

        :param job_name: name of the training job to check status and display logs for
        :type job_name: str
        :param non_terminal_states: the set of non_terminal states
        :type non_terminal_states: set
        :param failed_states: the set of failed states
        :type failed_states: set
        :param wait_for_completion: Whether to keep looking for new log entries
            until the job completes
        :type wait_for_completion: bool
        :param check_interval: The interval in seconds between polling for new log entries and job completion
        :type check_interval: int
        :param max_ingestion_time: the maximum ingestion time in seconds. Any
            SageMaker jobs that run longer than this will fail. Setting this to
            None implies no timeout for any SageMaker job.
        :type max_ingestion_time: int
        :return: None
        """

        sec = 0
        description = self.describe_training_job(job_name)
        self.log.info(secondary_training_status_message(description, None))
        instance_count = description['ResourceConfig']['InstanceCount']
        status = description['TrainingJobStatus']

        stream_names = []  # The list of log streams
        positions = {}     # The current position in each stream, map of stream name -> position

        job_already_completed = status not in non_terminal_states

        state = LogState.TAILING if wait_for_completion and not job_already_completed else LogState.COMPLETE

        # The loop below implements a state machine that alternates between checking the job status and
        # reading whatever is available in the logs at this point. Note, that if we were called with
        # wait_for_completion == False, we never check the job status.
        #
        # If wait_for_completion == TRUE and job is not completed, the initial state is TAILING
        # If wait_for_completion == FALSE, the initial state is COMPLETE
        # (doesn't matter if the job really is complete).
        #
        # The state table:
        #
        # STATE               ACTIONS                        CONDITION             NEW STATE
        # ----------------    ----------------               -----------------     ----------------
        # TAILING             Read logs, Pause, Get status   Job complete          JOB_COMPLETE
        #                                                    Else                  TAILING
        # JOB_COMPLETE        Read logs, Pause               Any                   COMPLETE
        # COMPLETE            Read logs, Exit                                      N/A
        #
        # Notes:
        # - The JOB_COMPLETE state forces us to do an extra pause and read any items that
        # got to Cloudwatch after the job was marked complete.
        last_describe_job_call = time.time()
        last_description = description

        while True:
            time.sleep(check_interval)
            sec = sec + check_interval

            state, last_description, last_describe_job_call = \
                self.describe_training_job_with_log(job_name, positions, stream_names,
                                                    instance_count, state, last_description,
                                                    last_describe_job_call)
            if state == LogState.COMPLETE:
                break

            if max_ingestion_time and sec > max_ingestion_time:
                # ensure that the job gets killed if the max ingestion time is exceeded
                raise AirflowException('SageMaker job took more than %s seconds', max_ingestion_time)

        if wait_for_completion:
            status = last_description['TrainingJobStatus']
            if status in failed_states:
                reason = last_description.get('FailureReason', '(No reason provided)')
                raise AirflowException('Error training {}: {} Reason: {}'.format(job_name, status, reason))
            billable_time = (last_description['TrainingEndTime'] - last_description['TrainingStartTime']) \
                * instance_count
            self.log.info('Billable seconds:{}'.format(int(billable_time.total_seconds()) + 1))

Display the logs for a given training job optionally tailing them until the job is complete .


def execute(self, context):
        """Execute the python dataflow job."""
        bucket_helper = GoogleCloudBucketHelper(
            self.gcp_conn_id, self.delegate_to)
        self.py_file = bucket_helper.google_cloud_to_local(self.py_file)
        hook = DataFlowHook(gcp_conn_id=self.gcp_conn_id,
                            delegate_to=self.delegate_to,
                            poll_sleep=self.poll_sleep)
        dataflow_options = self.dataflow_default_options.copy()
        dataflow_options.update(self.options)
        # Convert argument names from lowerCamelCase to snake case.
        camel_to_snake = lambda name: re.sub(
            r'[A-Z]', lambda x: '_' + x.group(0).lower(), name)
        formatted_options = {camel_to_snake(key): dataflow_options[key]
                             for key in dataflow_options}
        hook.start_python_dataflow(
            self.job_name, formatted_options,
            self.py_file, self.py_options)

def execute ( self , context ) : bucket_helper = GoogleCloudBucketHelper ( self . gcp_conn_id , self . delegate_to ) self . py_file = bucket_helper . google_cloud_to_local ( self . py_file ) hook = DataFlowHook ( gcp_conn_id = self . gcp_conn_id , delegate_to = self . delegate_to , poll_sleep = self . poll_sleep ) dataflow_options = self . dataflow_default_options . copy ( ) dataflow_options . update ( self . options ) camel_to_snake = lambda name : re . sub ( r'[A-Z]' , lambda x : '_' + x . group ( 0 ) . lower ( ) , name ) formatted_options = { camel_to_snake ( key ) : dataflow_options [ key ] for key in dataflow_options } hook . start_python_dataflow ( self . job_name , formatted_options , self . py_file , self . py_options )