def fix_env_variable(name, value):
    """Fix environment variable to a value within context. Unset if value is None."""
    orig = os.environ.get(name, None)
    if value is not None:
        # Set if value is not None
        os.environ[name] = value
    elif name in os.environ:
        # Unset if value is None
        del os.environ[name]
    try:
        yield
    finally:
        if orig is not None:
            # Restore original value
            os.environ[name] = orig
        elif name in os.environ:
            # Unset
            del os.environ[name]

def fix_env_variable ( name , value ) : orig = os . environ . get ( name , None ) if value is not None : os . environ [ name ] = value elif name in os . environ : del os . environ [ name ] try : yield finally : if orig is not None : os . environ [ name ] = orig elif name in os . environ : del os . environ [ name ]

Fix environment variable to a value within context . Unset if value is None .


def configure_arggroup(cls, parser):
        """Configure argument parser."""
        parser.add_argument(
            '--no-phenoecm', action='store_true',
            help='Disables the phenomenological ECM model building.')
        parser.add_argument(
            '--iterations', type=int, default=10,
            help='Number of outer-loop iterations (e.g. time loop) during benchmarking. '
                 'Default is 10, but actual number will be adapted to at least 0.2s runtime.')
        parser.add_argument(
            '--ignore-warnings', action='store_true',
            help='Ignore warnings about missmatched CPU model and frequency.')


def report(self, output_file=sys.stdout):
        """Report gathered analysis data in human readable form."""
        if self.verbose > 1:
            with pprint_nosort():
                pprint.pprint(self.results)

        if self.verbose > 0:
            print('Runtime (per repetition): {:.2g} s'.format(
                self.results['Runtime (per repetition) [s]']),
                file=output_file)
        if self.verbose > 0:
            print('Iterations per repetition: {!s}'.format(
                self.results['Iterations per repetition']),
                file=output_file)
        print('Runtime (per cacheline update): {:.2f} cy/CL'.format(
            self.results['Runtime (per cacheline update) [cy/CL]']),
            file=output_file)
        print('MEM volume (per repetition): {:.0f} Byte'.format(
            self.results['MEM volume (per repetition) [B]']),
            file=output_file)
        print('Performance: {:.2f} MFLOP/s'.format(self.results['Performance [MFLOP/s]']),
              file=output_file)
        print('Performance: {:.2f} MLUP/s'.format(self.results['Performance [MLUP/s]']),
              file=output_file)
        print('Performance: {:.2f} It/s'.format(self.results['Performance [MIt/s]']),
              file=output_file)
        if self.verbose > 0:
            print('MEM bandwidth: {:.2f} MByte/s'.format(self.results['MEM BW [MByte/s]']),
                  file=output_file)
        print('', file=output_file)

        if not self.no_phenoecm:
            print("Data Transfers:")
            print("{:^8} |".format("cache"), end='')
            for metrics in self.results['data transfers'].values():
                for metric_name in sorted(metrics):
                    print(" {:^14}".format(metric_name), end='')
                print()
                break
            for cache, metrics in sorted(self.results['data transfers'].items()):
                print("{!s:^8} |".format(cache), end='')
                for k, v in sorted(metrics.items()):
                    print(" {!s:^14}".format(v), end='')
                print()
            print()

            print('Phenomenological ECM model: {{ {T_OL:.1f} || {T_nOL:.1f} | {T_L1L2:.1f} | '
                  '{T_L2L3:.1f} | {T_L3MEM:.1f} }} cy/CL'.format(
                **{k: float(v) for k, v in self.results['ECM'].items()}),
                file=output_file)
            print('T_OL assumes that two loads per cycle may be retiered, which is true for '
                  '128bit SSE/half-AVX loads on SNB and IVY, and 256bit full-AVX loads on HSW, '
                  'BDW, SKL and SKX, but it also depends on AGU availability.',
                  file=output_file)

Report gathered analysis data in human readable form .


def parse_description():
    """
    Parse the description in the README file

    CommandLine:
        python -c "import setup; print(setup.parse_description())"
    """
    from os.path import dirname, join, exists
    readme_fpath = join(dirname(__file__), 'README.md')
    # print('readme_fpath = %r' % (readme_fpath,))
    # This breaks on pip install, so check that it exists.
    if exists(readme_fpath):
        # try:
        #     # convert markdown to rst for pypi
        #     import pypandoc
        #     return pypandoc.convert(readme_fpath, 'rst')
        # except Exception as ex:
            # strip out markdown to make a clean readme for pypi
            textlines = []
            with open(readme_fpath, 'r') as f:
                capture = False
                for line in f.readlines():
                    if '# Purpose' in line:
                        capture = True
                    elif line.startswith('##'):
                        break
                    elif capture:
                        textlines += [line]
            text = ''.join(textlines).strip()
            text = text.replace('\n\n', '_NLHACK_')
            text = text.replace('\n', ' ')
            text = text.replace('_NLHACK_', '\n\n')
            return text
    return ''

def
parse_description
(
)
:
from
os
.
path
import
dirname
,
join
,
exists
readme_fpath
=
join
(
dirname
(
__file__
)
,
'README.md'
)
if
exists
(
readme_fpath
)
:
textlines
=
[
]
with
open
(
readme_fpath
,
'r'
)
as
f
:
capture
=
False
for
line
in
f
.
readlines
(
)
:
if
'# Purpose'
in
line
:
capture
=
True
elif
line
.
startswith
(
'##'
)
:
break
elif
capture
:
textlines
+=
[
line
]
text
=
''
.
join
(
textlines
)
.
strip
(
)
text
=
text
.
replace
(
'\n\n'
,
'_NLHACK_'
)
text
=
text
.
replace
(
'\n'
,
' '
)
text
=
text
.
replace
(
'_NLHACK_'
,
'\n\n'
)
return
text
return
''


def schedule_retry(self, config):
    """Schedule a retry"""
    raise self.retry(countdown=config.get('SAILTHRU_RETRY_SECONDS'),
                     max_retries=config.get('SAILTHRU_RETRY_ATTEMPTS'))

def schedule_retry ( self , config ) : raise self . retry ( countdown = config . get ( 'SAILTHRU_RETRY_SECONDS' ) , max_retries = config . get ( 'SAILTHRU_RETRY_ATTEMPTS' ) )


def _build_purchase_item(course_id, course_url, cost_in_cents, mode, course_data, sku):
    """Build and return Sailthru purchase item object"""

    # build item description
    item = {
        'id': "{}-{}".format(course_id, mode),
        'url': course_url,
        'price': cost_in_cents,
        'qty': 1,
    }

    # get title from course info if we don't already have it from Sailthru
    if 'title' in course_data:
        item['title'] = course_data['title']
    else:
        # can't find, just invent title
        item['title'] = 'Course {} mode: {}'.format(course_id, mode)

    if 'tags' in course_data:
        item['tags'] = course_data['tags']

    # add vars to item
    item['vars'] = dict(course_data.get('vars', {}), mode=mode, course_run_id=course_id)

    item['vars']['purchase_sku'] = sku

    return item

def
_build_purchase_item
(
course_id
,
course_url
,
cost_in_cents
,
mode
,
course_data
,
sku
)
:
item
=
{
'id'
:
"{}-{}"
.
format
(
course_id
,
mode
)
,
'url'
:
course_url
,
'price'
:
cost_in_cents
,
'qty'
:
1
,
}
if
'title'
in
course_data
:
item
[
'title'
]
=
course_data
[
'title'
]
else
:
item
[
'title'
]
=
'Course {} mode: {}'
.
format
(
course_id
,
mode
)
if
'tags'
in
course_data
:
item
[
'tags'
]
=
course_data
[
'tags'
]
item
[
'vars'
]
=
dict
(
course_data
.
get
(
'vars'
,
{
}
)
,
mode
=
mode
,
course_run_id
=
course_id
)
item
[
'vars'
]
[
'purchase_sku'
]
=
sku
return
item


def _record_purchase(sailthru_client, email, item, purchase_incomplete, message_id, options):
    """Record a purchase in Sailthru

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        item (dict): Sailthru required information about the course
        purchase_incomplete (boolean): True if adding item to shopping cart
        message_id (str): Cookie used to identify marketing campaign
        options (dict): Sailthru purchase API options (e.g. template name)

    Returns:
        False if retryable error, else True
    """
    try:
        sailthru_response = sailthru_client.purchase(email, [item],
                                                     incomplete=purchase_incomplete, message_id=message_id,
                                                     options=options)

        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to record purchase in Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

    except SailthruClientError as exc:
        logger.exception("Exception attempting to record purchase for %s in Sailthru - %s", email, text_type(exc))
        return False

    return True


def _get_course_content(course_id, course_url, sailthru_client, site_code, config):
    """Get course information using the Sailthru content api or from cache.

    If there is an error, just return with an empty response.

    Arguments:
        course_id (str): course key of the course
        course_url (str): LMS url for course info page.
        sailthru_client (object): SailthruClient
        site_code (str): site code
        config (dict): config options

    Returns:
        course information from Sailthru
    """
    # check cache first
    cache_key = "{}:{}".format(site_code, course_url)
    response = cache.get(cache_key)
    if not response:
        try:
            sailthru_response = sailthru_client.api_get("content", {"id": course_url})
            if not sailthru_response.is_ok():
                response = {}
            else:
                response = sailthru_response.json
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

        except SailthruClientError:
            response = {}

        if not response:
            logger.error('Could not get course data from Sailthru on enroll/purchase event. '
                         'Calling Ecommerce Course API to get course info for enrollment confirmation email')
            response = _get_course_content_from_ecommerce(course_id, site_code=site_code)
            if response:
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

    return response

Get course information using the Sailthru content api or from cache .


def _get_course_content_from_ecommerce(course_id, site_code=None):
    """
    Get course information using the Ecommerce course api.

    In case of error returns empty response.
    Arguments:
        course_id (str): course key of the course
        site_code (str): site code

    Returns:
        course information from Ecommerce
    """
    api = get_ecommerce_client(site_code=site_code)
    try:
        api_response = api.courses(course_id).get()
    except Exception:  # pylint: disable=broad-except
        logger.exception(
            'An error occurred while retrieving data for course run [%s] from the Catalog API.',
            course_id,
            exc_info=True
        )
        return {}

    return {
        'title': api_response.get('name'),
        'verification_deadline': api_response.get('verification_deadline')
    }

Get course information using the Ecommerce course api .


def _update_unenrolled_list(sailthru_client, email, course_url, unenroll):
    """Maintain a list of courses the user has unenrolled from in the Sailthru user record

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        course_url (str): LMS url for course info page.
        unenroll (boolean): True if unenrolling, False if enrolling

    Returns:
        False if retryable error, else True
    """
    try:
        # get the user 'vars' values from sailthru
        sailthru_response = sailthru_client.api_get("user", {"id": email, "fields": {"vars": 1}})
        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to read user record from Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

        response_json = sailthru_response.json

        unenroll_list = []
        if response_json and "vars" in response_json and response_json["vars"] \
           and "unenrolled" in response_json["vars"]:
            unenroll_list = response_json["vars"]["unenrolled"]

        changed = False
        # if unenrolling, add course to unenroll list
        if unenroll:
            if course_url not in unenroll_list:
                unenroll_list.append(course_url)
                changed = True

        # if enrolling, remove course from unenroll list
        elif course_url in unenroll_list:
            unenroll_list.remove(course_url)
            changed = True

        if changed:
            # write user record back
            sailthru_response = sailthru_client.api_post(
                'user', {'id': email, 'key': 'email', 'vars': {'unenrolled': unenroll_list}})

            if not sailthru_response.is_ok():
                error = sailthru_response.get_error()
                logger.error("Error attempting to update user record in Sailthru: %s", error.get_message())
                return not can_retry_sailthru_request(error)

        return True

    except SailthruClientError as exc:
        logger.exception("Exception attempting to update user record for %s in Sailthru - %s", email, text_type(exc))
        return False


def send_course_refund_email(self, email, refund_id, amount, course_name, order_number, order_url, site_code=None):
    """ Sends the course refund email.

    Args:
        self: Ignore.
        email (str): Recipient's email address.
        refund_id (int): ID of the refund that initiated this task.
        amount (str): Formatted amount of the refund.
        course_name (str): Name of the course for which payment was refunded.
        order_number (str): Order number of the order that was refunded.
        order_url (str): Receipt URL of the refunded order.
        site_code (str): Identifier of the site sending the email.
    """
    config = get_sailthru_configuration(site_code)

    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        # NOTE: We rely on the function to log the error for us
        return

    email_vars = {
        'amount': amount,
        'course_name': course_name,
        'order_number': order_number,
        'order_url': order_url,
    }

    try:
        response = sailthru_client.send(
            template=config['templates']['course_refund'],
            email=email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            'A client error occurred while attempting to send a course refund notification for refund [%d].',
            refund_id
        )
        return

    if response.is_ok():
        logger.info('Course refund notification sent for refund %d.', refund_id)
    else:
        error = response.get_error()
        logger.error(
            'An error occurred while attempting to send a course refund notification for refund [%d]: %d - %s',
            refund_id, error.get_error_code(), error.get_message()
        )

        if can_retry_sailthru_request(error):
            logger.info(
                'An attempt will be made again to send a course refund notification for refund [%d].',
                refund_id
            )
            schedule_retry(self, config)
        else:
            logger.warning(
                'No further attempts will be made to send a course refund notification for refund [%d].',
                refund_id
            )


def _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, task):
    """Handles sending offer assignment notification emails and retrying failed emails when appropriate."""
    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None
    email_vars = {
        'subject': subject,
        'email_body': email_body,
    }
    try:
        response = sailthru_client.send(
            template=config['templates']['assignment_email'],
            email=user_email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None

    if not response.is_ok():
        error = response.get_error()
        logger.error(
            '[Offer Assignment] A {token_error_code} - {token_error_message} error occurred'
            ' while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(
                message=email_body,
                token_error_code=error.get_error_code(),
                token_error_message=error.get_message()
            )
        )
        if can_retry_sailthru_request(error):
            logger.info(
                '[Offer Assignment] An attempt will be made to resend the offer assignment notification.'
                ' Message: {message}'.format(message=email_body)
            )
            schedule_retry(task, config)
        else:
            logger.warning(
                '[Offer Assignment] No further attempts will be made to send the offer assignment notification.'
                ' Failed Message: {message}'.format(message=email_body)
            )

    return response

Handles sending offer assignment notification emails and retrying failed emails when appropriate .


def get_logger_config(log_dir='/var/tmp',
                      logging_env='no_env',
                      edx_filename='edx.log',
                      dev_env=False,
                      debug=False,
                      local_loglevel='INFO',
                      service_variant='ecomworker'):

    """
    Returns a dictionary containing logging configuration.

    If dev_env is True, logging will not be done via local rsyslogd.
    Instead, application logs will be dropped into log_dir. 'edx_filename'
    is ignored unless dev_env is True.
    """
    # Revert to INFO if an invalid string is passed in
    if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
        local_loglevel = 'INFO'

    hostname = platform.node().split('.')[0]
    syslog_format = (
        '[service_variant={service_variant}]'
        '[%(name)s][env:{logging_env}] %(levelname)s '
        '[{hostname}  %(process)d] [%(filename)s:%(lineno)d] '
        '- %(message)s'
    ).format(
        service_variant=service_variant,
        logging_env=logging_env, hostname=hostname
    )

    if debug:
        handlers = ['console']
    else:
        handlers = ['local']

    logger_config = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'standard': {
                'format': '%(asctime)s %(levelname)s %(process)d '
                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
            },
            'syslog_format': {'format': syslog_format},
            'raw': {'format': '%(message)s'},
        },
        'handlers': {
            'console': {
                'level': 'DEBUG' if debug else 'INFO',
                'class': 'logging.StreamHandler',
                'formatter': 'standard',
                'stream': sys.stdout,
            },
        },
        'loggers': {
            'requests': {
                'handlers': handlers,
                'level': 'WARNING',
                'propagate': True
            },
            '': {
                'handlers': handlers,
                'level': 'DEBUG',
                'propagate': False
            },
        }
    }

    if dev_env:
        edx_file_loc = os.path.join(log_dir, edx_filename)
        logger_config['handlers'].update({
            'local': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': local_loglevel,
                'formatter': 'standard',
                'filename': edx_file_loc,
                'maxBytes': 1024 * 1024 * 2,
                'backupCount': 5,
            },
        })
    else:
        logger_config['handlers'].update({
            'local': {
                'level': local_loglevel,
                'class': 'logging.handlers.SysLogHandler',
                # Use a different address for Mac OS X
                'address': '/var/run/syslog' if sys.platform == 'darwin' else '/dev/log',
                'formatter': 'syslog_format',
                'facility': SysLogHandler.LOG_LOCAL0,
            },
        })

    return logger_config

Returns a dictionary containing logging configuration .


def _retry_order(self, exception, max_fulfillment_retries, order_number):
    """
    Retry with exponential backoff until fulfillment
    succeeds or the retry limit is reached. If the retry limit is exceeded,
    the exception is re-raised.
    """
    retries = self.request.retries
    if retries == max_fulfillment_retries:
        logger.exception('Fulfillment of order [%s] failed. Giving up.', order_number)
    else:
        logger.warning('Fulfillment of order [%s] failed. Retrying.', order_number)

    countdown = 2 ** retries
    raise self.retry(exc=exception, countdown=countdown, max_retries=max_fulfillment_retries)

Retry with exponential backoff until fulfillment succeeds or the retry limit is reached . If the retry limit is exceeded the exception is re - raised .


def fulfill_order(self, order_number, site_code=None, email_opt_in=False):
    """Fulfills an order.

    Arguments:
        order_number (str): Order number indicating which order to fulfill.

    Returns:
        None
    """
    max_fulfillment_retries = get_configuration('MAX_FULFILLMENT_RETRIES', site_code=site_code)
    api = get_ecommerce_client(site_code=site_code)
    try:
        logger.info('Requesting fulfillment of order [%s].', order_number)
        api.orders(order_number).fulfill.put(email_opt_in=email_opt_in)
    except exceptions.HttpClientError as exc:
        status_code = exc.response.status_code  # pylint: disable=no-member
        if status_code == 406:
            # The order is not fulfillable. Therefore, it must be complete.
            logger.info('Order [%s] has already been fulfilled. Ignoring.', order_number)
            raise Ignore()
        else:
            # Unknown client error. Let's retry to resolve it.
            logger.warning(
                'Fulfillment of order [%s] failed because of HttpClientError. Retrying',
                order_number,
                exc_info=True
            )
            _retry_order(self, exc, max_fulfillment_retries, order_number)

    except (exceptions.HttpServerError, exceptions.Timeout, SSLError) as exc:
        # Fulfillment failed, retry
        _retry_order(self, exc, max_fulfillment_retries, order_number)


def get_sailthru_client(site_code):
    """
    Returns a Sailthru client for the specified site.

    Args:
        site_code (str): Site for which the client should be configured.

    Returns:
        SailthruClient

    Raises:
        SailthruNotEnabled: If Sailthru is not enabled for the specified site.
        ConfigurationError: If either the Sailthru API key or secret are not set for the site.
    """
    # Get configuration
    config = get_sailthru_configuration(site_code)

    # Return if Sailthru integration disabled
    if not config.get('SAILTHRU_ENABLE'):
        msg = 'Sailthru is not enabled for site {}'.format(site_code)
        log.debug(msg)
        raise SailthruNotEnabled(msg)

    # Make sure key and secret configured
    key = config.get('SAILTHRU_KEY')
    secret = config.get('SAILTHRU_SECRET')

    if not (key and secret):
        msg = 'Both key and secret are required for site {}'.format(site_code)
        log.error(msg)
        raise ConfigurationError(msg)

    return SailthruClient(key, secret)

Returns a Sailthru client for the specified site .


def get(self, key):
        """Get an object from the cache

        Arguments:
            key (str): Cache key

        Returns:
            Cached object
        """
        lock.acquire()
        try:
            if key not in self:
                return None

            current_time = time.time()
            if self[key].expire > current_time:
                return self[key].value

            # expired key, clean out all expired keys
            deletes = []
            for k, val in self.items():
                if val.expire <= current_time:
                    deletes.append(k)
            for k in deletes:
                del self[k]

            return None
        finally:
            lock.release()

def get ( self , key ) : lock . acquire ( ) try : if key not in self : return None current_time = time . time ( ) if self [ key ] . expire > current_time : return self [ key ] . value deletes = [ ] for k , val in self . items ( ) : if val . expire <= current_time : deletes . append ( k ) for k in deletes : del self [ k ] return None finally : lock . release ( )


def set(self, key, value, duration):
        """Save an object in the cache

        Arguments:
            key (str): Cache key
            value (object): object to cache
            duration (int): time in seconds to keep object in cache

        """
        lock.acquire()
        try:
            self[key] = CacheObject(value, duration)
        finally:
            lock.release()

def set ( self , key , value , duration ) : lock . acquire ( ) try : self [ key ] = CacheObject ( value , duration ) finally : lock . release ( )


def get_configuration(variable, site_code=None):
    """
    Get a value from configuration.

    Retrieves the value corresponding to the given variable from the configuration module
    currently in use by the app.  Specify a site_code value to check for a site-specific override.

    Arguments:
        variable (str): The name of a variable from the configuration module.

    Keyword Arguments:
        site_code (str): The SITE_OVERRIDES key to inspect for site-specific values

    Returns:
        The value corresponding to the variable, or None if the variable is not found.
    """
    name = os.environ.get(CONFIGURATION_MODULE)

    # __import__ performs a full import, but only returns the top-level
    # package, not the targeted module. sys.modules is a dictionary
    # mapping module names to loaded modules.
    __import__(name)
    module = sys.modules[name]

    # Locate the setting in the specified module, then attempt to apply a site-specific override
    setting_value = getattr(module, variable, None)
    site_overrides = getattr(module, 'SITE_OVERRIDES', None)
    if site_overrides and site_code is not None:
        site_specific_overrides = site_overrides.get(site_code)
        if site_specific_overrides:
            override_value = site_specific_overrides.get(variable)
            if override_value:
                setting_value = override_value

    if setting_value is None:
        raise RuntimeError('Worker is improperly configured: {} is unset in {}.'.format(variable, module))
    return setting_value


def get_overrides_filename(variable):
    """
    Get the name of the file containing configuration overrides
    from the provided environment variable.
    """
    filename = os.environ.get(variable)

    if filename is None:
        msg = 'Please set the {} environment variable.'.format(variable)
        raise EnvironmentError(msg)

    return filename

Get the name of the file containing configuration overrides from the provided environment variable .


def get_value_by_version(d):
    """
    Finds the value depending in current eplus version.

    Parameters
    ----------
    d: dict
        {(0, 0): value, (x, x): value, ...}
        for current version (cv), current value is the value of version v such as v <= cv < v+1
    """
    from oplus import CONF  # touchy import

    cv = CONF.eplus_version[:2]
    for v, value in sorted(d.items(), reverse=True):
        if cv >= v:
            return value

def get_value_by_version ( d ) : from oplus import CONF cv = CONF . eplus_version [ : 2 ] for v , value in sorted ( d . items ( ) , reverse = True ) : if cv >= v : return value

Finds the value depending in current eplus version .


def eplus_version(self):
        """
        if _eplus_version is defined => _eplus_version
        else most recent eplus available version
        """
        # check energy plus is installed
        if len(self.eplus_available_versions) == 0:
            raise RuntimeError("Energy plus is not install, can't use oplus package.")

        # see if version is defined
        if self._eplus_version is not None:
            return self._eplus_version

        # return most recent version
        return sorted(self.eplus_available_versions.keys(), reverse=True)[0]

if _eplus_version is defined = > _eplus_version else most recent eplus available version


def _file_refs(self):
        """
        Defined here so that we can use the class variables, in order to subclass in oplusplus
        """
        if self._prepared_file_refs is None:
            self._prepared_file_refs = {
                FILE_REFS.idf: FileInfo(
                    constructor=lambda path: self._epm_cls.from_idf(path, idd_or_buffer_or_path=self._idd),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.idf)
                ),
                FILE_REFS.epw: FileInfo(
                    constructor=lambda path: self._weather_data_cls.from_epw(path),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.epw)
                ),
                FILE_REFS.eio: FileInfo(
                    constructor=lambda path: self._eio_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.eio)
                ),
                FILE_REFS.eso: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(
                        self.dir_path,
                        FILE_REFS.eso
                    )
                ),
                FILE_REFS.mtr: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtr)
                ),
                FILE_REFS.mtd: FileInfo(
                    constructor=lambda path: self._mtd_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtd)
                ),
                FILE_REFS.mdd: FileInfo(
                    constructor=lambda path: open(path).read(),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mdd)

                ),
                FILE_REFS.err: FileInfo(
                    constructor=lambda path: self._err_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.err)
                ),
                FILE_REFS.summary_table: FileInfo(
                    constructor=lambda path: self._summary_table_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.summary_table)
                )
            }
        return self._prepared_file_refs

def _file_refs ( self ) : if self . _prepared_file_refs is None : self . _prepared_file_refs = { FILE_REFS . idf : FileInfo ( constructor = lambda path : self . _epm_cls . from_idf ( path , idd_or_buffer_or_path = self . _idd ) , get_path = lambda : get_input_file_path ( self . dir_path , FILE_REFS . idf ) ) , FILE_REFS . epw : FileInfo ( constructor = lambda path : self . _weather_data_cls . from_epw ( path ) , get_path = lambda : get_input_file_path ( self . dir_path , FILE_REFS . epw ) ) , FILE_REFS . eio : FileInfo ( constructor = lambda path : self . _eio_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . eio ) ) , FILE_REFS . eso : FileInfo ( constructor = lambda path : self . _standard_output_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . eso ) ) , FILE_REFS . mtr : FileInfo ( constructor = lambda path : self . _standard_output_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mtr ) ) , FILE_REFS . mtd : FileInfo ( constructor = lambda path : self . _mtd_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mtd ) ) , FILE_REFS . mdd : FileInfo ( constructor = lambda path : open ( path ) . read ( ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mdd ) ) , FILE_REFS . err : FileInfo ( constructor = lambda path : self . _err_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . err ) ) , FILE_REFS . summary_table : FileInfo ( constructor = lambda path : self . _summary_table_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . summary_table ) ) } return self . _prepared_file_refs

Defined here so that we can use the class variables in order to subclass in oplusplus


def _dev_populate_from_json_data(self, json_data):
        """
        !! Must only be called once, when empty !!
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        # manage comment if any
        comment = json_data.pop("_comment", None)
        if comment is not None:
            self._comment = comment

        # populate external files
        external_files_data = json_data.pop("_external_files", dict())
        self._dev_external_files_manager.populate_from_json_data(external_files_data)

        # manage records
        added_records = []
        for table_ref, json_data_records in json_data.items():
            # find table
            table = getattr(self, table_ref)

            # create record (inert)
            records = table._dev_add_inert(json_data_records)

            # add records (inert)
            added_records.extend(records)

        # activate hooks
        for r in added_records:
            r._dev_activate_hooks()

        # activate links and external files
        for r in added_records:
            r._dev_activate_links()
            r._dev_activate_external_files()

def _dev_populate_from_json_data ( self , json_data ) : comment = json_data . pop ( "_comment" , None ) if comment is not None : self . _comment = comment external_files_data = json_data . pop ( "_external_files" , dict ( ) ) self . _dev_external_files_manager . populate_from_json_data ( external_files_data ) added_records = [ ] for table_ref , json_data_records in json_data . items ( ) : table = getattr ( self , table_ref ) records = table . _dev_add_inert ( json_data_records ) added_records . extend ( records ) for r in added_records : r . _dev_activate_hooks ( ) for r in added_records : r . _dev_activate_links ( ) r . _dev_activate_external_files ( )

!! Must only be called once when empty !!


def get_external_files(self):
        """
        An external file manages file paths.
        """
        external_files = []
        for table in self._tables.values():
            for r in table:
                external_files.extend([ef for ef in r.get_external_files()])
        return external_files

def get_external_files ( self ) : external_files = [ ] for table in self . _tables . values ( ) : for r in table : external_files . extend ( [ ef for ef in r . get_external_files ( ) ] ) return external_files


def set_defaults(self):
        """
        All fields of Epm with a default value and that are null will be set to their default value.
        """
        for table in self._tables.values():
            for r in table:
                r.set_defaults()

def set_defaults ( self ) : for table in self . _tables . values ( ) : for r in table : r . set_defaults ( )

All fields of Epm with a default value and that are null will be set to their default value .


def prepare_extensible(self):
        """
        This function finishes initialization, must be called once all field descriptors and tag have been filled.
        """
        # see if extensible and store cycle len
        for k in self._tags:
            if "extensible" in k:
                cycle_len = int(k.split(":")[1])
                break
        else:
            # not extensible
            return

        # find cycle start and prepare patterns
        cycle_start = None
        cycle_patterns = []
        for i, field_descriptor in enumerate(self._field_descriptors):
            # quit if finished
            if (cycle_start is not None) and (i >= (cycle_start + cycle_len)):
                break

            # set cycle start if not set yet
            if (cycle_start is None) and ("begin-extensible" in field_descriptor.tags):
                cycle_start = i

            # leave if cycle start not reached yet
            if cycle_start is None:
                continue

            # store pattern
            cycle_patterns.append(field_descriptor.ref.replace("1", r"(\d+)"))
        else:
            raise RuntimeError("cycle start not found")

        # detach unnecessary field descriptors
        self._field_descriptors = self._field_descriptors[:cycle_start + cycle_len]

        # store cycle info
        self.extensible_info = (cycle_start, cycle_len, tuple(cycle_patterns))

        # set field descriptor cycle_start index (for error messages while serialization)
        for i, fd in enumerate(self._field_descriptors[cycle_start:]):
            fd.set_extensible_info(cycle_start, cycle_len, cycle_patterns[i])

def
prepare_extensible
(
self
)
:
for
k
in
self
.
_tags
:
if
"extensible"
in
k
:
cycle_len
=
int
(
k
.
split
(
":"
)
[
1
]
)
break
else
:
return
cycle_start
=
cycle_patterns
=
[
]
for
i
,
field_descriptor
in
enumerate
(
self
.
_field_descriptors
)
:
if
(
cycle_start
is
not
)
and
(
i
>=
(
cycle_start
+
cycle_len
)
)
:
break
if
(
cycle_start
is
)
and
(
"begin-extensible"
in
field_descriptor
.
tags
)
:
cycle_start
=
i
if
cycle_start
is
:
continue
cycle_patterns
.
append
(
field_descriptor
.
ref
.
replace
(
"1"
,
r"(\d+)"
)
)
else
:
raise
RuntimeError
(
"cycle start not found"
)
self
.
_field_descriptors
=
self
.
_field_descriptors
[
:
cycle_start
+
cycle_len
]
self
.
extensible_info
=
(
cycle_start
,
cycle_len
,
tuple
(
cycle_patterns
)
)
for
i
,
fd
in
enumerate
(
self
.
_field_descriptors
[
cycle_start
:
]
)
:
fd
.
set_extensible_info
(
cycle_start
,
cycle_len
,
cycle_patterns
[
i
]
)

This function finishes initialization must be called once all field descriptors and tag have been filled .


def get_extended_name(self, index):
        """
        manages extensible names
        """
        field_descriptor = self.get_field_descriptor(index)
        if self.extensible_info is None:
            return field_descriptor.name
        cycle_start, cycle_len, _ = self.extensible_info
        cycle_num = (index - cycle_start) // cycle_len
        return None if field_descriptor.name is None else field_descriptor.name.replace("1", str(cycle_num))

def get_extended_name ( self , index ) : field_descriptor = self . get_field_descriptor ( index ) if self . extensible_info is None : return field_descriptor . name cycle_start , cycle_len , _ = self . extensible_info cycle_num = ( index - cycle_start ) // cycle_len return None if field_descriptor . name is None else field_descriptor . name . replace ( "1" , str ( cycle_num ) )


def short_refs(self):
        """
        we calculate on the fly to avoid managing registrations and un-registrations

        Returns
        -------
        {ref: short_ref, ...
        """
        naive_short_refs_d = dict()  # naive_short_ref: {refs, ...}
        for ef in self._external_files:
            if ef.naive_short_ref not in naive_short_refs_d:
                naive_short_refs_d[ef.naive_short_ref] = set()
            naive_short_refs_d[ef.naive_short_ref].add(ef.ref)

        short_refs = dict()
        for naive_short_ref, refs in naive_short_refs_d.items():
            if len(refs) == 1:
                short_refs[refs.pop()] = naive_short_ref
                continue
            base, ext = os.path.splitext(naive_short_ref)
            for i, ref in enumerate(sorted(refs)):
                short_refs[ref] = f"{base}-{i}.{ext}"

        return short_refs

def short_refs ( self ) : naive_short_refs_d = dict ( ) for ef in self . _external_files : if ef . naive_short_ref not in naive_short_refs_d : naive_short_refs_d [ ef . naive_short_ref ] = set ( ) naive_short_refs_d [ ef . naive_short_ref ] . add ( ef . ref ) short_refs = dict ( ) for naive_short_ref , refs in naive_short_refs_d . items ( ) : if len ( refs ) == 1 : short_refs [ refs . pop ( ) ] = naive_short_ref continue base , ext = os . path . splitext ( naive_short_ref ) for i , ref in enumerate ( sorted ( refs ) ) : short_refs [ ref ] = f"{base}-{i}.{ext}" return short_refs

we calculate on the fly to avoid managing registrations and un - registrations


def get_value(self, column_name_or_i, filter_column_name_or_i, filter_criterion):
        """
        Returns first occurrence of value of filter column matching filter criterion.
        """
        # find column indexes
        column_i = self._get_column_index(column_name_or_i)
        filter_column_i = self._get_column_index(filter_column_name_or_i)

        filter_fct = {
            float: lambda x: float(x) == filter_criterion,
            int: lambda x: int(x) == filter_criterion,
            str: lambda x: x.lower() == filter_criterion.lower()
        }[type(filter_criterion)]

        for row_i, row in enumerate(self._data):
            if filter_fct(row[filter_column_i]):
                break
        else:
            raise ValueError("Filter did not return any values.")

        return self._data[row_i][column_i]

def
get_value
(
self
,
column_name_or_i
,
filter_column_name_or_i
,
filter_criterion
)
:
column_i
=
self
.
_get_column_index
(
column_name_or_i
)
filter_column_i
=
self
.
_get_column_index
(
filter_column_name_or_i
)
filter_fct
=
{
float
:
lambda
x
:
float
(
x
)
==
filter_criterion
,
int
:
lambda
x
:
int
(
x
)
==
filter_criterion
,
str
:
lambda
x
:
x
.
lower
(
)
==
filter_criterion
.
lower
(
)
}
[
type
(
filter_criterion
)
]
for
row_i
,
row
in
enumerate
(
self
.
_data
)
:
if
filter_fct
(
row
[
filter_column_i
]
)
:
break
else
:
raise
ValueError
(
"Filter did not return any values."
)
return
self
.
_data
[
row_i
]
[
column_i
]

Returns first occurrence of value of filter column matching filter criterion .


def _update_value_inert(self, index, value):
        """
        is only called by _update_inert
        """
        # get field descriptor
        field_descriptor = self._table._dev_descriptor.get_field_descriptor(index)

        # prepare value
        value = field_descriptor.deserialize(value, index)

        # unregister previous link if relevant
        if isinstance(value, Link):
            # de-activate current link if any
            current_link = self._data.get(index)
            if current_link is not None:
                current_link.unregister()

        # unregister previous hook if relevant
        if isinstance(value, RecordHook):
            current_record_hook = self._data.get(index)
            if current_record_hook is not None:
                current_record_hook.unregister()

        # unregister previous external file if relevant
        if isinstance(value, ExternalFile):
            current_external_file = self._data.get(index)
            if current_external_file is not None:
                current_external_file._dev_unregister()

        # if None remove and leave
        if value in (None, NONE_RECORD_HOOK, NONE_LINK, NONE_EXTERNAL_FILE):
            # we don't check required, because this method is called by _update_inert which does the job
            self._dev_set_none_without_unregistering(index, check_not_required=False)
            return

        # if relevant, store current pk to signal table
        old_hook = None
        if index == 0 and not self._table._dev_auto_pk:
            old_hook = self._data.get(0)  # we use get, because record may not have a pk yet if it is being created

        # set value
        self._data[index] = value

        # signal pk update if relevant
        if old_hook is not None:
            self._table._dev_record_pk_was_updated(old_hook.target_value)

def _update_value_inert ( self , index , value ) : field_descriptor = self . _table . _dev_descriptor . get_field_descriptor ( index ) value = field_descriptor . deserialize ( value , index ) if isinstance ( value , Link ) : current_link = self . _data . get ( index ) if current_link is not None : current_link . unregister ( ) if isinstance ( value , RecordHook ) : current_record_hook = self . _data . get ( index ) if current_record_hook is not None : current_record_hook . unregister ( ) if isinstance ( value , ExternalFile ) : current_external_file = self . _data . get ( index ) if current_external_file is not None : current_external_file . _dev_unregister ( ) if value in ( None , NONE_RECORD_HOOK , NONE_LINK , NONE_EXTERNAL_FILE ) : self . _dev_set_none_without_unregistering ( index , check_not_required = False ) return old_hook = None if index == 0 and not self . _table . _dev_auto_pk : old_hook = self . _data . get ( 0 ) self . _data [ index ] = value if old_hook is not None : self . _table . _dev_record_pk_was_updated ( old_hook . target_value )


def update(self, data=None, **or_data):
        """
        Updates simultaneously all given fields.

        Parameters
        ----------
        data: dictionary containing field lowercase names or index as keys, and field values as values (dict syntax)
        or_data: keyword arguments containing field names as keys (kwargs syntax)
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        data = or_data if data is None else data

        self._update_inert(data)

        self._dev_activate_hooks()
        self._dev_activate_links()
        self._dev_activate_external_files()

def update ( self , data = None , ** or_data ) : data = or_data if data is None else data self . _update_inert ( data ) self . _dev_activate_hooks ( ) self . _dev_activate_links ( ) self . _dev_activate_external_files ( )

Updates simultaneously all given fields .


def set_defaults(self):
        """
        sets all empty fields for which a default value is defined to default value
        """
        defaults = {}
        for i in range(len(self)):
            if i in self._data:
                continue
            default = self.get_field_descriptor(i).tags.get("default", [None])[0]
            if default is not None:
                defaults[i] = default

        self.update(defaults)

def set_defaults ( self ) : defaults = { } for i in range ( len ( self ) ) : if i in self . _data : continue default = self . get_field_descriptor ( i ) . tags . get ( "default" , [ None ] ) [ 0 ] if default is not None : defaults [ i ] = default self . update ( defaults )


def add_fields(self, *args):
        """
        This method only works for extensible fields. It allows to add values without precising their fields' names
        or indexes.

        Parameters
        ----------
        args: field values
        """
        if not self.is_extensible():
            raise TypeError("Can't use add_fields on a non extensible record.")

        # prepare update data
        self_len = len(self)
        data = dict([(self_len + i, args[i]) for i in range(len(args))])

        # update
        self.update(data)

This method only works for extensible fields . It allows to add values without precising their fields names or indexes .


def pop(self, index=None):
        """
        This method only works for extensible fields. It allows to remove a value and shift all other values to fill
        the gap.

        Parameters
        ----------
        index: int, default None
            index of field to remove.

        Returns
        -------
        serialize value of popped field
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # get extensible info
        cycle_start, cycle_len, patterns = self.get_extensible_info()

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # pop
        serialized_value = fields.pop(index-cycle_start)

        # add remaining
        self.add_fields(*fields)

        return serialized_value

def pop ( self , index = None ) : index = self . _prepare_pop_insert_index ( index = index ) cycle_start , cycle_len , patterns = self . get_extensible_info ( ) fields = self . clear_extensible_fields ( ) serialized_value = fields . pop ( index - cycle_start ) self . add_fields ( * fields ) return serialized_value

This method only works for extensible fields . It allows to remove a value and shift all other values to fill the gap .


def insert(self, index, value):
        """
        This method only works for extensible fields. It allows to insert a value, and shifts all other following
        values.

        Parameters
        ----------
        index: position of insertion
        value: value to insert
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # insert
        fields.insert(index, value)

        # add new list
        self.add_fields(*fields)

def insert ( self , index , value ) : index = self . _prepare_pop_insert_index ( index = index ) fields = self . clear_extensible_fields ( ) fields . insert ( index , value ) self . add_fields ( * fields )

This method only works for extensible fields . It allows to insert a value and shifts all other following values .


def delete(self):
        """
        Deletes record, and removes it from database.
        """
        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. unregister: links, hooks and external files
        # 3. remove from table without unregistering

        # unregister links
        self._unregister_links()

        # unregister hooks
        self._unregister_hooks()

        # unregister external files
        self._unregister_external_files()

        # tell table to remove without unregistering
        self.get_table()._dev_remove_record_without_unregistering(self)

        # make stale
        self._table = None
        self._data = None

Deletes record and removes it from database .


def register_record_hook(self, hook):
        """
        target record must have been set
        """
        for key in hook.keys:
            if key in self._record_hooks:
                field_descriptor = hook.target_record.get_field_descriptor(hook.target_index)
                raise FieldValidationError(
                    f"Reference key already exists, can't create: {key}. "
                    f"{field_descriptor.get_error_location_message(hook.target_value, hook.target_index)}"
                )
            self._record_hooks[key] = hook


def register_link(self, link):
        """
        source record and index must have been set
        """
        keys = tuple((ref, link.initial_hook_value) for ref in link.hook_references)

        # look for a record hook
        for k in keys:
            if k in self._record_hooks:
                # set link target
                link.set_target(target_record=self._record_hooks[k].target_record)
                break
        else:
            # look for a table hook
            for k in keys:
                if k in self._table_hooks:
                    # set link target
                    link.set_target(target_table=self._table_hooks[k])
                    break
            else:
                field_descriptor = link.source_record.get_field_descriptor(link.source_index)
                raise FieldValidationError(
                    f"No object found with any of given references : {keys}. "
                    f"{field_descriptor.get_error_location_message(link.initial_hook_value)}"
                )

        # store by source
        if link.source_record not in self._links_by_source:
            self._links_by_source[link.source_record] = set()
        self._links_by_source[link.source_record].add(link)

        # store by target
        if link.target not in self._links_by_target:
            self._links_by_target[link.target] = set()
        self._links_by_target[link.target].add(link)


def _create_regex(self, line, intent_name):
        """ Create regex and return. If error occurs returns None. """
        try:
            return re.compile(self._create_intent_pattern(line, intent_name),
                              re.IGNORECASE)
        except sre_constants.error as e:
            LOG.warning('Failed to parse the line "{}" '
                        'for {}'.format(line, intent_name))
            return None

def
_create_regex
(
self
,
line
,
intent_name
)
:
try
:
return
re
.
compile
(
self
.
_create_intent_pattern
(
line
,
intent_name
)
,
re
.
IGNORECASE
)
except
sre_constants
.
error
as
e
:
LOG
.
warning
(
'Failed to parse the line "{}" '
'for {}'
.
format
(
line
,
intent_name
)
)
return

Create regex and return . If error occurs returns None .


def remaining_duration(self, time):
        '''Returns the remaining duration for a recording.
        '''
        return max(0, self.end - max(self.start, time))

def remaining_duration ( self , time ) : return max ( 0 , self . end - max ( self . start , time ) )

Returns the remaining duration for a recording .


def serialize(self):
        '''Serialize this object as dictionary usable for conversion to JSON.

        :return: Dictionary representing this object.
        '''
        return {
            'type': 'event',
            'id': self.uid,
            'attributes': {
                'start': self.start,
                'end': self.end,
                'uid': self.uid,
                'title': self.title,
                'data': self.get_data(),
                'status': Status.str(self.status)
                }
            }

def serialize ( self ) : return { 'type' : 'event' , 'id' : self . uid , 'attributes' : { 'start' : self . start , 'end' : self . end , 'uid' : self . uid , 'title' : self . title , 'data' : self . get_data ( ) , 'status' : Status . str ( self . status ) } }

Serialize this object as dictionary usable for conversion to JSON .


def http_request(url, post_data=None):
    '''Make an HTTP request to a given URL with optional parameters.
    '''
    logger.debug('Requesting URL: %s' % url)
    buf = bio()
    curl = pycurl.Curl()
    curl.setopt(curl.URL, url.encode('ascii', 'ignore'))

    # Disable HTTPS verification methods if insecure is set
    if config()['server']['insecure']:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    if config()['server']['certificate']:
        # Make sure verification methods are turned on
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        # Import your certificates
        curl.setopt(pycurl.CAINFO, config()['server']['certificate'])

    if post_data:
        curl.setopt(curl.HTTPPOST, post_data)
    curl.setopt(curl.WRITEFUNCTION, buf.write)
    curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
    curl.setopt(pycurl.USERPWD, "%s:%s" % (config()['server']['username'],
                                           config()['server']['password']))
    curl.setopt(curl.HTTPHEADER, ['X-Requested-Auth: Digest'])
    curl.setopt(curl.FAILONERROR, True)
    curl.setopt(curl.FOLLOWLOCATION, True)
    curl.perform()
    curl.close()
    result = buf.getvalue()
    buf.close()
    return result

def
http_request
(
url
,
post_data
=
)
:
logger
.
debug
(
'Requesting URL: %s'
%
url
)
buf
=
bio
(
)
curl
=
pycurl
.
Curl
(
)
curl
.
setopt
(
curl
.
URL
,
url
.
encode
(
'ascii'
,
'ignore'
)
)
if
config
(
)
[
'server'
]
[
'insecure'
]
:
curl
.
setopt
(
curl
.
SSL_VERIFYPEER
,
0
)
curl
.
setopt
(
curl
.
SSL_VERIFYHOST
,
0
)
if
config
(
)
[
'server'
]
[
'certificate'
]
:
curl
.
setopt
(
curl
.
SSL_VERIFYPEER
,
1
)
curl
.
setopt
(
curl
.
SSL_VERIFYHOST
,
2
)
curl
.
setopt
(
pycurl
.
CAINFO
,
config
(
)
[
'server'
]
[
'certificate'
]
)
if
post_data
:
curl
.
setopt
(
curl
.
HTTPPOST
,
post_data
)
curl
.
setopt
(
curl
.
WRITEFUNCTION
,
buf
.
write
)
curl
.
setopt
(
pycurl
.
HTTPAUTH
,
pycurl
.
HTTPAUTH_DIGEST
)
curl
.
setopt
(
pycurl
.
USERPWD
,
"%s:%s"
%
(
config
(
)
[
'server'
]
[
'username'
]
,
config
(
)
[
'server'
]
[
'password'
]
)
)
curl
.
setopt
(
curl
.
HTTPHEADER
,
[
'X-Requested-Auth: Digest'
]
)
curl
.
setopt
(
curl
.
FAILONERROR
,
True
)
curl
.
setopt
(
curl
.
FOLLOWLOCATION
,
True
)
curl
.
perform
(
)
curl
.
close
(
)
result
=
buf
.
getvalue
(
)
buf
.
close
(
)
return
result

Make an HTTP request to a given URL with optional parameters .


def get_service(service_type):
    '''Get available service endpoints for a given service type from the
    Opencast ServiceRegistry.
    '''
    endpoint = '/services/available.json?serviceType=' + str(service_type)
    url = '%s%s' % (config()['server']['url'], endpoint)
    response = http_request(url).decode('utf-8')
    services = (json.loads(response).get('services') or {}).get('service', [])
    services = ensurelist(services)
    endpoints = [service['host'] + service['path'] for service in services
                 if service['online'] and service['active']]
    for endpoint in endpoints:
        logger.info(u'Endpoint for %s: %s', service_type, endpoint)
    return endpoints

def
get_service
(
service_type
)
:
endpoint
=
'/services/available.json?serviceType='
+
str
(
service_type
)
url
=
'%s%s'
%
(
config
(
)
[
'server'
]
[
'url'
]
,
endpoint
)
response
=
http_request
(
url
)
.
decode
(
'utf-8'
)
services
=
(
json
.
loads
(
response
)
.
get
(
'services'
)
or
{
}
)
.
get
(
'service'
,
[
]
)
services
=
ensurelist
(
services
)
endpoints
=
[
service
[
'host'
]
+
service
[
'path'
]
for
service
in
services
if
service
[
'online'
]
and
service
[
'active'
]
]
for
endpoint
in
endpoints
:
logger
.
info
(
u'Endpoint for %s: %s'
,
service_type
,
endpoint
)
return
endpoints

Get available service endpoints for a given service type from the Opencast ServiceRegistry .


def try_mkdir(directory):
    '''Try to create a directory. Pass without error if it already exists.
    '''
    try:
        os.mkdir(directory)
    except OSError as err:
        if err.errno != errno.EEXIST:
            raise err

def try_mkdir ( directory ) : try : os . mkdir ( directory ) except OSError as err : if err . errno != errno . EEXIST : raise err

Try to create a directory . Pass without error if it already exists .


def configure_service(service):
    '''Get the location of a given service from Opencast and add it to the
    current configuration.
    '''
    while not config().get('service-' + service) and not terminate():
        try:
            config()['service-' + service] = \
                get_service('org.opencastproject.' + service)
        except pycurl.error as e:
            logger.error('Could not get %s endpoint: %s. Retrying in 5s' %
                         (service, e))
            time.sleep(5.0)

Get the location of a given service from Opencast and add it to the current configuration .


def register_ca(status='idle'):
    '''Register this capture agent at the Matterhorn admin server so that it
    shows up in the admin interface.

    :param address: Address of the capture agent web ui
    :param status: Current status of the capture agent
    '''
    # If this is a backup CA we don't tell the Matterhorn core that we are
    # here.  We will just run silently in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('address', config()['ui']['url']), ('state', status)]
    name = urlquote(config()['agent']['name'].encode('utf-8'), safe='')
    url = '%s/agents/%s' % (config()['service-capture.admin'][0], name)
    try:
        response = http_request(url, params).decode('utf-8')
        if response:
            logger.info(response)
    except pycurl.error as e:
        logger.warning('Could not set agent state to %s: %s' % (status, e))

def
register_ca
(
status
=
'idle'
)
:
if
config
(
)
[
'agent'
]
[
'backup_mode'
]
:
return
params
=
[
(
'address'
,
config
(
)
[
'ui'
]
[
'url'
]
)
,
(
'state'
,
status
)
]
name
=
urlquote
(
config
(
)
[
'agent'
]
[
'name'
]
.
encode
(
'utf-8'
)
,
safe
=
''
)
url
=
'%s/agents/%s'
%
(
config
(
)
[
'service-capture.admin'
]
[
0
]
,
name
)
try
:
response
=
http_request
(
url
,
params
)
.
decode
(
'utf-8'
)
if
response
:
logger
.
info
(
response
)
except
pycurl
.
error
as
e
:
logger
.
warning
(
'Could not set agent state to %s: %s'
%
(
status
,
e
)
)

Register this capture agent at the Matterhorn admin server so that it shows up in the admin interface .


def recording_state(recording_id, status):
    '''Send the state of the current recording to the Matterhorn core.

    :param recording_id: ID of the current recording
    :param status: Status of the recording
    '''
    # If this is a backup CA we do not update the recording state since the
    # actual CA does that and we want to interfere.  We will just run silently
    # in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('state', status)]
    url = config()['service-capture.admin'][0]
    url += '/recordings/%s' % recording_id
    try:
        result = http_request(url, params)
        logger.info(result)
    except pycurl.error as e:
        logger.warning('Could not set recording state to %s: %s' % (status, e))

Send the state of the current recording to the Matterhorn core .


def update_event_status(event, status):
    '''Update the status of a particular event in the database.
    '''
    dbs = db.get_session()
    dbs.query(db.RecordedEvent).filter(db.RecordedEvent.start == event.start)\
                               .update({'status': status})
    event.status = status
    dbs.commit()

def update_event_status ( event , status ) : dbs = db . get_session ( ) dbs . query ( db . RecordedEvent ) . filter ( db . RecordedEvent . start == event . start ) . update ( { 'status' : status } ) event . status = status dbs . commit ( )

Update the status of a particular event in the database .


def update_agent_state():
    '''Update the current agent state in opencast.
    '''
    configure_service('capture.admin')
    status = 'idle'

    # Determine reported agent state with priority list
    if get_service_status(db.Service.SCHEDULE) == db.ServiceStatus.STOPPED:
        status = 'offline'
    elif get_service_status(db.Service.CAPTURE) == db.ServiceStatus.BUSY:
        status = 'capturing'
    elif get_service_status(db.Service.INGEST) == db.ServiceStatus.BUSY:
        status = 'uploading'

    register_ca(status=status)

def update_agent_state ( ) : configure_service ( 'capture.admin' ) status = 'idle' if get_service_status ( db . Service . SCHEDULE ) == db . ServiceStatus . STOPPED : status = 'offline' elif get_service_status ( db . Service . CAPTURE ) == db . ServiceStatus . BUSY : status = 'capturing' elif get_service_status ( db . Service . INGEST ) == db . ServiceStatus . BUSY : status = 'uploading' register_ca ( status = status )

Update the current agent state in opencast .


def configuration_file(cfgfile):
    '''Find the best match for the configuration file.
    '''
    if cfgfile is not None:
        return cfgfile
    # If no file is explicitely specified, probe for the configuration file
    # location.
    cfg = './etc/pyca.conf'
    if not os.path.isfile(cfg):
        return '/etc/pyca.conf'
    return cfg

def configuration_file ( cfgfile ) : if cfgfile is not None : return cfgfile cfg = './etc/pyca.conf' if not os . path . isfile ( cfg ) : return '/etc/pyca.conf' return cfg

Find the best match for the configuration file .


def update_configuration(cfgfile=None):
    '''Update configuration from file.

    :param cfgfile: Configuration file to load.
    '''
    configobj.DEFAULT_INTERPOLATION = 'template'
    cfgfile = configuration_file(cfgfile)
    cfg = configobj.ConfigObj(cfgfile, configspec=cfgspec, encoding='utf-8')
    validator = Validator()
    val = cfg.validate(validator)
    if val is not True:
        raise ValueError('Invalid configuration: %s' % val)
    if len(cfg['capture']['files']) != len(cfg['capture']['flavors']):
        raise ValueError('List of files and flavors do not match')
    globals()['__config'] = cfg
    logger_init()
    if cfg['server'].get('url', '').endswith('/'):
        logger.warning('Base URL ends with /. This is most likely a '
                       'configuration error. The URL should contain nothing '
                       'of the service paths.')
    logger.info('Configuration loaded from %s' % cfgfile)
    check()
    return cfg


def check():
    '''Check configuration for sanity.
    '''
    if config('server')['insecure']:
        logger.warning('HTTPS CHECKS ARE TURNED OFF. A SECURE CONNECTION IS '
                       'NOT GUARANTEED')
    if config('server')['certificate']:
        # Ensure certificate exists and is readable
        open(config('server')['certificate'], 'rb').close()
    if config('agent')['backup_mode']:
        logger.info('Agent runs in backup mode. No data will be sent to '
                    'Opencast')


def logger_init():
    '''Initialize logger based on configuration
    '''
    handlers = []
    logconf = config('logging')
    if logconf['syslog']:
        handlers.append(logging.handlers.SysLogHandler(address='/dev/log'))
    if logconf['stderr']:
        handlers.append(logging.StreamHandler(sys.stderr))
    if logconf['file']:
        handlers.append(logging.handlers.WatchedFileHandler(logconf['file']))
    for handler in handlers:
        handler.setFormatter(logging.Formatter(logconf['format']))
        logging.root.addHandler(handler)

    logging.root.setLevel(logconf['level'].upper())
    logger.info('Log level set to %s' % logconf['level'])

def
logger_init
(
)
:
handlers
=
[
]
logconf
=
config
(
'logging'
)
if
logconf
[
'syslog'
]
:
handlers
.
append
(
logging
.
handlers
.
SysLogHandler
(
address
=
'/dev/log'
)
)
if
logconf
[
'stderr'
]
:
handlers
.
append
(
logging
.
StreamHandler
(
sys
.
stderr
)
)
if
logconf
[
'file'
]
:
handlers
.
append
(
logging
.
handlers
.
WatchedFileHandler
(
logconf
[
'file'
]
)
)
for
handler
in
handlers
:
handler
.
setFormatter
(
logging
.
Formatter
(
logconf
[
'format'
]
)
)
logging
.
root
.
addHandler
(
handler
)
logging
.
root
.
setLevel
(
logconf
[
'level'
]
.
upper
(
)
)
logger
.
info
(
'Log level set to %s'
%
logconf
[
'level'
]
)


def home():
    '''Serve the status page of the capture agent.
    '''
    # Get IDs of existing preview images
    preview = config()['capture']['preview']
    previewdir = config()['capture']['preview_dir']
    preview = [p.replace('{{previewdir}}', previewdir) for p in preview]
    preview = zip(preview, range(len(preview)))
    preview = [p[1] for p in preview if os.path.isfile(p[0])]

    # Get limits for recording table
    try:
        limit_upcoming = int(request.args.get('limit_upcoming', 5))
        limit_processed = int(request.args.get('limit_processed', 15))
    except ValueError:
        limit_upcoming = 5
        limit_processed = 15

    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)\
                        .limit(limit_upcoming)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())\
                        .limit(limit_processed)
    recording = get_service_status(Service.CAPTURE) == ServiceStatus.BUSY
    uploading = get_service_status(Service.INGEST) == ServiceStatus.BUSY
    processed = db.query(RecordedEvent).count()
    upcoming = db.query(UpcomingEvent).count()

    return render_template('home.html', preview=preview, config=config(),
                           recorded_events=recorded_events,
                           upcoming_events=upcoming_events,
                           recording=recording, uploading=uploading,
                           processed=processed, upcoming=upcoming,
                           limit_upcoming=limit_upcoming,
                           limit_processed=limit_processed,
                           dtfmt=dtfmt)

def home ( ) : preview = config ( ) [ 'capture' ] [ 'preview' ] previewdir = config ( ) [ 'capture' ] [ 'preview_dir' ] preview = [ p . replace ( '{{previewdir}}' , previewdir ) for p in preview ] preview = zip ( preview , range ( len ( preview ) ) ) preview = [ p [ 1 ] for p in preview if os . path . isfile ( p [ 0 ] ) ] try : limit_upcoming = int ( request . args . get ( 'limit_upcoming' , 5 ) ) limit_processed = int ( request . args . get ( 'limit_processed' , 15 ) ) except ValueError : limit_upcoming = 5 limit_processed = 15 db = get_session ( ) upcoming_events = db . query ( UpcomingEvent ) . order_by ( UpcomingEvent . start ) . limit ( limit_upcoming ) recorded_events = db . query ( RecordedEvent ) . order_by ( RecordedEvent . start . desc ( ) ) . limit ( limit_processed ) recording = get_service_status ( Service . CAPTURE ) == ServiceStatus . BUSY uploading = get_service_status ( Service . INGEST ) == ServiceStatus . BUSY processed = db . query ( RecordedEvent ) . count ( ) upcoming = db . query ( UpcomingEvent ) . count ( ) return render_template ( 'home.html' , preview = preview , config = config ( ) , recorded_events = recorded_events , upcoming_events = upcoming_events , recording = recording , uploading = uploading , processed = processed , upcoming = upcoming , limit_upcoming = limit_upcoming , limit_processed = limit_processed , dtfmt = dtfmt )

Serve the status page of the capture agent .


def serve_image(image_id):
    '''Serve the preview image with the given id
    '''
    try:
        preview_dir = config()['capture']['preview_dir']
        filepath = config()['capture']['preview'][image_id]
        filepath = filepath.replace('{{previewdir}}', preview_dir)
        filepath = os.path.abspath(filepath)
        if os.path.isfile(filepath):
            directory, filename = filepath.rsplit('/', 1)
            return send_from_directory(directory, filename)
    except (IndexError, KeyError):
        pass
    return '', 404

def serve_image ( image_id ) : try : preview_dir = config ( ) [ 'capture' ] [ 'preview_dir' ] filepath = config ( ) [ 'capture' ] [ 'preview' ] [ image_id ] filepath = filepath . replace ( '{{previewdir}}' , preview_dir ) filepath = os . path . abspath ( filepath ) if os . path . isfile ( filepath ) : directory , filename = filepath . rsplit ( '/' , 1 ) return send_from_directory ( directory , filename ) except ( IndexError , KeyError ) : pass return '' , 404


def run_all(*modules):
    '''Start all services.
    '''
    processes = [multiprocessing.Process(target=mod.run) for mod in modules]
    for p in processes:
        p.start()
    for p in processes:
        p.join()

def run_all ( * modules ) : processes = [ multiprocessing . Process ( target = mod . run ) for mod in modules ] for p in processes : p . start ( ) for p in processes : p . join ( )


def parse_ical(vcal):
    '''Parse Opencast schedule iCalendar file and return events as dict
    '''
    vcal = vcal.replace('\r\n ', '').replace('\r\n\r\n', '\r\n')
    vevents = vcal.split('\r\nBEGIN:VEVENT\r\n')
    del(vevents[0])
    events = []
    for vevent in vevents:
        event = {}
        for line in vevent.split('\r\n'):
            line = line.split(':', 1)
            key = line[0].lower()
            if len(line) <= 1 or key == 'end':
                continue
            if key.startswith('dt'):
                event[key] = unix_ts(dateutil.parser.parse(line[1]))
                continue
            if not key.startswith('attach'):
                event[key] = line[1]
                continue
            # finally handle attachments
            event['attach'] = event.get('attach', [])
            attachment = {}
            for x in [x.split('=') for x in line[0].split(';')]:
                if x[0].lower() in ['fmttype', 'x-apple-filename']:
                    attachment[x[0].lower()] = x[1]
            attachment['data'] = b64decode(line[1]).decode('utf-8')
            event['attach'].append(attachment)
        events.append(event)
    return events

def
parse_ical
(
vcal
)
:
vcal
=
vcal
.
replace
(
'\r\n '
,
''
)
.
replace
(
'\r\n\r\n'
,
'\r\n'
)
vevents
=
vcal
.
split
(
'\r\nBEGIN:VEVENT\r\n'
)
del
(
vevents
[
0
]
)
events
=
[
]
for
vevent
in
vevents
:
event
=
{
}
for
line
in
vevent
.
split
(
'\r\n'
)
:
line
=
line
.
split
(
':'
,
1
)
key
=
line
[
0
]
.
lower
(
)
if
len
(
line
)
<=
1
or
key
==
'end'
:
continue
if
key
.
startswith
(
'dt'
)
:
event
[
key
]
=
unix_ts
(
dateutil
.
parser
.
parse
(
line
[
1
]
)
)
continue
if
not
key
.
startswith
(
'attach'
)
:
event
[
key
]
=
line
[
1
]
continue
event
[
'attach'
]
=
event
.
get
(
'attach'
,
[
]
)
attachment
=
{
}
for
x
in
[
x
.
split
(
'='
)
for
x
in
line
[
0
]
.
split
(
';'
)
]
:
if
x
[
0
]
.
lower
(
)
in
[
'fmttype'
,
'x-apple-filename'
]
:
attachment
[
x
[
0
]
.
lower
(
)
]
=
x
[
1
]
attachment
[
'data'
]
=
b64decode
(
line
[
1
]
)
.
decode
(
'utf-8'
)
event
[
'attach'
]
.
append
(
attachment
)
events
.
append
(
event
)
return
events


def get_schedule():
    '''Try to load schedule from the Matterhorn core. Returns a valid schedule
    or None on failure.
    '''
    params = {'agentid': config()['agent']['name'].encode('utf8')}
    lookahead = config()['agent']['cal_lookahead'] * 24 * 60 * 60
    if lookahead:
        params['cutoff'] = str((timestamp() + lookahead) * 1000)
    uri = '%s/calendars?%s' % (config()['service-scheduler'][0],
                               urlencode(params))
    try:
        vcal = http_request(uri)
    except pycurl.error as e:
        logger.error('Could not get schedule: %s' % e)
        return

    try:
        cal = parse_ical(vcal.decode('utf-8'))
    except Exception:
        logger.error('Could not parse ical')
        logger.error(traceback.format_exc())
        return
    db = get_session()
    db.query(UpcomingEvent).delete()
    for event in cal:
        # Ignore events that have already ended
        if event['dtend'] <= timestamp():
            continue
        e = UpcomingEvent()
        e.start = event['dtstart']
        e.end = event['dtend']
        e.uid = event.get('uid')
        e.title = event.get('summary')
        e.set_data(event)
        db.add(e)
    db.commit()

def
get_schedule
(
)
:
params
=
{
'agentid'
:
config
(
)
[
'agent'
]
[
'name'
]
.
encode
(
'utf8'
)
}
lookahead
=
config
(
)
[
'agent'
]
[
'cal_lookahead'
]
*
24
*
60
*
60
if
lookahead
:
params
[
'cutoff'
]
=
str
(
(
timestamp
(
)
+
lookahead
)
*
1000
)
uri
=
'%s/calendars?%s'
%
(
config
(
)
[
'service-scheduler'
]
[
0
]
,
urlencode
(
params
)
)
try
:
vcal
=
http_request
(
uri
)
except
pycurl
.
error
as
e
:
logger
.
error
(
'Could not get schedule: %s'
%
e
)
return
try
:
cal
=
parse_ical
(
vcal
.
decode
(
'utf-8'
)
)
except
Exception
:
logger
.
error
(
'Could not parse ical'
)
logger
.
error
(
traceback
.
format_exc
(
)
)
return
db
=
get_session
(
)
db
.
query
(
UpcomingEvent
)
.
delete
(
)
for
event
in
cal
:
if
event
[
'dtend'
]
<=
timestamp
(
)
:
continue
e
=
UpcomingEvent
(
)
e
.
start
=
event
[
'dtstart'
]
e
.
end
=
event
[
'dtend'
]
e
.
uid
=
event
.
get
(
'uid'
)
e
.
title
=
event
.
get
(
'summary'
)
e
.
set_data
(
event
)
db
.
add
(
e
)
db
.
commit
(
)

Try to load schedule from the Matterhorn core . Returns a valid schedule or None on failure .


def control_loop():
    '''Main loop, retrieving the schedule.
    '''
    set_service_status(Service.SCHEDULE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    while not terminate():
        notify.notify('WATCHDOG=1')
        # Try getting an updated schedule
        get_schedule()
        session = get_session()
        next_event = session.query(UpcomingEvent)\
                            .filter(UpcomingEvent.end > timestamp())\
                            .order_by(UpcomingEvent.start)\
                            .first()
        if next_event:
            logger.info('Next scheduled recording: %s',
                        datetime.fromtimestamp(next_event.start))
            notify.notify('STATUS=Next scheduled recording: %s' %
                          datetime.fromtimestamp(next_event.start))
        else:
            logger.info('No scheduled recording')
            notify.notify('STATUS=No scheduled recording')
        session.close()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down schedule service')
    set_service_status(Service.SCHEDULE, ServiceStatus.STOPPED)

def
control_loop
(
)
:
set_service_status
(
Service
.
SCHEDULE
,
ServiceStatus
.
BUSY
)
notify
.
notify
(
'READY=1'
)
while
not
terminate
(
)
:
notify
.
notify
(
'WATCHDOG=1'
)
get_schedule
(
)
session
=
get_session
(
)
next_event
=
session
.
query
(
UpcomingEvent
)
.
filter
(
UpcomingEvent
.
end
>
timestamp
(
)
)
.
order_by
(
UpcomingEvent
.
start
)
.
first
(
)
if
next_event
:
logger
.
info
(
'Next scheduled recording: %s'
,
datetime
.
fromtimestamp
(
next_event
.
start
)
)
notify
.
notify
(
'STATUS=Next scheduled recording: %s'
%
datetime
.
fromtimestamp
(
next_event
.
start
)
)
else
:
logger
.
info
(
'No scheduled recording'
)
notify
.
notify
(
'STATUS=No scheduled recording'
)
session
.
close
(
)
next_update
=
timestamp
(
)
+
config
(
)
[
'agent'
]
[
'update_frequency'
]
while
not
terminate
(
)
and
timestamp
(
)
<
next_update
:
time
.
sleep
(
0.1
)
logger
.
info
(
'Shutting down schedule service'
)
set_service_status
(
Service
.
SCHEDULE
,
ServiceStatus
.
STOPPED
)


def control_loop():
    '''Main loop, updating the capture agent state.
    '''
    set_service_status(Service.AGENTSTATE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    notify.notify('STATUS=Running')
    while not terminate():
        notify.notify('WATCHDOG=1')
        update_agent_state()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down agentstate service')
    set_service_status(Service.AGENTSTATE, ServiceStatus.STOPPED)

def
control_loop
(
)
:
set_service_status
(
Service
.
AGENTSTATE
,
ServiceStatus
.
BUSY
)
notify
.
notify
(
'READY=1'
)
notify
.
notify
(
'STATUS=Running'
)
while
not
terminate
(
)
:
notify
.
notify
(
'WATCHDOG=1'
)
update_agent_state
(
)
next_update
=
timestamp
(
)
+
config
(
)
[
'agent'
]
[
'update_frequency'
]
while
not
terminate
(
)
and
timestamp
(
)
<
next_update
:
time
.
sleep
(
0.1
)
logger
.
info
(
'Shutting down agentstate service'
)
set_service_status
(
Service
.
AGENTSTATE
,
ServiceStatus
.
STOPPED
)

Main loop updating the capture agent state .


def make_error_response(error, status=500):
    ''' Return a response with a jsonapi error object
    '''
    content = {
        'errors': [{
            'status': status,
            'title': error
        }]
    }
    return make_response(jsonify(content), status)

def make_error_response ( error , status = 500 ) : content = { 'errors' : [ { 'status' : status , 'title' : error } ] } return make_response ( jsonify ( content ) , status )


def make_data_response(data, status=200):
    ''' Return a response with a list of jsonapi data objects
    '''
    content = {'data': ensurelist(data)}
    return make_response(jsonify(content), status)

def make_data_response ( data , status = 200 ) : content = { 'data' : ensurelist ( data ) } return make_response ( jsonify ( content ) , status )


def internal_state():
    '''Serve a json representation of internal agentstate as meta data
    '''
    data = {'services': {
        'capture': ServiceStatus.str(get_service_status(Service.CAPTURE)),
        'ingest': ServiceStatus.str(get_service_status(Service.INGEST)),
        'schedule': ServiceStatus.str(get_service_status(Service.SCHEDULE)),
        'agentstate': ServiceStatus.str(get_service_status(Service.AGENTSTATE))
        }
    }
    return make_response(jsonify({'meta': data}))

def internal_state ( ) : data = { 'services' : { 'capture' : ServiceStatus . str ( get_service_status ( Service . CAPTURE ) ) , 'ingest' : ServiceStatus . str ( get_service_status ( Service . INGEST ) ) , 'schedule' : ServiceStatus . str ( get_service_status ( Service . SCHEDULE ) ) , 'agentstate' : ServiceStatus . str ( get_service_status ( Service . AGENTSTATE ) ) } } return make_response ( jsonify ( { 'meta' : data } ) )


def events():
    '''Serve a JSON representation of events
    '''
    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())

    result = [event.serialize() for event in upcoming_events]
    result += [event.serialize() for event in recorded_events]
    return make_data_response(result)

def events ( ) : db = get_session ( ) upcoming_events = db . query ( UpcomingEvent ) . order_by ( UpcomingEvent . start ) recorded_events = db . query ( RecordedEvent ) . order_by ( RecordedEvent . start . desc ( ) ) result = [ event . serialize ( ) for event in upcoming_events ] result += [ event . serialize ( ) for event in recorded_events ] return make_data_response ( result )


def event(uid):
    '''Return a specific events JSON
    '''
    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first() \
        or db.query(UpcomingEvent).filter(UpcomingEvent.uid == uid).first()

    if event:
        return make_data_response(event.serialize())
    return make_error_response('No event with specified uid', 404)

def
event
(
uid
)
:
db
=
get_session
(
)
event
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
.
first
(
)
or
db
.
query
(
UpcomingEvent
)
.
filter
(
UpcomingEvent
.
uid
==
uid
)
.
first
(
)
if
event
:
return
make_data_response
(
event
.
serialize
(
)
)
return
make_error_response
(
'No event with specified uid'
,
404
)


def delete_event(uid):
    '''Delete a specific event identified by its uid. Note that only recorded
    events can be deleted. Events in the buffer for upcoming events are
    regularly replaced anyway and a manual removal could have unpredictable
    effects.

    Use ?hard=true parameter to delete the recorded files on disk as well.

    Returns 204 if the action was successful.
    Returns 404 if event does not exist
    '''
    logger.info('deleting event %s via api', uid)
    db = get_session()
    events = db.query(RecordedEvent).filter(RecordedEvent.uid == uid)
    if not events.count():
        return make_error_response('No event with specified uid', 404)
    hard_delete = request.args.get('hard', 'false')
    if hard_delete == 'true':
        logger.info('deleting recorded files at %s', events[0].directory())
        shutil.rmtree(events[0].directory())
    events.delete()
    db.commit()
    return make_response('', 204)

def
delete_event
(
uid
)
:
logger
.
info
(
'deleting event %s via api'
,
uid
)
db
=
get_session
(
)
events
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
if
not
events
.
count
(
)
:
return
make_error_response
(
'No event with specified uid'
,
404
)
hard_delete
=
request
.
args
.
get
(
'hard'
,
'false'
)
if
hard_delete
==
'true'
:
logger
.
info
(
'deleting recorded files at %s'
,
events
[
0
]
.
directory
(
)
)
shutil
.
rmtree
(
events
[
0
]
.
directory
(
)
)
events
.
delete
(
)
db
.
commit
(
)
return
make_response
(
''
,
204
)

Delete a specific event identified by its uid . Note that only recorded events can be deleted . Events in the buffer for upcoming events are regularly replaced anyway and a manual removal could have unpredictable effects .


def modify_event(uid):
    '''Modify an event specified by its uid. The modifications for the event
    are expected as JSON with the content type correctly set in the request.

    Note that this method works for recorded events only. Upcoming events part
    of the scheduler cache cannot be modified.
    '''
    try:
        data = request.get_json()['data'][0]
        if data['type'] != 'event' or data['id'] != uid:
            return make_error_response('Invalid data', 400)
        # Check attributes
        for key in data['attributes'].keys():
            if key not in ('status', 'start', 'end'):
                return make_error_response('Invalid data', 400)
        # Check new status
        new_status = data['attributes'].get('status')
        if new_status:
            new_status = new_status.upper().replace(' ', '_')
            data['attributes']['status'] = int(getattr(Status, new_status))
    except Exception:
        return make_error_response('Invalid data', 400)

    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first()
    if not event:
        return make_error_response('No event with specified uid', 404)
    event.start = data['attributes'].get('start', event.start)
    event.end = data['attributes'].get('end', event.end)
    event.status = data['attributes'].get('status', event.status)
    logger.debug('Updating event %s via api', uid)
    db.commit()
    return make_data_response(event.serialize())

def
modify_event
(
uid
)
:
try
:
data
=
request
.
get_json
(
)
[
'data'
]
[
0
]
if
data
[
'type'
]
!=
'event'
or
data
[
'id'
]
!=
uid
:
return
make_error_response
(
'Invalid data'
,
400
)
for
key
in
data
[
'attributes'
]
.
keys
(
)
:
if
key
not
in
(
'status'
,
'start'
,
'end'
)
:
return
make_error_response
(
'Invalid data'
,
400
)
new_status
=
data
[
'attributes'
]
.
get
(
'status'
)
if
new_status
:
new_status
=
new_status
.
upper
(
)
.
replace
(
' '
,
'_'
)
data
[
'attributes'
]
[
'status'
]
=
int
(
getattr
(
Status
,
new_status
)
)
except
Exception
:
return
make_error_response
(
'Invalid data'
,
400
)
db
=
get_session
(
)
event
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
.
first
(
)
if
not
event
:
return
make_error_response
(
'No event with specified uid'
,
404
)
event
.
start
=
data
[
'attributes'
]
.
get
(
'start'
,
event
.
start
)
event
.
end
=
data
[
'attributes'
]
.
get
(
'end'
,
event
.
end
)
event
.
status
=
data
[
'attributes'
]
.
get
(
'status'
,
event
.
status
)
logger
.
debug
(
'Updating event %s via api'
,
uid
)
db
.
commit
(
)
return
make_data_response
(
event
.
serialize
(
)
)

Modify an event specified by its uid . The modifications for the event are expected as JSON with the content type correctly set in the request .


def get_config_params(properties):
    '''Extract the set of configuration parameters from the properties attached
    to the schedule
    '''
    param = []
    wdef = ''
    for prop in properties.split('\n'):
        if prop.startswith('org.opencastproject.workflow.config'):
            key, val = prop.split('=', 1)
            key = key.split('.')[-1]
            param.append((key, val))
        elif prop.startswith('org.opencastproject.workflow.definition'):
            wdef = prop.split('=', 1)[-1]
    return wdef, param


def ingest(event):
    '''Ingest a finished recording to the Opencast server.
    '''
    # Update status
    set_service_status(Service.INGEST, ServiceStatus.BUSY)
    notify.notify('STATUS=Uploading')
    recording_state(event.uid, 'uploading')
    update_event_status(event, Status.UPLOADING)

    # Select ingest service
    # The ingest service to use is selected at random from the available
    # ingest services to ensure that not every capture agent uses the same
    # service at the same time
    service = config('service-ingest')
    service = service[randrange(0, len(service))]
    logger.info('Selecting ingest service to use: ' + service)

    # create mediapackage
    logger.info('Creating new mediapackage')
    mediapackage = http_request(service + '/createMediaPackage')

    # extract workflow_def, workflow_config and add DC catalogs
    prop = 'org.opencastproject.capture.agent.properties'
    dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/'
    for attachment in event.get_data().get('attach'):
        data = attachment.get('data')
        if attachment.get('x-apple-filename') == prop:
            workflow_def, workflow_config = get_config_params(data)

        # Check for dublincore catalogs
        elif attachment.get('fmttype') == 'application/xml' and dcns in data:
            name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0]
            logger.info('Adding %s DC catalog' % name)
            fields = [('mediaPackage', mediapackage),
                      ('flavor', 'dublincore/%s' % name),
                      ('dublinCore', data.encode('utf-8'))]
            mediapackage = http_request(service + '/addDCCatalog', fields)

    # add track
    for (flavor, track) in event.get_tracks():
        logger.info('Adding track ({0} -> {1})'.format(flavor, track))
        track = track.encode('ascii', 'ignore')
        fields = [('mediaPackage', mediapackage), ('flavor', flavor),
                  ('BODY1', (pycurl.FORM_FILE, track))]
        mediapackage = http_request(service + '/addTrack', fields)

    # ingest
    logger.info('Ingest recording')
    fields = [('mediaPackage', mediapackage)]
    if workflow_def:
        fields.append(('workflowDefinitionId', workflow_def))
    if event.uid:
        fields.append(('workflowInstanceId',
                       event.uid.encode('ascii', 'ignore')))
    fields += workflow_config
    mediapackage = http_request(service + '/ingest', fields)

    # Update status
    recording_state(event.uid, 'upload_finished')
    update_event_status(event, Status.FINISHED_UPLOADING)
    notify.notify('STATUS=Running')
    set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)

    logger.info('Finished ingest')

Ingest a finished recording to the Opencast server .


def start_capture(upcoming_event):
    '''Start the capture process, creating all necessary files and directories
    as well as ingesting the captured files if no backup mode is configured.
    '''
    logger.info('Start recording')

    # First move event to recording_event table
    db = get_session()
    event = db.query(RecordedEvent)\
              .filter(RecordedEvent.uid == upcoming_event.uid)\
              .filter(RecordedEvent.start == upcoming_event.start)\
              .first()
    if not event:
        event = RecordedEvent(upcoming_event)
        db.add(event)
        db.commit()

    try_mkdir(config()['capture']['directory'])
    os.mkdir(event.directory())

    # Set state
    update_event_status(event, Status.RECORDING)
    recording_state(event.uid, 'capturing')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)

    # Recording
    tracks = recording_command(event)
    event.set_tracks(tracks)
    db.commit()

    # Set status
    update_event_status(event, Status.FINISHED_RECORDING)
    recording_state(event.uid, 'capture_finished')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)

    logger.info('Finished recording')

def
start_capture
(
upcoming_event
)
:
logger
.
info
(
'Start recording'
)
db
=
get_session
(
)
event
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
upcoming_event
.
uid
)
.
filter
(
RecordedEvent
.
start
==
upcoming_event
.
start
)
.
first
(
)
if
not
event
:
event
=
RecordedEvent
(
upcoming_event
)
db
.
add
(
event
)
db
.
commit
(
)
try_mkdir
(
config
(
)
[
'capture'
]
[
'directory'
]
)
os
.
mkdir
(
event
.
directory
(
)
)
update_event_status
(
event
,
Status
.
RECORDING
)
recording_state
(
event
.
uid
,
'capturing'
)
set_service_status_immediate
(
Service
.
CAPTURE
,
ServiceStatus
.
BUSY
)
tracks
=
recording_command
(
event
)
event
.
set_tracks
(
tracks
)
db
.
commit
(
)
update_event_status
(
event
,
Status
.
FINISHED_RECORDING
)
recording_state
(
event
.
uid
,
'capture_finished'
)
set_service_status_immediate
(
Service
.
CAPTURE
,
ServiceStatus
.
IDLE
)
logger
.
info
(
'Finished recording'
)

Start the capture process creating all necessary files and directories as well as ingesting the captured files if no backup mode is configured .


def render_to_fragment(self, request, **kwargs):
        """
        Returns a simple fragment
        """
        fragment = Fragment(TEST_HTML)
        fragment.add_javascript(TEST_JS)
        fragment.add_css(TEST_CSS)
        return fragment

def render_to_fragment ( self , request , ** kwargs ) : fragment = Fragment ( TEST_HTML ) fragment . add_javascript ( TEST_JS ) fragment . add_css ( TEST_CSS ) return fragment


def resources(self):
        """
        Returns list of unique `FragmentResource`s by order of first appearance.
        """
        seen = set()
        # seen.add always returns None, so 'not seen.add(x)' is always True,
        # but will only be called if the value is not already in seen (because
        # 'and' short-circuits)
        return [x for x in self._resources if x not in seen and not seen.add(x)]

def resources ( self ) : seen = set ( ) return [ x for x in self . _resources if x not in seen and not seen . add ( x ) ]

Returns list of unique FragmentResource s by order of first appearance .


def to_dict(self):
        """
        Returns the fragment in a dictionary representation.
        """
        return {
            'content': self.content,
            'resources': [r._asdict() for r in self.resources],  # pylint: disable=W0212
            'js_init_fn': self.js_init_fn,
            'js_init_version': self.js_init_version,
            'json_init_args': self.json_init_args
        }

def to_dict ( self ) : return { 'content' : self . content , 'resources' : [ r . _asdict ( ) for r in self . resources ] , 'js_init_fn' : self . js_init_fn , 'js_init_version' : self . js_init_version , 'json_init_args' : self . json_init_args }

Returns the fragment in a dictionary representation .


def from_dict(cls, pods):
        """
        Returns a new Fragment from a dictionary representation.
        """
        frag = cls()
        frag.content = pods['content']
        frag._resources = [FragmentResource(**d) for d in pods['resources']]  # pylint: disable=protected-access
        frag.js_init_fn = pods['js_init_fn']
        frag.js_init_version = pods['js_init_version']
        frag.json_init_args = pods['json_init_args']
        return frag

def from_dict ( cls , pods ) : frag = cls ( ) frag . content = pods [ 'content' ] frag . _resources = [ FragmentResource ( ** d ) for d in pods [ 'resources' ] ] frag . js_init_fn = pods [ 'js_init_fn' ] frag . js_init_version = pods [ 'js_init_version' ] frag . json_init_args = pods [ 'json_init_args' ] return frag

Returns a new Fragment from a dictionary representation .


def add_content(self, content):
        """
        Add content to this fragment.

        `content` is a Unicode string, HTML to append to the body of the
        fragment.  It must not contain a ``<body>`` tag, or otherwise assume
        that it is the only content on the page.
        """
        assert isinstance(content, six.text_type)
        self.content += content

def add_content ( self , content ) : assert isinstance ( content , six . text_type ) self . content += content


def add_resource(self, text, mimetype, placement=None):
        """
        Add a resource needed by this Fragment.

        Other helpers, such as :func:`add_css` or :func:`add_javascript` are
        more convenient for those common types of resource.

        `text`: the actual text of this resource, as a unicode string.

        `mimetype`: the MIME type of the resource.

        `placement`: where on the page the resource should be placed:

            None: let the Fragment choose based on the MIME type.

            "head": put this resource in the ``<head>`` of the page.

            "foot": put this resource at the end of the ``<body>`` of the
            page.

        """
        if not placement:
            placement = self._default_placement(mimetype)
        res = FragmentResource('text', text, mimetype, placement)
        self._resources.append(res)

def add_resource ( self , text , mimetype , placement = None ) : if not placement : placement = self . _default_placement ( mimetype ) res = FragmentResource ( 'text' , text , mimetype , placement ) self . _resources . append ( res )

Add a resource needed by this Fragment .


def add_resource_url(self, url, mimetype, placement=None):
        """
        Add a resource by URL needed by this Fragment.

        Other helpers, such as :func:`add_css_url` or
        :func:`add_javascript_url` are more convenent for those common types of
        resource.

        `url`: the URL to the resource.

        Other parameters are as defined for :func:`add_resource`.
        """
        if not placement:
            placement = self._default_placement(mimetype)
        self._resources.append(FragmentResource('url', url, mimetype, placement))

def add_resource_url ( self , url , mimetype , placement = None ) : if not placement : placement = self . _default_placement ( mimetype ) self . _resources . append ( FragmentResource ( 'url' , url , mimetype , placement ) )

Add a resource by URL needed by this Fragment .