def list_files(self, dataset_id, glob=".", is_dir=False):
        """
        List matched filenames in a dataset on Citrination.

        :param dataset_id: The ID of the dataset to search for files.
        :type dataset_id: int
        :param glob: A pattern which will be matched against files in the dataset.
        :type glob: str
        :param is_dir: A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset.
        :type is_dir: bool
        :return: A list of filepaths in the dataset matching the provided glob.
        :rtype: list of strings
        """
        data = {
            "list": {
                "glob": glob,
                "isDir": is_dir
            }
        }
        return self._get_success_json(self._post_json(routes.list_files(dataset_id), data, failure_message="Failed to list files for dataset {}".format(dataset_id)))['files']

def
list_files
(
self
,
dataset_id
,
glob
=
"."
,
is_dir
=
False
)
:
data
=
{
"list"
:
{
"glob"
:
glob
,
"isDir"
:
is_dir
}
}
return
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
list_files
(
dataset_id
)
,
data
,
failure_message
=
"Failed to list files for dataset {}"
.
format
(
dataset_id
)
)
)
[
'files'
]

List matched filenames in a dataset on Citrination .


def matched_file_count(self, dataset_id, glob=".", is_dir=False):
        """
        Returns the number of files matching a pattern in a dataset.

        :param dataset_id: The ID of the dataset to search for files.
        :type dataset_id: int
        :param glob: A pattern which will be matched against files in the dataset.
        :type glob: str
        :param is_dir: A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset.
        :type is_dir: bool
        :return: The number of matching files
        :rtype: int
        """
        list_result = self.list_files(dataset_id, glob, is_dir)
        return len(list_result)

def matched_file_count ( self , dataset_id , glob = "." , is_dir = False ) : list_result = self . list_files ( dataset_id , glob , is_dir ) return len ( list_result )

Returns the number of files matching a pattern in a dataset .


def get_dataset_files(self, dataset_id, glob=".", is_dir=False, version_number=None):
        """
        Retrieves URLs for the files matched by a glob or a path to a directory
        in a given dataset.

        :param dataset_id: The id of the dataset to retrieve files from
        :type dataset_id: int
        :param glob: A regex used to select one or more files in the dataset
        :type glob: str
        :param is_dir: Whether or not the supplied pattern should be treated as a directory to search in
        :type is_dir: bool
        :param version_number: The version number of the dataset to retrieve files from
        :type version_number: int
        :return: A list of dataset files whose paths match the provided pattern.
        :rtype: list of :class:`DatasetFile`
        """
        if version_number is None:
            latest = True
        else:
            latest = False

        data = {
            "download_request": {
                "glob": glob,
                "isDir": is_dir,
                "latest": latest
            }
        }

        failure_message = "Failed to get matched files in dataset {}".format(dataset_id)

        versions = self._get_success_json(self._post_json(routes.matched_files(dataset_id), data, failure_message=failure_message))['versions']

        # if you don't provide a version number, only the latest
        # will be included in the response body
        if version_number is None:
            version = versions[0]
        else:
            try:
                version = list(filter(lambda v: v['number'] == version_number, versions))[0]
            except IndexError:
                raise ResourceNotFoundException()

        return list(
            map(
                lambda f: DatasetFile(path=f['filename'], url=f['url']), version['files']
                )
            )

Retrieves URLs for the files matched by a glob or a path to a directory in a given dataset .


def get_dataset_file(self, dataset_id, file_path, version = None):
        """
        Retrieves a dataset file matching a provided file path

        :param dataset_id: The id of the dataset to retrieve file from
        :type dataset_id: int
        :param file_path: The file path within the dataset
        :type file_path: str
        :param version: The dataset version to look for the file in. If nothing is supplied, the latest dataset version will be searched
        :type version: int
        :return: A dataset file matching the filepath provided
        :rtype: :class:`DatasetFile`
        """
        return self.get_dataset_files(dataset_id, "^{}$".format(file_path), version_number=version)[0]

def get_dataset_file ( self , dataset_id , file_path , version = None ) : return self . get_dataset_files ( dataset_id , "^{}$" . format ( file_path ) , version_number = version ) [ 0 ]


def get_pif(self, dataset_id, uid, dataset_version = None):
        """
        Retrieves a PIF from a given dataset.

        :param dataset_id: The id of the dataset to retrieve PIF from
        :type dataset_id: int
        :param uid: The uid of the PIF to retrieve
        :type uid: str
        :param dataset_version: The dataset version to look for the PIF in. If nothing is supplied, the latest dataset version will be searched
        :type dataset_version: int
        :return: A :class:`Pif` object
        :rtype: :class:`Pif`
        """
        failure_message = "An error occurred retrieving PIF {}".format(uid)
        if dataset_version == None:
            response = self._get(routes.pif_dataset_uid(dataset_id, uid), failure_message=failure_message)
        else:
            response = self._get(routes.pif_dataset_version_uid(dataset_id, uid, dataset_version), failure_message=failure_message)

        return pif.loads(response.content.decode("utf-8"))

def
get_pif
(
self
,
dataset_id
,
uid
,
dataset_version
=
)
:
failure_message
=
"An error occurred retrieving PIF {}"
.
format
(
uid
)
if
dataset_version
==
:
response
=
self
.
_get
(
routes
.
pif_dataset_uid
(
dataset_id
,
uid
)
,
failure_message
=
failure_message
)
else
:
response
=
self
.
_get
(
routes
.
pif_dataset_version_uid
(
dataset_id
,
uid
,
dataset_version
)
,
failure_message
=
failure_message
)
return
pif
.
loads
(
response
.
content
.
decode
(
"utf-8"
)
)


def create_dataset(self, name=None, description=None, public=False):
        """
        Create a new data set.

        :param name: name of the dataset
        :type name: str
        :param description: description for the dataset
        :type description: str
        :param public: A boolean indicating whether or not the dataset should be public.
        :type public: bool
        :return: The newly created dataset.
        :rtype: :class:`Dataset`
        """
        data = {
            "public": _convert_bool_to_public_value(public)
        }
        if name:
            data["name"] = name
        if description:
            data["description"] = description
        dataset = {"dataset": data}
        failure_message = "Unable to create dataset"
        result = self._get_success_json(self._post_json(routes.create_dataset(), dataset, failure_message=failure_message))

        return _dataset_from_response_dict(result)

def
create_dataset
(
self
,
name
=
,
description
=
,
public
=
False
)
:
data
=
{
"public"
:
_convert_bool_to_public_value
(
public
)
}
if
name
:
data
[
"name"
]
=
name
if
description
:
data
[
"description"
]
=
description
dataset
=
{
"dataset"
:
data
}
failure_message
=
"Unable to create dataset"
result
=
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
create_dataset
(
)
,
dataset
,
failure_message
=
failure_message
)
)
return
_dataset_from_response_dict
(
result
)


def update_dataset(self, dataset_id, name=None, description=None, public=None):
        """
        Update a data set.

        :param dataset_id: The ID of the dataset to update
        :type dataset_id: int
        :param name: name of the dataset
        :type name: str
        :param description: description for the dataset
        :type description: str
        :param public: A boolean indicating whether or not the dataset should
            be public.
        :type public: bool
        :return: The updated dataset.
        :rtype: :class:`Dataset`
        """
        data = {
            "public": _convert_bool_to_public_value(public)
        }

        if name:
            data["name"] = name
        if description:
            data["description"] = description

        dataset = {"dataset": data}
        failure_message = "Failed to update dataset {}".format(dataset_id)
        response = self._get_success_json(self._post_json(routes.update_dataset(dataset_id), data=dataset, failure_message=failure_message))

        return _dataset_from_response_dict(response)

def
update_dataset
(
self
,
dataset_id
,
name
=
,
description
=
,
public
=
)
:
data
=
{
"public"
:
_convert_bool_to_public_value
(
public
)
}
if
name
:
data
[
"name"
]
=
name
if
description
:
data
[
"description"
]
=
description
dataset
=
{
"dataset"
:
data
}
failure_message
=
"Failed to update dataset {}"
.
format
(
dataset_id
)
response
=
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
update_dataset
(
dataset_id
)
,
data
=
dataset
,
failure_message
=
failure_message
)
)
return
_dataset_from_response_dict
(
response
)


def create_dataset_version(self, dataset_id):
        """
        Create a new data set version.

        :param dataset_id: The ID of the dataset for which the version must be bumped.
        :type dataset_id: int
        :return: The new dataset version.
        :rtype: :class:`DatasetVersion`
        """
        failure_message = "Failed to create dataset version for dataset {}".format(dataset_id)
        number = self._get_success_json(self._post_json(routes.create_dataset_version(dataset_id), data={}, failure_message=failure_message))['dataset_scoped_id']

        return DatasetVersion(number=number)


def get_available_columns(self, dataset_ids):
        """
        Retrieves the set of columns from the combination of dataset ids given

        :param dataset_ids: The id of the dataset to retrieve columns from
        :type dataset_ids: list of int
        :return: A list of column names from the dataset ids given.
        :rtype: list of str
        """
        if not isinstance(dataset_ids, list):
            dataset_ids = [dataset_ids]

        data = {
            "dataset_ids":
                dataset_ids
        }

        failure_message = "Failed to get available columns in dataset(s) {}".format(dataset_ids)

        return self._get_success_json(self._post_json(
            'v1/datasets/get-available-columns', data, failure_message=failure_message))['data']


def __generate_search_template(self, dataset_ids):
        """
        Generates a default search templates from the available columns in the dataset ids given.

        :param dataset_ids: The id of the dataset to retrieve files from
        :type dataset_ids: list of int
        :return: A search template based on the columns in the datasets given
        """

        data = {
            "dataset_ids":
                dataset_ids
        }

        failure_message = "Failed to generate a search template from columns in dataset(s) {}".format(dataset_ids)

        return self._get_success_json(self._post_json(
            'v1/search_templates/builders/from-dataset-ids', data, failure_message=failure_message))['data']

Generates a default search templates from the available columns in the dataset ids given .


def __prune_search_template(self, extract_as_keys, search_template):
        """
        Returns a new search template, but the new template has only the extract_as_keys given.

        :param extract_as_keys: List of extract as keys to keep
        :param search_template: The search template to prune
        :return: New search template with pruned columns
        """

        data = {
            "extract_as_keys":
                extract_as_keys,
            "search_template":
                search_template
        }

        failure_message = "Failed to prune a search template"

        return self._get_success_json(self._post_json(
            'v1/search_templates/prune-to-extract-as', data, failure_message=failure_message))['data']

Returns a new search template but the new template has only the extract_as_keys given .


def default(self, obj):
        """
        Convert an object to a form ready to dump to json.

        :param obj: Object being serialized. The type of this object must be one of the following: None; a single object derived from the Pio class; or a list of objects, each derived from the Pio class.
        :return: List of dictionaries, each representing a physical information object, ready to be serialized.
        """
        if obj is None:
            return []
        elif isinstance(obj, list):
            return [i.as_dictionary() for i in obj]
        elif isinstance(obj, dict):
            return self._keys_to_camel_case(obj)
        else:
            return obj.as_dictionary()

def default ( self , obj ) : if obj is None : return [ ] elif isinstance ( obj , list ) : return [ i . as_dictionary ( ) for i in obj ] elif isinstance ( obj , dict ) : return self . _keys_to_camel_case ( obj ) else : return obj . as_dictionary ( )

Convert an object to a form ready to dump to json .


def _keys_to_camel_case(self, obj):
        """
        Make a copy of a dictionary with all keys converted to camel case. This is just calls to_camel_case on each of the keys in the dictionary and returns a new dictionary.

        :param obj: Dictionary to convert keys to camel case.
        :return: Dictionary with the input values and all keys in camel case
        """
        return dict((to_camel_case(key), value) for (key, value) in obj.items())

def _keys_to_camel_case ( self , obj ) : return dict ( ( to_camel_case ( key ) , value ) for ( key , value ) in obj . items ( ) )

Make a copy of a dictionary with all keys converted to camel case . This is just calls to_camel_case on each of the keys in the dictionary and returns a new dictionary .


def validate(self, ml_template):
        """
        Runs the template against the validation endpoint, returns a message indicating status of the templte

        :param ml_template: Template to validate
        :return: OK or error message if validation failed
        """

        data = {
            "ml_template":
                ml_template
        }

        failure_message = "ML template validation invoke failed"

        res = self._get_success_json(self._post_json(
            'ml_templates/validate', data, failure_message=failure_message))['data']
        if res['valid']:
            return 'OK'
        return res['reason']

def
validate
(
self
,
ml_template
)
:
data
=
{
"ml_template"
:
ml_template
}
failure_message
=
"ML template validation invoke failed"
res
=
self
.
_get_success_json
(
self
.
_post_json
(
'ml_templates/validate'
,
data
,
failure_message
=
failure_message
)
)
[
'data'
]
if
res
[
'valid'
]
:
return
'OK'
return
res
[
'reason'
]

Runs the template against the validation endpoint returns a message indicating status of the templte


def add_organization_course(organization_data, course_key):
    """
    Adds a organization-course link to the system
    """
    _validate_course_key(course_key)
    _validate_organization_data(organization_data)
    data.create_organization_course(
        organization=organization_data,
        course_key=course_key
    )

Adds a organization - course link to the system


def remove_organization_course(organization, course_key):
    """
    Removes the specfied course from the specified organization
    """
    _validate_organization_data(organization)
    _validate_course_key(course_key)
    return data.delete_organization_course(course_key=course_key, organization=organization)

def remove_organization_course ( organization , course_key ) : _validate_organization_data ( organization ) _validate_course_key ( course_key ) return data . delete_organization_course ( course_key = course_key , organization = organization )


def course_key_is_valid(course_key):
    """
    Course key object validation
    """
    if course_key is None:
        return False
    try:
        CourseKey.from_string(text_type(course_key))
    except (InvalidKeyError, UnicodeDecodeError):
        return False
    return True

def course_key_is_valid ( course_key ) : if course_key is None : return False try : CourseKey . from_string ( text_type ( course_key ) ) except ( InvalidKeyError , UnicodeDecodeError ) : return False return True


def organization_data_is_valid(organization_data):
    """
    Organization data validation
    """
    if organization_data is None:
        return False
    if 'id' in organization_data and not organization_data.get('id'):
        return False
    if 'name' in organization_data and not organization_data.get('name'):
        return False
    return True

def organization_data_is_valid ( organization_data ) : if organization_data is None : return False if 'id' in organization_data and not organization_data . get ( 'id' ) : return False if 'name' in organization_data and not organization_data . get ( 'name' ) : return False return True


def _inactivate_organization(organization):
    """
    Inactivates an activated organization as well as any active relationships
    """
    [_inactivate_organization_course_relationship(record) for record
     in internal.OrganizationCourse.objects.filter(organization_id=organization.id, active=True)]

    [_inactivate_record(record) for record
     in internal.Organization.objects.filter(id=organization.id, active=True)]


def _activate_organization_course_relationship(relationship):  # pylint: disable=invalid-name
    """
    Activates an inactive organization-course relationship
    """
    # If the relationship doesn't exist or the organization isn't active we'll want to raise an error
    relationship = internal.OrganizationCourse.objects.get(
        id=relationship.id,
        active=False,
        organization__active=True
    )
    _activate_record(relationship)

Activates an inactive organization - course relationship


def _inactivate_organization_course_relationship(relationship):  # pylint: disable=invalid-name
    """
    Inactivates an active organization-course relationship
    """
    relationship = internal.OrganizationCourse.objects.get(
        id=relationship.id,
        active=True
    )
    _inactivate_record(relationship)

Inactivates an active organization - course relationship


def fetch_organization_courses(organization):
    """
    Retrieves the set of courses currently linked to the specified organization
    """
    organization_obj = serializers.deserialize_organization(organization)
    queryset = internal.OrganizationCourse.objects.filter(
        organization=organization_obj,
        active=True
    ).select_related('organization')
    return [serializers.serialize_organization_with_course(organization) for organization in queryset]

def fetch_organization_courses ( organization ) : organization_obj = serializers . deserialize_organization ( organization ) queryset = internal . OrganizationCourse . objects . filter ( organization = organization_obj , active = True ) . select_related ( 'organization' ) return [ serializers . serialize_organization_with_course ( organization ) for organization in queryset ]


def fetch_course_organizations(course_key):
    """
    Retrieves the organizations linked to the specified course
    """
    queryset = internal.OrganizationCourse.objects.filter(
        course_id=text_type(course_key),
        active=True
    ).select_related('organization')
    return [serializers.serialize_organization_with_course(organization) for organization in queryset]

def fetch_course_organizations ( course_key ) : queryset = internal . OrganizationCourse . objects . filter ( course_id = text_type ( course_key ) , active = True ) . select_related ( 'organization' ) return [ serializers . serialize_organization_with_course ( organization ) for organization in queryset ]


def serialize_organization(organization):
    """
    Organization object-to-dict serialization
    """
    return {
        'id': organization.id,
        'name': organization.name,
        'short_name': organization.short_name,
        'description': organization.description,
        'logo': organization.logo
    }

def serialize_organization ( organization ) : return { 'id' : organization . id , 'name' : organization . name , 'short_name' : organization . short_name , 'description' : organization . description , 'logo' : organization . logo }

Organization object - to - dict serialization


def deserialize_organization(organization_dict):
    """
    Organization dict-to-object serialization
    """
    return models.Organization(
        id=organization_dict.get('id'),
        name=organization_dict.get('name', ''),
        short_name=organization_dict.get('short_name', ''),
        description=organization_dict.get('description', ''),
        logo=organization_dict.get('logo', '')
    )

def deserialize_organization ( organization_dict ) : return models . Organization ( id = organization_dict . get ( 'id' ) , name = organization_dict . get ( 'name' , '' ) , short_name = organization_dict . get ( 'short_name' , '' ) , description = organization_dict . get ( 'description' , '' ) , logo = organization_dict . get ( 'logo' , '' ) )

Organization dict - to - object serialization


def is_valid_filename(self, image_node):
        """\
        will check the image src against a list
        of bad image files we know of like buttons, etc...
        """
        src = self.parser.getAttribute(image_node, attr='src')

        if not src:
            return False

        if self.badimages_names_re.search(src):
            return False

        return True

def is_valid_filename ( self , image_node ) : src = self . parser . getAttribute ( image_node , attr = 'src' ) if not src : return False if self . badimages_names_re . search ( src ) : return False return True

\ will check the image src against a list of bad image files we know of like buttons etc ...


def get_images_bytesize_match(self, images):
        """\
        loop through all the images and find the ones
        that have the best bytez to even make them a candidate
        """
        cnt = 0
        max_bytes_size = 15728640
        good_images = []
        for image in images:
            if cnt > 30:
                return good_images
            src = self.parser.getAttribute(image, attr='src')
            src = self.build_image_path(src)
            src = self.add_schema_if_none(src)
            local_image = self.get_local_image(src)
            if local_image:
                filesize = local_image.bytes
                if (filesize == 0 or filesize > self.images_min_bytes) and filesize < max_bytes_size:
                    good_images.append(image)
                else:
                    images.remove(image)
            cnt += 1
        return good_images if len(good_images) > 0 else None

def get_images_bytesize_match ( self , images ) : cnt = 0 max_bytes_size = 15728640 good_images = [ ] for image in images : if cnt > 30 : return good_images src = self . parser . getAttribute ( image , attr = 'src' ) src = self . build_image_path ( src ) src = self . add_schema_if_none ( src ) local_image = self . get_local_image ( src ) if local_image : filesize = local_image . bytes if ( filesize == 0 or filesize > self . images_min_bytes ) and filesize < max_bytes_size : good_images . append ( image ) else : images . remove ( image ) cnt += 1 return good_images if len ( good_images ) > 0 else None


def check_link_tag(self):
        """\
        checks to see if we were able to
        find open link_src on this page
        """
        node = self.article.raw_doc
        meta = self.parser.getElementsByTag(node, tag='link', attr='rel', value='image_src')
        for item in meta:
            src = self.parser.getAttribute(item, attr='href')
            if src:
                return self.get_image(src, extraction_type='linktag')
        return None

def check_link_tag ( self ) : node = self . article . raw_doc meta = self . parser . getElementsByTag ( node , tag = 'link' , attr = 'rel' , value = 'image_src' ) for item in meta : src = self . parser . getAttribute ( item , attr = 'href' ) if src : return self . get_image ( src , extraction_type = 'linktag' ) return None


def get_local_image(self, src):
        """\
        returns the bytes of the image file on disk
        """
        return ImageUtils.store_image(self.fetcher, self.article.link_hash, src, self.config)

def get_local_image ( self , src ) : return ImageUtils . store_image ( self . fetcher , self . article . link_hash , src , self . config )


def get_video(self, node):
        """
        Create a video object from a video embed
        """
        video = Video()
        video._embed_code = self.get_embed_code(node)
        video._embed_type = self.get_embed_type(node)
        video._width = self.get_width(node)
        video._height = self.get_height(node)
        video._src = self.get_src(node)
        video._provider = self.get_provider(video.src)
        return video

def get_video ( self , node ) : video = Video ( ) video . _embed_code = self . get_embed_code ( node ) video . _embed_type = self . get_embed_type ( node ) video . _width = self . get_width ( node ) video . _height = self . get_height ( node ) video . _src = self . get_src ( node ) video . _provider = self . get_provider ( video . src ) return video


def store_image(cls, http_client, link_hash, src, config):
        """\
        Writes an image src http string to disk as a temporary file
        and returns the LocallyStoredImage object
        that has the info you should need on the image
        """
        # check for a cache hit already on disk
        image = cls.read_localfile(link_hash, src, config)
        if image:
            return image

        # no cache found; do something else

        # parse base64 image
        if src.startswith('data:image'):
            image = cls.write_localfile_base64(link_hash, src, config)
            return image

        # download the image
        data = http_client.fetch(src)
        if data:
            image = cls.write_localfile(data, link_hash, src, config)
            if image:
                return image

        return None

def store_image ( cls , http_client , link_hash , src , config ) : image = cls . read_localfile ( link_hash , src , config ) if image : return image if src . startswith ( 'data:image' ) : image = cls . write_localfile_base64 ( link_hash , src , config ) return image data = http_client . fetch ( src ) if data : image = cls . write_localfile ( data , link_hash , src , config ) if image : return image return None


def get_siblings_content(self, current_sibling, baselinescore_siblings_para):
        """
        adds any siblings that may have a decent score to this node
        """
        if current_sibling.tag == 'p' and self.parser.getText(current_sibling):
            tmp = current_sibling
            if tmp.tail:
                tmp = deepcopy(tmp)
                tmp.tail = ''
            return [tmp]
        else:
            potential_paragraphs = self.parser.getElementsByTag(current_sibling, tag='p')
            if potential_paragraphs is None:
                return None

            paragraphs = list()
            for first_paragraph in potential_paragraphs:
                text = self.parser.getText(first_paragraph)
                if text:  # no len(text) > 0
                    word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text)
                    paragraph_score = word_stats.get_stopword_count()
                    sibling_baseline_score = float(.30)
                    high_link_density = self.is_highlink_density(first_paragraph)
                    score = float(baselinescore_siblings_para * sibling_baseline_score)
                    if score < paragraph_score and not high_link_density:
                        para = self.parser.createElement(tag='p', text=text, tail=None)
                        paragraphs.append(para)
            return paragraphs

def get_siblings_content ( self , current_sibling , baselinescore_siblings_para ) : if current_sibling . tag == 'p' and self . parser . getText ( current_sibling ) : tmp = current_sibling if tmp . tail : tmp = deepcopy ( tmp ) tmp . tail = '' return [ tmp ] else : potential_paragraphs = self . parser . getElementsByTag ( current_sibling , tag = 'p' ) if potential_paragraphs is None : return None paragraphs = list ( ) for first_paragraph in potential_paragraphs : text = self . parser . getText ( first_paragraph ) if text : word_stats = self . stopwords_class ( language = self . get_language ( ) ) . get_stopword_count ( text ) paragraph_score = word_stats . get_stopword_count ( ) sibling_baseline_score = float ( .30 ) high_link_density = self . is_highlink_density ( first_paragraph ) score = float ( baselinescore_siblings_para * sibling_baseline_score ) if score < paragraph_score and not high_link_density : para = self . parser . createElement ( tag = 'p' , text = text , tail = None ) paragraphs . append ( para ) return paragraphs


def is_highlink_density(self, element):
        """
        checks the density of links within a node,
        is there not much text and most of it contains linky shit?
        if so it's no good
        """
        links = self.parser.getElementsByTag(element, tag='a')
        if not links:
            return False

        text = self.parser.getText(element)
        words = text.split(' ')
        words_number = float(len(words))
        link_text_parts = []
        for link in links:
            link_text_parts.append(self.parser.getText(link))

        link_text = ''.join(link_text_parts)
        link_words = link_text.split(' ')
        number_of_link_words = float(len(link_words))
        number_of_links = float(len(links))
        link_divisor = float(number_of_link_words / words_number)
        score = float(link_divisor * number_of_links)
        if score >= 1.0:
            return True
        return False

def
is_highlink_density
(
self
,
element
)
:
links
=
self
.
parser
.
getElementsByTag
(
element
,
tag
=
'a'
)
if
not
links
:
return
False
text
=
self
.
parser
.
getText
(
element
)
words
=
text
.
split
(
' '
)
words_number
=
float
(
len
(
words
)
)
link_text_parts
=
[
]
for
link
in
links
:
link_text_parts
.
append
(
self
.
parser
.
getText
(
link
)
)
link_text
=
''
.
join
(
link_text_parts
)
link_words
=
link_text
.
split
(
' '
)
number_of_link_words
=
float
(
len
(
link_words
)
)
number_of_links
=
float
(
len
(
links
)
)
link_divisor
=
float
(
number_of_link_words
/
words_number
)
score
=
float
(
link_divisor
*
number_of_links
)
if
score
>=
1.0
:
return
True
return
False

checks the density of links within a node is there not much text and most of it contains linky shit? if so it s no good


def nodes_to_check(self, docs):
        """\
        returns a list of nodes we want to search
        on like paragraphs and tables
        """
        nodes_to_check = []

        for doc in docs:
            for tag in ['p', 'pre', 'td']:
                items = self.parser.getElementsByTag(doc, tag=tag)
                nodes_to_check += items
        return nodes_to_check

def nodes_to_check ( self , docs ) : nodes_to_check = [ ] for doc in docs : for tag in [ 'p' , 'pre' , 'td' ] : items = self . parser . getElementsByTag ( doc , tag = tag ) nodes_to_check += items return nodes_to_check


def post_cleanup(self):
        """\
        remove any divs that looks like non-content,
        clusters of links, or paras with no gusto
        """
        parse_tags = ['p']
        if self.config.parse_lists:
            parse_tags.extend(['ul', 'ol'])
        if self.config.parse_headers:
            parse_tags.extend(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])

        target_node = self.article.top_node
        node = self.add_siblings(target_node)
        for elm in self.parser.getChildren(node):
            e_tag = self.parser.getTag(elm)
            if e_tag not in parse_tags:
                if (self.is_highlink_density(elm) or self.is_table_and_no_para_exist(elm) or
                        not self.is_nodescore_threshold_met(node, elm)):
                    self.parser.remove(elm)
        return node

def post_cleanup ( self ) : parse_tags = [ 'p' ] if self . config . parse_lists : parse_tags . extend ( [ 'ul' , 'ol' ] ) if self . config . parse_headers : parse_tags . extend ( [ 'h1' , 'h2' , 'h3' , 'h4' , 'h5' , 'h6' ] ) target_node = self . article . top_node node = self . add_siblings ( target_node ) for elm in self . parser . getChildren ( node ) : e_tag = self . parser . getTag ( elm ) if e_tag not in parse_tags : if ( self . is_highlink_density ( elm ) or self . is_table_and_no_para_exist ( elm ) or not self . is_nodescore_threshold_met ( node , elm ) ) : self . parser . remove ( elm ) return node

\ remove any divs that looks like non - content clusters of links or paras with no gusto


def get_title(self):
        """\
        Fetch the article title and analyze it
        """
        title = ''

        # rely on opengraph in case we have the data
        if "title" in list(self.article.opengraph.keys()):
            return self.clean_title(self.article.opengraph['title'])
        elif self.article.schema and "headline" in self.article.schema:
            return self.clean_title(self.article.schema['headline'])

        # try to fetch the meta headline
        meta_headline = self.parser.getElementsByTag(self.article.doc,
                                                     tag="meta",
                                                     attr="name",
                                                     value="headline")
        if meta_headline is not None and len(meta_headline) > 0:
            title = self.parser.getAttribute(meta_headline[0], 'content')
            return self.clean_title(title)

        # otherwise use the title meta
        title_element = self.parser.getElementsByTag(self.article.doc, tag='title')
        if title_element is not None and len(title_element) > 0:
            title = self.parser.getText(title_element[0])
            return self.clean_title(title)

        return title

def get_title ( self ) : title = '' if "title" in list ( self . article . opengraph . keys ( ) ) : return self . clean_title ( self . article . opengraph [ 'title' ] ) elif self . article . schema and "headline" in self . article . schema : return self . clean_title ( self . article . schema [ 'headline' ] ) meta_headline = self . parser . getElementsByTag ( self . article . doc , tag = "meta" , attr = "name" , value = "headline" ) if meta_headline is not None and len ( meta_headline ) > 0 : title = self . parser . getAttribute ( meta_headline [ 0 ] , 'content' ) return self . clean_title ( title ) title_element = self . parser . getElementsByTag ( self . article . doc , tag = 'title' ) if title_element is not None and len ( title_element ) > 0 : title = self . parser . getText ( title_element [ 0 ] ) return self . clean_title ( title ) return title


def get_canonical_link(self):
        """
        if the article has meta canonical link set in the url
        """
        if self.article.final_url:
            kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'}
            meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
            if meta is not None and len(meta) > 0:
                href = self.parser.getAttribute(meta[0], 'href')
                if href:
                    href = href.strip()
                    o = urlparse(href)
                    if not o.hostname:
                        tmp = urlparse(self.article.final_url)
                        domain = '%s://%s' % (tmp.scheme, tmp.hostname)
                        href = urljoin(domain, href)
                    return href
        return self.article.final_url

def get_canonical_link ( self ) : if self . article . final_url : kwargs = { 'tag' : 'link' , 'attr' : 'rel' , 'value' : 'canonical' } meta = self . parser . getElementsByTag ( self . article . doc , ** kwargs ) if meta is not None and len ( meta ) > 0 : href = self . parser . getAttribute ( meta [ 0 ] , 'href' ) if href : href = href . strip ( ) o = urlparse ( href ) if not o . hostname : tmp = urlparse ( self . article . final_url ) domain = '%s://%s' % ( tmp . scheme , tmp . hostname ) href = urljoin ( domain , href ) return href return self . article . final_url


def close(self):
        ''' Close the network connection and perform any other required cleanup

            Note:
                Auto closed when using goose as a context manager or when garbage collected '''
        if self.fetcher is not None:
            self.shutdown_network()
        self.finalizer.atexit = False

def close ( self ) : if self . fetcher is not None : self . shutdown_network ( ) self . finalizer . atexit = False


def extract(self, url=None, raw_html=None):
        ''' Extract the most likely article content from the html page

            Args:
                url (str): URL to pull and parse
                raw_html (str): String representation of the HTML page
            Returns:
                Article: Representation of the article contents \
                including other parsed and extracted metadata '''
        crawl_candidate = CrawlCandidate(self.config, url, raw_html)
        return self.__crawl(crawl_candidate)

def extract ( self , url = None , raw_html = None ) : crawl_candidate = CrawlCandidate ( self . config , url , raw_html ) return self . __crawl ( crawl_candidate )


def __crawl(self, crawl_candidate):
        ''' wrap the crawling functionality '''
        def crawler_wrapper(parser, parsers_lst, crawl_candidate):
            try:
                crawler = Crawler(self.config, self.fetcher)
                article = crawler.crawl(crawl_candidate)
            except (UnicodeDecodeError, ValueError) as ex:
                if parsers_lst:
                    parser = parsers_lst.pop(0)  # remove it also!
                    return crawler_wrapper(parser, parsers_lst, crawl_candidate)
                else:
                    raise ex
            return article

        # use the wrapper
        parsers = list(self.config.available_parsers)
        parsers.remove(self.config.parser_class)
        return crawler_wrapper(self.config.parser_class, parsers, crawl_candidate)

def __crawl ( self , crawl_candidate ) : def crawler_wrapper ( parser , parsers_lst , crawl_candidate ) : try : crawler = Crawler ( self . config , self . fetcher ) article = crawler . crawl ( crawl_candidate ) except ( UnicodeDecodeError , ValueError ) as ex : if parsers_lst : parser = parsers_lst . pop ( 0 ) return crawler_wrapper ( parser , parsers_lst , crawl_candidate ) else : raise ex return article parsers = list ( self . config . available_parsers ) parsers . remove ( self . config . parser_class ) return crawler_wrapper ( self . config . parser_class , parsers , crawl_candidate )


def smart_unicode(string, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Returns a unicode object representing 's'. Treats bytestrings using the
    'encoding' codec.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    # if isinstance(s, Promise):
    #     # The input is the result of a gettext_lazy() call.
    #     return s
    return force_unicode(string, encoding, strings_only, errors)

def smart_unicode ( string , encoding = 'utf-8' , strings_only = False , errors = 'strict' ) : return force_unicode ( string , encoding , strings_only , errors )

Returns a unicode object representing s . Treats bytestrings using the encoding codec .


def force_unicode(string, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Similar to smart_unicode, except that lazy instances are resolved to
    strings, rather than kept as lazy objects.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    # Handle the common case first, saves 30-40% in performance when s
    # is an instance of unicode. This function gets called often in that
    # setting.
    if isinstance(string, str):
        return string
    if strings_only and is_protected_type(string):
        return string
    try:
        if not isinstance(string, str):
            if hasattr(string, '__unicode__'):
                string = string.__unicode__()
            else:
                try:
                    string = str(string, encoding, errors)
                except UnicodeEncodeError:
                    if not isinstance(string, Exception):
                        raise
                    # If we get to here, the caller has passed in an Exception
                    # subclass populated with non-ASCII data without special
                    # handling to display as a string. We need to handle this
                    # without raising a further exception. We do an
                    # approximation to what the Exception's standard str()
                    # output should be.
                    string = ' '.join([force_unicode(arg, encoding,
                                                     strings_only,
                                                     errors) for arg in string])
        elif not isinstance(string, str):
            # Note: We use .decode() here, instead of unicode(s, encoding,
            # errors), so that if s is a SafeString, it ends up being a
            # SafeUnicode at the end.
            string = string.decode(encoding, errors)
    except UnicodeDecodeError as ex:
        if not isinstance(string, Exception):
            raise DjangoUnicodeDecodeError(string, *ex.args)
        else:
            # If we get to here, the caller has passed in an Exception
            # subclass populated with non-ASCII bytestring data without a
            # working unicode method. Try to handle this without raising a
            # further exception by individually forcing the exception args
            # to unicode.
            string = ' '.join([force_unicode(arg, encoding, strings_only,
                                             errors) for arg in string])
    return string

def
force_unicode
(
string
,
encoding
=
'utf-8'
,
strings_only
=
False
,
errors
=
'strict'
)
:
if
isinstance
(
string
,
str
)
:
return
string
if
strings_only
and
is_protected_type
(
string
)
:
return
string
try
:
if
not
isinstance
(
string
,
str
)
:
if
hasattr
(
string
,
'__unicode__'
)
:
string
=
string
.
__unicode__
(
)
else
:
try
:
string
=
str
(
string
,
encoding
,
errors
)
except
UnicodeEncodeError
:
if
not
isinstance
(
string
,
Exception
)
:
raise
string
=
' '
.
join
(
[
force_unicode
(
arg
,
encoding
,
strings_only
,
errors
)
for
arg
in
string
]
)
elif
not
isinstance
(
string
,
str
)
:
string
=
string
.
decode
(
encoding
,
errors
)
except
UnicodeDecodeError
as
ex
:
if
not
isinstance
(
string
,
Exception
)
:
raise
DjangoUnicodeDecodeError
(
string
,
*
ex
.
args
)
else
:
string
=
' '
.
join
(
[
force_unicode
(
arg
,
encoding
,
strings_only
,
errors
)
for
arg
in
string
]
)
return
string

Similar to smart_unicode except that lazy instances are resolved to strings rather than kept as lazy objects .


def smart_str(string, encoding='utf-8', strings_only=False, errors='strict'):
    """
    Returns a bytestring version of 's', encoded as specified in 'encoding'.

    If strings_only is True, don't convert (some) non-string-like objects.
    """
    if strings_only and isinstance(string, (type(None), int)):
        return string
    # if isinstance(s, Promise):
    #     return unicode(s).encode(encoding, errors)
    if isinstance(string, str):
        try:
            return string.encode(encoding, errors)
        except UnicodeEncodeError:
            return string.encode('utf-8', errors)
    elif not isinstance(string, bytes):
        try:
            return str(string).encode(encoding, errors)
        except UnicodeEncodeError:
            if isinstance(string, Exception):
                # An Exception subclass containing non-ASCII data that doesn't
                # know how to print itself properly. We shouldn't raise a
                # further exception.
                return ' '.join([smart_str(arg, encoding, strings_only,
                                           errors) for arg in string])
            return str(string).encode(encoding, errors)
    else:
        return string

def
smart_str
(
string
,
encoding
=
'utf-8'
,
strings_only
=
False
,
errors
=
'strict'
)
:
if
strings_only
and
isinstance
(
string
,
(
type
(
)
,
int
)
)
:
return
string
if
isinstance
(
string
,
str
)
:
try
:
return
string
.
encode
(
encoding
,
errors
)
except
UnicodeEncodeError
:
return
string
.
encode
(
'utf-8'
,
errors
)
elif
not
isinstance
(
string
,
bytes
)
:
try
:
return
str
(
string
)
.
encode
(
encoding
,
errors
)
except
UnicodeEncodeError
:
if
isinstance
(
string
,
Exception
)
:
return
' '
.
join
(
[
smart_str
(
arg
,
encoding
,
strings_only
,
errors
)
for
arg
in
string
]
)
return
str
(
string
)
.
encode
(
encoding
,
errors
)
else
:
return
string

Returns a bytestring version of s encoded as specified in encoding .


def get_urls(self):
        """Add URLs needed to handle image uploads."""
        urls = patterns(
            '',
            url(r'^upload/$', self.admin_site.admin_view(self.handle_upload), name='quill-file-upload'),
        )
        return urls + super(QuillAdmin, self).get_urls()

def get_urls ( self ) : urls = patterns ( '' , url ( r'^upload/$' , self . admin_site . admin_view ( self . handle_upload ) , name = 'quill-file-upload' ) , ) return urls + super ( QuillAdmin , self ) . get_urls ( )

Add URLs needed to handle image uploads .


def handle_upload(self, request):
        """Handle file uploads from WYSIWYG."""
        if request.method != 'POST':
            raise Http404

        if request.is_ajax():
            try:
                filename = request.GET['quillUploadFile']
                data = request
                is_raw = True
            except KeyError:
                return HttpResponseBadRequest("Invalid file upload.")
        else:
            if len(request.FILES) != 1:
                return HttpResponseBadRequest("Can only upload 1 file at a time.")
            try:
                data = request.FILES['quillUploadFile']
                filename = data.name
                is_raw = False
            except KeyError:
                return HttpResponseBadRequest('Missing image `quillUploadFile`.')

        url = save_file(data, filename, is_raw, default_storage)
        response_data = {}
        response_data['url'] = url

        # Response content type needs to be text/html here or else
        # IE will try to download the file.
        return HttpResponse(json.dumps(response_data), content_type="text/html; charset=utf-8")

def
handle_upload
(
self
,
request
)
:
if
request
.
method
!=
'POST'
:
raise
Http404
if
request
.
is_ajax
(
)
:
try
:
filename
=
request
.
GET
[
'quillUploadFile'
]
data
=
request
is_raw
=
True
except
KeyError
:
return
HttpResponseBadRequest
(
"Invalid file upload."
)
else
:
if
len
(
request
.
FILES
)
!=
1
:
return
HttpResponseBadRequest
(
"Can only upload 1 file at a time."
)
try
:
data
=
request
.
FILES
[
'quillUploadFile'
]
filename
=
data
.
name
is_raw
=
False
except
KeyError
:
return
HttpResponseBadRequest
(
'Missing image `quillUploadFile`.'
)
url
=
save_file
(
data
,
filename
,
is_raw
,
default_storage
)
response_data
=
{
}
response_data
[
'url'
]
=
url
return
HttpResponse
(
json
.
dumps
(
response_data
)
,
content_type
=
"text/html; charset=utf-8"
)


def render(self, name, value, attrs={}):
        """Render the Quill WYSIWYG."""
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, name=name)
        quill_app = apps.get_app_config('quill')
        quill_config = getattr(quill_app, self.config)

        return mark_safe(render_to_string(quill_config['template'], {
            'final_attrs': flatatt(final_attrs),
            'value': value,
            'id': final_attrs['id'],
            'config': self.config,
        }))

def render ( self , name , value , attrs = { } ) : if value is None : value = '' final_attrs = self . build_attrs ( attrs , name = name ) quill_app = apps . get_app_config ( 'quill' ) quill_config = getattr ( quill_app , self . config ) return mark_safe ( render_to_string ( quill_config [ 'template' ] , { 'final_attrs' : flatatt ( final_attrs ) , 'value' : value , 'id' : final_attrs [ 'id' ] , 'config' : self . config , } ) )


def formfield(self, **kwargs):
        """Get the form for field."""
        defaults = {
            'form_class': RichTextFormField,
            'config': self.config,
        }
        defaults.update(kwargs)
        return super(RichTextField, self).formfield(**defaults)

def formfield ( self , ** kwargs ) : defaults = { 'form_class' : RichTextFormField , 'config' : self . config , } defaults . update ( kwargs ) return super ( RichTextField , self ) . formfield ( ** defaults )


def render_toolbar(context, config):
    """Render the toolbar for the given config."""
    quill_config = getattr(quill_app, config)
    t = template.loader.get_template(quill_config['toolbar_template'])
    return t.render(context)

def render_toolbar ( context , config ) : quill_config = getattr ( quill_app , config ) t = template . loader . get_template ( quill_config [ 'toolbar_template' ] ) return t . render ( context )

Render the toolbar for the given config .


def get_meta_image_url(request, image):
    """
    Resize an image for metadata tags, and return an absolute URL to it.
    """
    rendition = image.get_rendition(filter='original')
    return request.build_absolute_uri(rendition.url)

def get_meta_image_url ( request , image ) : rendition = image . get_rendition ( filter = 'original' ) return request . build_absolute_uri ( rendition . url )

Resize an image for metadata tags and return an absolute URL to it .


def check_mdrun_success(logfile):
    """Check if ``mdrun`` finished successfully.

    Analyses the output from ``mdrun`` in *logfile*. Right now we are
    simply looking for the line "Finished mdrun on node" in the last 1kb of
    the file. (The file must be seeakable.)

    :Arguments:
      *logfile* : filename
         Logfile produced by ``mdrun``.

    :Returns: ``True`` if all ok, ``False`` if not finished, and
              ``None`` if the *logfile* cannot be opened
    """
    if not os.path.exists(logfile):
        return None
    with open(logfile, 'rb') as log:
        log.seek(-1024, 2)
        for line in log:
            line = line.decode('ASCII')
            if line.startswith("Finished mdrun on"):
                return True
    return False

def
check_mdrun_success
(
logfile
)
:
if
not
os
.
path
.
exists
(
logfile
)
:
return
with
open
(
logfile
,
'rb'
)
as
log
:
log
.
seek
(
1024
,
2
)
for
line
in
log
:
line
=
line
.
decode
(
'ASCII'
)
if
line
.
startswith
(
"Finished mdrun on"
)
:
return
True
return
False


def get_double_or_single_prec_mdrun():
    """Return double precision ``mdrun`` or fall back to single precision.

    This convenience function tries :func:`gromacs.mdrun_d` first and
    if it cannot run it, falls back to :func:`gromacs.mdrun` (without
    further checking).

    .. versionadded:: 0.5.1
    """
    try:
        gromacs.mdrun_d(h=True, stdout=False, stderr=False)
        logger.debug("using double precision gromacs.mdrun_d")
        return gromacs.mdrun_d
    except (AttributeError, GromacsError, OSError):
        # fall back to mdrun if no double precision binary
        wmsg = "No 'mdrun_d' binary found so trying 'mdrun' instead.\n"\
            "(Note that energy minimization runs better with mdrun_d.)"
        logger.warn(wmsg)
        warnings.warn(wmsg, category=AutoCorrectionWarning)
        return gromacs.mdrun

Return double precision mdrun or fall back to single precision .


def commandline(self, **mpiargs):
        """Returns simple command line to invoke mdrun.

        If :attr:`mpiexec` is set then :meth:`mpicommand` provides the mpi
        launcher command that prefixes the actual ``mdrun`` invocation:

           :attr:`mpiexec` [*mpiargs*]  :attr:`mdrun` [*mdrun-args*]

        The *mdrun-args* are set on initializing the class. Override
        :meth:`mpicommand` to fit your system if the simple default
        OpenMP launcher is not appropriate.
        """
        cmd = self.MDRUN.commandline()
        if self.mpiexec:
            cmd = self.mpicommand(**mpiargs) + cmd
        return cmd

def commandline ( self , ** mpiargs ) : cmd = self . MDRUN . commandline ( ) if self . mpiexec : cmd = self . mpicommand ( ** mpiargs ) + cmd return cmd

Returns simple command line to invoke mdrun .


def mpicommand(self, *args, **kwargs):
        """Return a list of the mpi command portion of the commandline.

        Only allows primitive mpi at the moment:
           *mpiexec* -n *ncores* *mdrun* *mdrun-args*

        (This is a primitive example for OpenMP. Override it for more
        complicated cases.)
        """
        if self.mpiexec is None:
            raise NotImplementedError("Override mpiexec to enable the simple OpenMP launcher")
        # example implementation
        ncores = kwargs.pop('ncores', 8)
        return [self.mpiexec, '-n', str(ncores)]

Return a list of the mpi command portion of the commandline .


def prehook(self, **kwargs):
        """Launch local smpd."""
        cmd = ['smpd', '-s']
        logger.info("Starting smpd: "+" ".join(cmd))
        rc = subprocess.call(cmd)
        return rc

def
prehook
(
self
,
**
kwargs
)
:
cmd
=
[
'smpd'
,
'-s'
]
logger
.
info
(
"Starting smpd: "
+
" "
.
join
(
cmd
)
)
rc
=
subprocess
.
call
(
cmd
)
return
rc


def glob_parts(prefix, ext):
    """Find files from a continuation run"""
    if ext.startswith('.'):
        ext = ext[1:]
    files = glob.glob(prefix+'.'+ext) + glob.glob(prefix+'.part[0-9][0-9][0-9][0-9].'+ext)
    files.sort()   # at least some rough sorting...
    return files

def glob_parts ( prefix , ext ) : if ext . startswith ( '.' ) : ext = ext [ 1 : ] files = glob . glob ( prefix + '.' + ext ) + glob . glob ( prefix + '.part[0-9][0-9][0-9][0-9].' + ext ) files . sort ( ) return files


def grompp_qtot(*args, **kwargs):
    """Run ``gromacs.grompp`` and return the total charge of the  system.

    :Arguments:
       The arguments are the ones one would pass to :func:`gromacs.grompp`.
    :Returns:
       The total charge as reported

    Some things to keep in mind:

    * The stdout output of grompp is only shown when an error occurs. For
      debugging, look at the log file or screen output and try running the
      normal :func:`gromacs.grompp` command and analyze the output if the
      debugging messages are not sufficient.

    * Check that ``qtot`` is correct. Because the function is based on pattern
      matching of the informative output of :program:`grompp` it can break when
      the output format changes. This version recognizes lines like ::

            '  System has non-zero total charge: -4.000001e+00'

      using the regular expression
      :regexp:`System has non-zero total charge: *(?P<qtot>[-+]?\d*\.\d+([eE][-+]\d+)?)`.

    """
    qtot_pattern = re.compile('System has non-zero total charge: *(?P<qtot>[-+]?\d*\.\d+([eE][-+]\d+)?)')
    # make sure to capture ALL output
    kwargs['stdout'] = False
    kwargs['stderr'] = False
    rc, output, error = grompp_warnonly(*args, **kwargs)
    gmxoutput = "\n".join([x for x in [output, error] if x is not None])
    if rc != 0:
        # error occured and we want to see the whole output for debugging
        msg = "grompp_qtot() failed. See warning and screen output for clues."
        logger.error(msg)
        import sys
        sys.stderr.write("=========== grompp (stdout/stderr) ============\n")
        sys.stderr.write(gmxoutput)
        sys.stderr.write("===============================================\n")
        sys.stderr.flush()
        raise GromacsError(rc, msg)
    qtot = 0
    for line in gmxoutput.split('\n'):
        m = qtot_pattern.search(line)
        if m:
            qtot = float(m.group('qtot'))
            break
    logger.info("system total charge qtot = {qtot!r}".format(**vars()))
    return qtot

Run gromacs . grompp and return the total charge of the system .


def _mdp_include_string(dirs):
    """Generate a string that can be added to a mdp 'include = ' line."""
    include_paths = [os.path.expanduser(p) for p in dirs]
    return ' -I'.join([''] + include_paths)

def
_mdp_include_string
(
dirs
)
:
include_paths
=
[
os
.
path
.
expanduser
(
p
)
for
p
in
dirs
]
return
' -I'
.
join
(
[
''
]
+
include_paths
)

Generate a string that can be added to a mdp include = line .


def create_portable_topology(topol, struct, **kwargs):
    """Create a processed topology.

    The processed (or portable) topology file does not contain any
    ``#include`` statements and hence can be easily copied around. It
    also makes it possible to re-grompp without having any special itp
    files available.

    :Arguments:
      *topol*
          topology file
      *struct*
          coordinat (structure) file

    :Keywords:
      *processed*
          name of the new topology file; if not set then it is named like
          *topol* but with ``pp_`` prepended
      *includes*
          path or list of paths of directories in which itp files are
          searched for
      *grompp_kwargs**
          other options for :program:`grompp` such as ``maxwarn=2`` can
          also be supplied

    :Returns: full path to the processed topology
    """
    _topoldir, _topol = os.path.split(topol)
    processed = kwargs.pop('processed', os.path.join(_topoldir, 'pp_'+_topol))
    grompp_kwargs, mdp_kwargs = filter_grompp_options(**kwargs)
    mdp_kwargs = add_mdp_includes(topol, mdp_kwargs)
    with tempfile.NamedTemporaryFile(suffix='.mdp') as mdp:
        mdp.write('; empty mdp file\ninclude = {include!s}\n'.format(**mdp_kwargs))
        mdp.flush()
        grompp_kwargs['p'] = topol
        grompp_kwargs['pp'] = processed
        grompp_kwargs['f'] =  mdp.name
        grompp_kwargs['c'] = struct
        grompp_kwargs['v'] = False
        try:
            gromacs.grompp(**grompp_kwargs)
        finally:
            utilities.unlink_gmx('topol.tpr', 'mdout.mdp')
    return utilities.realpath(processed)


def edit_txt(filename, substitutions, newname=None):
    """Primitive text file stream editor.

    This function can be used to edit free-form text files such as the
    topology file. By default it does an **in-place edit** of
    *filename*. If *newname* is supplied then the edited
    file is written to *newname*.

    :Arguments:
       *filename*
           input text file
       *substitutions*
           substitution commands (see below for format)
       *newname*
           output filename; if ``None`` then *filename* is changed in
           place [``None``]

    *substitutions* is a list of triplets; the first two elements are regular
    expression strings, the last is the substitution value. It mimics
    ``sed`` search and replace. The rules for *substitutions*:

    .. productionlist::
       substitutions: "[" search_replace_tuple, ... "]"
       search_replace_tuple: "(" line_match_RE "," search_RE "," replacement ")"
       line_match_RE: regular expression that selects the line (uses match)
       search_RE: regular expression that is searched in the line
       replacement: replacement string for search_RE

    Running :func:`edit_txt` does pretty much what a simple ::

         sed /line_match_RE/s/search_RE/replacement/

    with repeated substitution commands does.

    Special replacement values:
    - ``None``: the rule is ignored
    - ``False``: the line is deleted (even if other rules match)

    .. note::

       * No sanity checks are performed and the substitutions must be supplied
         exactly as shown.
       * All substitutions are applied to a line; thus the order of the substitution
         commands may matter when one substitution generates a match for a subsequent rule.
       * If replacement is set to ``None`` then the whole expression is ignored and
         whatever is in the template is used. To unset values you must provided an
         empty string or similar.
       * Delete a matching line if replacement=``False``.
    """
    if newname is None:
        newname = filename

    # No sanity checks (figure out later how to give decent diagnostics).
    # Filter out any rules that have None in replacement.
    _substitutions = [{'lRE': re.compile(str(lRE)),
                       'sRE': re.compile(str(sRE)),
                       'repl': repl}
                      for lRE,sRE,repl in substitutions if repl is not None]

    with tempfile.TemporaryFile() as target:
        with open(filename, 'rb') as src:
            logger.info("editing txt = {0!r} ({1:d} substitutions)".format(filename, len(substitutions)))
            for line in src:
                line = line.decode("utf-8")
                keep_line = True
                for subst in _substitutions:
                    m = subst['lRE'].match(line)
                    if m:              # apply substition to this line?
                        logger.debug('match:    '+line.rstrip())
                        if subst['repl'] is False:   # special rule: delete line
                            keep_line = False
                        else:                        # standard replacement
                            line = subst['sRE'].sub(str(subst['repl']), line)
                            logger.debug('replaced: '+line.rstrip())
                if keep_line:
                    target.write(line.encode('utf-8'))
                else:
                    logger.debug("Deleting line %r", line)

        target.seek(0)
        with open(newname, 'wb') as final:
            shutil.copyfileobj(target, final)
    logger.info("edited txt = {newname!r}".format(**vars()))


def make_ndx_captured(**kwargs):
    """make_ndx that captures all output

    Standard :func:`~gromacs.make_ndx` command with the input and
    output pre-set in such a way that it can be conveniently used for
    :func:`parse_ndxlist`.

    Example::
      ndx_groups = parse_ndxlist(make_ndx_captured(n=ndx)[0])

    Note that the convenient :func:`get_ndx_groups` function does exactly
    that and can probably used in most cases.

    :Arguments:
        keywords are passed on to :func:`~gromacs.make_ndx`
    :Returns:
        (*returncode*, *output*, ``None``)
    """
    kwargs['stdout']=False   # required for proper output as described in doc
    user_input = kwargs.pop('input',[])
    user_input = [cmd for cmd in user_input if cmd != 'q']  # filter any quit
    kwargs['input'] = user_input + ['', 'q']                # necessary commands
    return gromacs.make_ndx(**kwargs)

def make_ndx_captured ( ** kwargs ) : kwargs [ 'stdout' ] = False user_input = kwargs . pop ( 'input' , [ ] ) user_input = [ cmd for cmd in user_input if cmd != 'q' ] kwargs [ 'input' ] = user_input + [ '' , 'q' ] return gromacs . make_ndx ( ** kwargs )


def parse_groups(output):
    """Parse ``make_ndx`` output and return groups as a list of dicts."""
    groups = []
    for line in output.split('\n'):
        m = NDXGROUP.match(line)
        if m:
            d = m.groupdict()
            groups.append({'name': d['GROUPNAME'],
                           'nr': int(d['GROUPNUMBER']),
                           'natoms': int(d['NATOMS'])})
    return groups

def parse_groups ( output ) : groups = [ ] for line in output . split ( '\n' ) : m = NDXGROUP . match ( line ) if m : d = m . groupdict ( ) groups . append ( { 'name' : d [ 'GROUPNAME' ] , 'nr' : int ( d [ 'GROUPNUMBER' ] ) , 'natoms' : int ( d [ 'NATOMS' ] ) } ) return groups

Parse make_ndx output and return groups as a list of dicts .


def delete_frames(self):
        """Delete all frames."""
        for frame in glob.glob(self.frameglob):
            os.unlink(frame)

def delete_frames ( self ) : for frame in glob . glob ( self . frameglob ) : os . unlink ( frame )


def gmx_resid(self, resid):
        """Returns resid in the Gromacs index by transforming with offset."""
        try:
            gmx_resid = int(self.offset[resid])
        except (TypeError, IndexError):
            gmx_resid = resid + self.offset
        except KeyError:
            raise KeyError("offset must be a dict that contains the gmx resid for {0:d}".format(resid))
        return gmx_resid

Returns resid in the Gromacs index by transforming with offset .


def combine(self, name_all=None, out_ndx=None, operation='|', defaultgroups=False):
        """Combine individual groups into a single one and write output.

        :Keywords:
           name_all : string
              Name of the combined group, ``None`` generates a name.  [``None``]
           out_ndx : filename
              Name of the output file that will contain the individual groups
              and the combined group. If ``None`` then default from the class
              constructor is used. [``None``]
           operation : character
              Logical operation that is used to generate the combined group from
              the individual groups: "|" (OR) or "&" (AND); if set to ``False``
              then no combined group is created and only the individual groups
              are written. ["|"]
           defaultgroups : bool
              ``True``: append everything to the default groups produced by
              :program:`make_ndx` (or rather, the groups provided in the ndx file on
              initialization --- if this was ``None`` then these are truly default groups);
              ``False``: only use the generated groups

        :Returns:
           ``(combinedgroup_name, output_ndx)``, a tuple showing the
           actual group name and the name of the file; useful when all names are autogenerated.

        .. Warning:: The order of the atom numbers in the combined group is
                     *not* guaranteed to be the same as the selections on input because
                     ``make_ndx`` sorts them ascending. Thus you should be careful when
                     using these index files for calculations of angles and dihedrals.
                     Use :class:`gromacs.formats.NDX` in these cases.

        .. SeeAlso:: :meth:`IndexBuilder.write`.
        """
        if not operation in ('|', '&', False):
            raise ValueError("Illegal operation {0!r}, only '|' (OR) and '&' (AND) or False allowed.".format(
                             operation))
        if name_all is None and operation:
            name_all = self.name_all or operation.join(self.indexfiles)
        if out_ndx is None:
            out_ndx = self.output

        if defaultgroups:
            # make a default file (using the original ndx where provided!!)
            fd, default_ndx = tempfile.mkstemp(suffix='.ndx', prefix='default__')
            try:
                self.make_ndx(o=default_ndx, input=['q'])
            except:
                utilities.unlink_gmx(default_ndx)
                raise
            ndxfiles = [default_ndx]
        else:
            ndxfiles = []

        ndxfiles.extend(self.indexfiles.values())

        if operation:
            # combine multiple selections and name them
            try:
                fd, tmp_ndx = tempfile.mkstemp(suffix='.ndx', prefix='combined__')
                # combine all selections by loading ALL temporary index files
                operation = ' '+operation.strip()+' '
                cmd = [operation.join(['"{0!s}"'.format(gname) for gname in self.indexfiles]),
                       '', 'q']
                rc,out,err = self.make_ndx(n=ndxfiles, o=tmp_ndx, input=cmd)
                if self._is_empty_group(out):
                    warnings.warn("No atoms found for {cmd!r}".format(**vars()),
                                  category=BadParameterWarning)

                # second pass for naming, sigh (or: use NDX ?)
                groups = parse_ndxlist(out)
                last = groups[-1]
                # name this group
                name_cmd = ["name {0:d} {1!s}".format(last['nr'], name_all), 'q']
                rc,out,err = self.make_ndx(n=tmp_ndx, o=out_ndx, input=name_cmd)
                # For debugging, look at out and err or set stdout=True, stderr=True
                # TODO: check out if at least 1 atom selected
                ##print "DEBUG: combine()"
                ##print out
            finally:
                utilities.unlink_gmx(tmp_ndx)
                if defaultgroups:
                    utilities.unlink_gmx(default_ndx)
        else:
            # just write individual groups in one file (name_all --> None)
            rc,out,err = self.make_ndx(n=ndxfiles, o=out_ndx, input=['','q'])

        return name_all, out_ndx

Combine individual groups into a single one and write output .


def cat(self, out_ndx=None):
        """Concatenate input index files.

        Generate a new index file that contains the default Gromacs index
        groups (if a structure file was defined) and all index groups from the
        input index files.

        :Arguments:
           out_ndx : filename
              Name of the output index file; if ``None`` then use the default
              provided to the constructore. [``None``].
        """
        if out_ndx is None:
            out_ndx = self.output
        self.make_ndx(o=out_ndx, input=['q'])
        return out_ndx

def cat ( self , out_ndx = None ) : if out_ndx is None : out_ndx = self . output self . make_ndx ( o = out_ndx , input = [ 'q' ] ) return out_ndx


def _process_command(self, command, name=None):
        """Process ``make_ndx`` command and  return name and temp index file."""

        self._command_counter += 1
        if name is None:
            name = "CMD{0:03d}".format(self._command_counter)

        # Need to build it with two make_ndx calls because I cannot reliably
        # name the new group without knowing its number.
        try:
            fd, tmp_ndx = tempfile.mkstemp(suffix='.ndx', prefix='tmp_'+name+'__')
            cmd = [command, '', 'q']   # empty command '' necessary to get list
            # This sometimes fails with 'OSError: Broken Pipe' --- hard to debug
            rc,out,err = self.make_ndx(o=tmp_ndx, input=cmd)
            self.check_output(out, "No atoms found for selection {command!r}.".format(**vars()), err=err)
            # For debugging, look at out and err or set stdout=True, stderr=True
            # TODO: check '  0 r_300_&_ALA_&_O     :     1 atoms' has at least 1 atom
            ##print "DEBUG: _process_command()"
            ##print out
            groups = parse_ndxlist(out)
            last = groups[-1]
            # reduce and name this group
            fd, ndx = tempfile.mkstemp(suffix='.ndx', prefix=name+'__')
            name_cmd = ["keep {0:d}".format(last['nr']),
                        "name 0 {0!s}".format(name), 'q']
            rc,out,err = self.make_ndx(n=tmp_ndx, o=ndx, input=name_cmd)
        finally:
            utilities.unlink_gmx(tmp_ndx)

        return name, ndx

Process make_ndx command and return name and temp index file .


def _process_range(self, selection, name=None):
        """Process a range selection.

        ("S234", "A300", "CA")   --> selected all CA in this range
        ("S234", "A300")         --> selected all atoms in this range

        .. Note:: Ignores residue type, only cares about the resid (but still required)
        """

        try:
            first, last, gmx_atomname = selection
        except ValueError:
            try:
                first, last = selection
                gmx_atomname = '*'
            except:
                logger.error("%r is not a valid range selection", selection)
                raise
        if name is None:
            name = "{first!s}-{last!s}_{gmx_atomname!s}".format(**vars())

        _first = self._translate_residue(first, default_atomname=gmx_atomname)
        _last = self._translate_residue(last, default_atomname=gmx_atomname)

        _selection = 'r {0:d} - {1:d} & & a {2!s}'.format(_first['resid'], _last['resid'], gmx_atomname)
        cmd = ['keep 0', 'del 0',
               _selection,
               'name 0 {name!s}'.format(**vars()),
               'q']
        fd, ndx = tempfile.mkstemp(suffix='.ndx', prefix=name+'__')
        rc,out,err = self.make_ndx(n=self.ndx, o=ndx, input=cmd)
        self.check_output(out, "No atoms found for "
                          "%(selection)r --> %(_selection)r" % vars())
        # For debugging, look at out and err or set stdout=True, stderr=True
        ##print "DEBUG: _process_residue()"
        ##print out

        return name, ndx

def
_process_range
(
self
,
selection
,
name
=
)
:
try
:
first
,
last
,
gmx_atomname
=
selection
except
ValueError
:
try
:
first
,
last
=
selection
gmx_atomname
=
'*'
except
:
logger
.
error
(
"%r is not a valid range selection"
,
selection
)
raise
if
name
is
:
name
=
"{first!s}-{last!s}_{gmx_atomname!s}"
.
format
(
**
vars
(
)
)
_first
=
self
.
_translate_residue
(
first
,
default_atomname
=
gmx_atomname
)
_last
=
self
.
_translate_residue
(
last
,
default_atomname
=
gmx_atomname
)
_selection
=
'r {0:d} - {1:d} & & a {2!s}'
.
format
(
_first
[
'resid'
]
,
_last
[
'resid'
]
,
gmx_atomname
)
cmd
=
[
'keep 0'
,
'del 0'
,
_selection
,
'name 0 {name!s}'
.
format
(
**
vars
(
)
)
,
'q'
]
fd
,
ndx
=
tempfile
.
mkstemp
(
suffix
=
'.ndx'
,
prefix
=
name
+
'__'
)
rc
,
out
,
err
=
self
.
make_ndx
(
n
=
self
.
ndx
,
o
=
ndx
,
input
=
cmd
)
self
.
check_output
(
out
,
"No atoms found for "
"%(selection)r
%
vars
(
)
)
return
name
,
ndx


def _translate_residue(self, selection, default_atomname='CA'):
        """Translate selection for a single res to make_ndx syntax."""
        m = self.RESIDUE.match(selection)
        if not m:
            errmsg = "Selection {selection!r} is not valid.".format(**vars())
            logger.error(errmsg)
            raise ValueError(errmsg)

        gmx_resid = self.gmx_resid(int(m.group('resid')))    # magic offset correction
        residue = m.group('aa')
        if len(residue) == 1:
            gmx_resname = utilities.convert_aa_code(residue) # only works for AA
        else:
            gmx_resname = residue                            # use 3-letter for any resname

        gmx_atomname = m.group('atom')
        if gmx_atomname is None:
            gmx_atomname = default_atomname

        return {'resname':gmx_resname, 'resid':gmx_resid, 'atomname':gmx_atomname}

def
_translate_residue
(
self
,
selection
,
default_atomname
=
'CA'
)
:
m
=
self
.
RESIDUE
.
match
(
selection
)
if
not
m
:
errmsg
=
"Selection {selection!r} is not valid."
.
format
(
**
vars
(
)
)
logger
.
error
(
errmsg
)
raise
ValueError
(
errmsg
)
gmx_resid
=
self
.
gmx_resid
(
int
(
m
.
group
(
'resid'
)
)
)
residue
=
m
.
group
(
'aa'
)
if
len
(
residue
)
==
1
:
gmx_resname
=
utilities
.
convert_aa_code
(
residue
)
else
:
gmx_resname
=
residue
gmx_atomname
=
m
.
group
(
'atom'
)
if
gmx_atomname
is
:
gmx_atomname
=
default_atomname
return
{
'resname'
:
gmx_resname
,
'resid'
:
gmx_resid
,
'atomname'
:
gmx_atomname
}

Translate selection for a single res to make_ndx syntax .


def check_output(self, make_ndx_output, message=None, err=None):
        """Simple tests to flag problems with a ``make_ndx`` run."""
        if message is None:
            message = ""
        else:
            message = '\n' + message
        def format(output, w=60):
            hrule = "====[ GromacsError (diagnostic output) ]".ljust(w,"=")
            return hrule + '\n' + str(output) + hrule

        rc = True
        if self._is_empty_group(make_ndx_output):
            warnings.warn("Selection produced empty group.{message!s}".format(**vars()), category=GromacsValueWarning)
            rc = False
        if self._has_syntax_error(make_ndx_output):
            rc = False
            out_formatted = format(make_ndx_output)
            raise GromacsError("make_ndx encountered a Syntax Error, "
                               "%(message)s\noutput:\n%(out_formatted)s" % vars())
        if make_ndx_output.strip() == "":
            rc = False
            out_formatted = format(err)
            raise GromacsError("make_ndx produced no output, "
                               "%(message)s\nerror output:\n%(out_formatted)s" % vars())
        return rc

Simple tests to flag problems with a make_ndx run .


def outfile(self, p):
        """Path for an output file.

        If :attr:`outdir` is set then the path is
        ``outdir/basename(p)`` else just ``p``
        """
        if self.outdir is not None:
            return os.path.join(self.outdir, os.path.basename(p))
        else:
            return p

def outfile ( self , p ) : if self . outdir is not None : return os . path . join ( self . outdir , os . path . basename ( p ) ) else : return p


def center_fit(self, **kwargs):
        """Write compact xtc that is fitted to the tpr reference structure.

        See :func:`gromacs.cbook.trj_fitandcenter` for details and
        description of *kwargs* (including *input*, *input1*, *n* and
        *n1* for how to supply custom index groups). The most important ones are listed
        here but in most cases the defaults should work.

        :Keywords:
           *s*
             Input structure (typically the default tpr file but can be set to
             some other file with a different conformation for fitting)
           *n*
             Alternative index file.
           *o*
             Name of the output trajectory.
           *xy* : Boolean
             If ``True`` then only fit in xy-plane (useful for a membrane normal
             to z). The default is ``False``.
           *force*
             - ``True``: overwrite existing trajectories
             - ``False``: throw a IOError exception
             - ``None``: skip existing and log a warning [default]

        :Returns:
              dictionary with keys *tpr*, *xtc*, which are the names of the
              the new files
        """
        kwargs.setdefault('s', self.tpr)
        kwargs.setdefault('n', self.ndx)
        kwargs['f'] = self.xtc
        kwargs.setdefault('o', self.outfile(self.infix_filename(None, self.xtc, '_centfit', 'xtc')))
        force = kwargs.pop('force', self.force)

        logger.info("Centering and fitting trajectory {f!r}...".format(**kwargs))
        with utilities.in_dir(self.dirname):
            if not self.check_file_exists(kwargs['o'], resolve="indicate", force=force):
                trj_fitandcenter(**kwargs)
            logger.info("Centered and fit trajectory: {o!r}.".format(**kwargs))
        return {'tpr': self.rp(kwargs['s']), 'xtc': self.rp(kwargs['o'])}

Write compact xtc that is fitted to the tpr reference structure .


def fit(self, xy=False, **kwargs):
        """Write xtc that is fitted to the tpr reference structure.

        Runs :class:`gromacs.tools.trjconv` with appropriate arguments
        for fitting. The most important *kwargs* are listed
        here but in most cases the defaults should work.

        Note that the default settings do *not* include centering or
        periodic boundary treatment as this often does not work well
        with fitting. It is better to do this as a separate step (see
        :meth:`center_fit` or :func:`gromacs.cbook.trj_fitandcenter`)

        :Keywords:
           *s*
             Input structure (typically the default tpr file but can be set to
             some other file with a different conformation for fitting)
           *n*
             Alternative index file.
           *o*
             Name of the output trajectory. A default name is created.
             If e.g. *dt* = 100  is one of the *kwargs* then the default name includes
             "_dt100ps".
          *xy* : boolean
             If ``True`` then only do a rot+trans fit in the xy plane
             (good for membrane simulations); default is ``False``.
          *force*
            ``True``: overwrite existing trajectories
            ``False``: throw a IOError exception
            ``None``: skip existing and log a warning [default]
          *fitgroup*
            index group to fit on ["backbone"]

            .. Note:: If keyword *input* is supplied then it will override
                      *fitgroup*; *input* = ``[fitgroup, outgroup]``
          *kwargs*
             kwargs are passed to :func:`~gromacs.cbook.trj_xyfitted`

        :Returns:
              dictionary with keys *tpr*, *xtc*, which are the names of the
              the new files
        """
        kwargs.setdefault('s', self.tpr)
        kwargs.setdefault('n', self.ndx)
        kwargs['f'] = self.xtc
        force = kwargs.pop('force', self.force)
        if xy:
            fitmode = 'rotxy+transxy'
            kwargs.pop('fit', None)
            infix_default = '_fitxy'
        else:
            fitmode = kwargs.pop('fit', 'rot+trans')  # user can use 'progressive', too
            infix_default = '_fit'

        dt = kwargs.get('dt')
        if dt:
            infix_default += '_dt{0:d}ps'.format(int(dt))    # dt in ps

        kwargs.setdefault('o', self.outfile(self.infix_filename(None, self.xtc, infix_default, 'xtc')))
        fitgroup = kwargs.pop('fitgroup', 'backbone')
        kwargs.setdefault('input', [fitgroup, "system"])

        if kwargs.get('center', False):
            logger.warn("Transformer.fit(): center=%(center)r used: centering should not be combined with fitting.", kwargs)
            if len(kwargs['inputs']) != 3:
                logger.error("If you insist on centering you must provide three groups in the 'input' kwarg: (center, fit, output)")
                raise ValuError("Insufficient index groups for centering,fitting,output")

        logger.info("Fitting trajectory %r to with xy=%r...", kwargs['f'], xy)
        logger.info("Fitting on index group %(fitgroup)r", vars())
        with utilities.in_dir(self.dirname):
            if self.check_file_exists(kwargs['o'], resolve="indicate", force=force):
                logger.warn("File %r exists; force regenerating it with force=True.", kwargs['o'])
            else:
                gromacs.trjconv(fit=fitmode, **kwargs)
                logger.info("Fitted trajectory (fitmode=%s): %r.", fitmode, kwargs['o'])
        return {'tpr': self.rp(kwargs['s']), 'xtc': self.rp(kwargs['o'])}

Write xtc that is fitted to the tpr reference structure .


def strip_fit(self, **kwargs):
        """Strip water and fit to the remaining system.

        First runs :meth:`strip_water` and then :meth:`fit`; see there
        for arguments.

        - *strip_input* is used for :meth:`strip_water` (but is only useful in
          special cases, e.g. when there is no Protein group defined. Then set
          *strip_input* = ``['Other']``.

        - *input* is passed on to :meth:`fit` and can contain the
          ``[center_group, fit_group, output_group]``

        - *fitgroup* is only passed to :meth:`fit` and just contains
          the group to fit to ("backbone" by default)

          .. warning:: *fitgroup* can only be a Gromacs default group and not
                       a custom group (because the indices change after stripping)

        - By default *fit* = "rot+trans" (and *fit* is passed to :meth:`fit`,
          together with the *xy* = ``False`` keyword)

        .. Note:: The call signature of :meth:`strip_water` is somewhat different from this one.
        """
        kwargs.setdefault('fit', 'rot+trans')
        kw_fit = {}
        for k in ('xy', 'fit', 'fitgroup', 'input'):
            if k in kwargs:
                kw_fit[k] = kwargs.pop(k)

        kwargs['input'] = kwargs.pop('strip_input', ['Protein'])
        kwargs['force'] = kw_fit['force'] = kwargs.pop('force', self.force)

        paths = self.strip_water(**kwargs)    # updates self.nowater
        transformer_nowater = self.nowater[paths['xtc']]  # make sure to get the one we just produced
        return transformer_nowater.fit(**kw_fit)

def strip_fit ( self , ** kwargs ) : kwargs . setdefault ( 'fit' , 'rot+trans' ) kw_fit = { } for k in ( 'xy' , 'fit' , 'fitgroup' , 'input' ) : if k in kwargs : kw_fit [ k ] = kwargs . pop ( k ) kwargs [ 'input' ] = kwargs . pop ( 'strip_input' , [ 'Protein' ] ) kwargs [ 'force' ] = kw_fit [ 'force' ] = kwargs . pop ( 'force' , self . force ) paths = self . strip_water ( ** kwargs ) transformer_nowater = self . nowater [ paths [ 'xtc' ] ] return transformer_nowater . fit ( ** kw_fit )

Strip water and fit to the remaining system .


def create(logger_name, logfile='gromacs.log'):
    """Create a top level logger.

    - The file logger logs everything (including DEBUG).
    - The console logger only logs INFO and above.

    Logging to a file and the console.

    See http://docs.python.org/library/logging.html?#logging-to-multiple-destinations

    The top level logger of the library is named 'gromacs'.  Note that
    we are configuring this logger with console output. If the root
    logger also does this then we will get two output lines to the
    console. We'll live with this because this is a simple
    convenience library...
    """

    logger = logging.getLogger(logger_name)

    logger.setLevel(logging.DEBUG)

    logfile = logging.FileHandler(logfile)
    logfile_formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
    logfile.setFormatter(logfile_formatter)
    logger.addHandler(logfile)

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
    console.setFormatter(formatter)

    logger.addHandler(console)

    return logger


def get_configuration(filename=CONFIGNAME):
    """Reads and parses the configuration file.

    Default values are loaded and then replaced with the values from
    ``~/.gromacswrapper.cfg`` if that file exists. The global
    configuration instance :data:`gromacswrapper.config.cfg` is updated
    as are a number of global variables such as :data:`configdir`,
    :data:`qscriptdir`, :data:`templatesdir`, :data:`logfilename`, ...

    Normally, the configuration is only loaded when the :mod:`gromacs`
    package is imported but a re-reading of the configuration can be forced
    anytime by calling :func:`get_configuration`.

    :Returns: a dict with all updated global configuration variables
    """
    global cfg, configuration    # very iffy --- most of the whole config mod should a class

    #: :data:`cfg` is the instance of :class:`GMXConfigParser` that makes all
    #: global configuration data accessible
    cfg = GMXConfigParser(filename=filename)   # update module-level cfg
    globals().update(cfg.configuration)        # update configdir, templatesdir ...
    configuration = cfg.configuration          # update module-level configuration
    return cfg

def get_configuration ( filename = CONFIGNAME ) : global cfg , configuration cfg = GMXConfigParser ( filename = filename ) globals ( ) . update ( cfg . configuration ) configuration = cfg . configuration return cfg

Reads and parses the configuration file .


def setup(filename=CONFIGNAME):
     """Prepare a default GromacsWrapper global environment.

     1) Create the global config file.
     2) Create the directories in which the user can store template and config files.

     This function can be run repeatedly without harm.
     """
     # setup() must be separate and NOT run automatically when config
     # is loaded so that easy_install installations work
     # (otherwise we get a sandbox violation)
     # populate cfg with defaults (or existing data)
     get_configuration()
     if not os.path.exists(filename):
          with open(filename, 'w') as configfile:
               cfg.write(configfile)  # write the default file so that user can edit
               msg = "NOTE: GromacsWrapper created the configuration file \n\t%r\n" \
                     "      for you. Edit the file to customize the package." % filename
               print(msg)

     # directories
     for d in config_directories:
          utilities.mkdir_p(d)

Prepare a default GromacsWrapper global environment .


def check_setup():
     """Check if templates directories are setup and issue a warning and help.

    Set the environment variable  :envvar:`GROMACSWRAPPER_SUPPRESS_SETUP_CHECK`
    skip the check and make it always return ``True``

    :return ``True`` if directories were found and ``False`` otherwise

     .. versionchanged:: 0.3.1
        Uses :envvar:`GROMACSWRAPPER_SUPPRESS_SETUP_CHECK` to suppress check
        (useful for scripts run on a server)
     """

     if "GROMACSWRAPPER_SUPPRESS_SETUP_CHECK" in os.environ:
         return True

     missing = [d for d in config_directories if not os.path.exists(d)]
     if len(missing) > 0:
         print("NOTE: Some configuration directories are not set up yet: ")
         print("\t{0!s}".format('\n\t'.join(missing)))
         print("NOTE: You can create the configuration file and directories with:")
         print("\t>>> import gromacs")
         print("\t>>> gromacs.config.setup()")
         return False
     return True

Check if templates directories are setup and issue a warning and help .


def get_tool_names():
    """ Get tool names from all configured groups.

    :return: list of tool names
    """
    names = []
    for group in cfg.get('Gromacs', 'groups').split():
        names.extend(cfg.get('Gromacs', group).split())
    return names

def get_tool_names ( ) : names = [ ] for group in cfg . get ( 'Gromacs' , 'groups' ) . split ( ) : names . extend ( cfg . get ( 'Gromacs' , group ) . split ( ) ) return names

Get tool names from all configured groups .


def configuration(self):
          """Dict of variables that we make available as globals in the module.

          Can be used as ::

             globals().update(GMXConfigParser.configuration)        # update configdir, templatesdir ...
          """
          configuration = {
               'configfilename': self.filename,
               'logfilename': self.getpath('Logging', 'logfilename'),
               'loglevel_console': self.getLogLevel('Logging', 'loglevel_console'),
               'loglevel_file': self.getLogLevel('Logging', 'loglevel_file'),
               'configdir': self.getpath('DEFAULT', 'configdir'),
               'qscriptdir': self.getpath('DEFAULT', 'qscriptdir'),
               'templatesdir': self.getpath('DEFAULT', 'templatesdir'),
               }
          configuration['path'] = [os.path.curdir,
                                   configuration['qscriptdir'],
                                   configuration['templatesdir']]
          return configuration

def configuration ( self ) : configuration = { 'configfilename' : self . filename , 'logfilename' : self . getpath ( 'Logging' , 'logfilename' ) , 'loglevel_console' : self . getLogLevel ( 'Logging' , 'loglevel_console' ) , 'loglevel_file' : self . getLogLevel ( 'Logging' , 'loglevel_file' ) , 'configdir' : self . getpath ( 'DEFAULT' , 'configdir' ) , 'qscriptdir' : self . getpath ( 'DEFAULT' , 'qscriptdir' ) , 'templatesdir' : self . getpath ( 'DEFAULT' , 'templatesdir' ) , } configuration [ 'path' ] = [ os . path . curdir , configuration [ 'qscriptdir' ] , configuration [ 'templatesdir' ] ] return configuration

Dict of variables that we make available as globals in the module .


def getpath(self, section, option):
          """Return option as an expanded path."""
          return os.path.expanduser(os.path.expandvars(self.get(section, option)))

def getpath ( self , section , option ) : return os . path . expanduser ( os . path . expandvars ( self . get ( section , option ) ) )


def getLogLevel(self, section, option):
          """Return the textual representation of logging level 'option' or the number.

          Note that option is always interpreted as an UPPERCASE string
          and hence integer log levels will not be recognized.

          .. SeeAlso: :mod:`logging` and :func:`logging.getLevelName`
          """
          return logging.getLevelName(self.get(section, option).upper())

def getLogLevel ( self , section , option ) : return logging . getLevelName ( self . get ( section , option ) . upper ( ) )

Return the textual representation of logging level option or the number .


def _canonicalize(self, filename):
        """Use .collection as extension unless provided"""
        path, ext = os.path.splitext(filename)
        if not ext:
            ext = ".collection"
        return path + ext

def _canonicalize ( self , filename ) : path , ext = os . path . splitext ( filename ) if not ext : ext = ".collection" return path + ext

Use . collection as extension unless provided


def scale_dihedrals(mol, dihedrals, scale, banned_lines=None):
        """Scale dihedral angles"""

        if banned_lines is None:
                banned_lines = []
        new_dihedrals = []
        for dh in mol.dihedrals:
                atypes = dh.atom1.get_atomtype(), dh.atom2.get_atomtype(), dh.atom3.get_atomtype(), dh.atom4.get_atomtype()
                atypes = [a.replace("_", "").replace("=","") for a in atypes]

                # special-case: this is a [ dihedral ] override in molecule block, continue and don't match
                if dh.gromacs['param'] != []:
                    for p in dh.gromacs['param']:
                        p['kch'] *= scale
                    new_dihedrals.append(dh)
                    continue

                for iswitch in range(32):
                        if  (iswitch%2==0 ):
                                a1=atypes[0]; a2=atypes[1]; a3=atypes[2]; a4=atypes[3]
                        else:
                                a1=atypes[3]; a2=atypes[2]; a3=atypes[1]; a4=atypes[0]

                        if((iswitch//2)%2==1): a1="X";
                        if((iswitch//4)%2==1): a2="X";
                        if((iswitch//8)%2==1): a3="X";
                        if((iswitch//16)%2==1): a4="X";
                        key = "{0}-{1}-{2}-{3}-{4}".format(a1, a2, a3, a4, dh.gromacs['func'])
                        if (key in dihedrals):
                                for i, dt in enumerate(dihedrals[key]):
                                        dhA = copy.deepcopy(dh)
                                        param = copy.deepcopy(dt.gromacs['param'])
                                        # Only check the first dihedral in a list
                                        if not dihedrals[key][0].line in banned_lines:
                                                for p in param: p['kchi'] *= scale
                                        dhA.gromacs['param'] = param
                                        #if key == "CT3-C-NH1-CT1-9": print i, dt, key
                                        if i == 0:
                                                dhA.comment = "; banned lines {0} found={1}\n".format(" ".join(
                                                        map(str, banned_lines)), 1 if dt.line in banned_lines else 0)
                                                dhA.comment += "; parameters for types {}-{}-{}-{}-9 at LINE({})\n".format(
                                                        dhA.atom1.atomtype, dhA.atom2.atomtype, dhA.atom3.atomtype,
                                                        dhA.atom4.atomtype, dt.line).replace("_","")
                                        name = "{}-{}-{}-{}-9".format(dhA.atom1.atomtype, dhA.atom2.atomtype,
                                                                      dhA.atom3.atomtype, dhA.atom4.atomtype).replace("_","")
                                        #if name == "CL-CTL2-CTL2-HAL2-9": print dihedrals[key], key
                                        new_dihedrals.append(dhA)
                                break

        mol.dihedrals = new_dihedrals
        #assert(len(mol.dihedrals) == new_dihedrals)
        return mol


def scale_impropers(mol, impropers, scale, banned_lines=None):
        """Scale improper dihedrals"""
        if banned_lines is None:
                banned_lines = []
        new_impropers = []
        for im in mol.impropers:
                atypes = (im.atom1.get_atomtype(), im.atom2.get_atomtype(),
                          im.atom3.get_atomtype(), im.atom4.get_atomtype())
                atypes = [a.replace("_", "").replace("=", "") for a in atypes]

                # special-case: this is a [ dihedral ] override in molecule block, continue and don't match
                if im.gromacs['param'] != []:
                    for p in im.gromacs['param']:
                        p['kpsi'] *= scale
                    new_impropers.append(im)
                    continue

                for iswitch in range(32):
                        if  (iswitch%2==0):
                                a1=atypes[0]; a2=atypes[1]; a3=atypes[2]; a4=atypes[3];
                        else:
                                a1=atypes[3]; a2=atypes[2]; a3=atypes[1]; a4=atypes[0];
                        if((iswitch//2)%2==1): a1="X";
                        if((iswitch//4)%2==1): a2="X";
                        if((iswitch//8)%2==1): a3="X";
                        if((iswitch//16)%2==1): a4="X";
                        key = "{0}-{1}-{2}-{3}-{4}".format(a1, a2, a3, a4, im.gromacs['func'])
                        if (key in impropers):
                                for i, imt in enumerate(impropers[key]):
                                        imA = copy.deepcopy(im)
                                        param = copy.deepcopy(imt.gromacs['param'])
                                        # Only check the first dihedral in a list
                                        if not impropers[key][0].line in banned_lines:
                                                for p in param: p['kpsi'] *= scale
                                        imA.gromacs['param'] = param
                                        if i == 0:
                                                imA.comment = "; banned lines {0} found={1}\n ; parameters for types {2}-{3}-{4}-{5}-9 at LINE({6})\n".format(
                                                        " ".join(map(str, banned_lines)),
                                                        1 if imt.line in banned_lines else 0,
                                                        imt.atype1, imt.atype2, imt.atype3, imt.atype4, imt.line)
                                        new_impropers.append(imA)
                                break
        #assert(len(mol.impropers) == new_impropers)
        mol.impropers = new_impropers
        return mol