def _update_secrets(self):
        '''update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. This is where you
           should do any customization of the secrets flie, or using
           it to update your client, if needed.
        '''
        # Get a setting for client myclient and some variable name VAR.
        # returns None if not set
        setting = self._get_setting('SREGISTRY_MYCLIENT_VAR')

        # Get (and if found in environment (1) settings (2) update the variable
        # It will still return None if not set
        setting = self._get_and_update_setting('SREGISTRY_MYCLIENT_VAR')

        # If you have a setting that is required and not found, you should exit.

        # Here is how to read all client secrets
        self.secrets = read_client_secrets()

        # If you don't want to use the shared settings file, you have your own.
        # Here is how to get if the user has a cache for you enabled, this
        # returns a path (enabled) or None (disabled) that you should honor
        # You can use this as a file path or folder and for both cases, you
        # need to create the file or folder
        if self._credential_cache is not None:
            bot.info("credential cache set to %s" %self._credential_cache)

def
_update_secrets
(
self
)
:
setting
=
self
.
_get_setting
(
'SREGISTRY_MYCLIENT_VAR'
)
setting
=
self
.
_get_and_update_setting
(
'SREGISTRY_MYCLIENT_VAR'
)
self
.
secrets
=
read_client_secrets
(
)
if
self
.
_credential_cache
is
not
:
bot
.
info
(
"credential cache set to %s"
%
self
.
_credential_cache
)

update secrets will take a secrets credential file either located at . sregistry or the environment variable SREGISTRY_CLIENT_SECRETS and update the current client secrets as well as the associated API base . This is where you should do any customization of the secrets flie or using it to update your client if needed .


def _make_repr(class_name, *args, **kwargs):
    """
    Generate a repr string.

    Positional arguments should be the positional arguments used to
    construct the class. Keyword arguments should consist of tuples of
    the attribute value and default. If the value is the default, then
    it won't be rendered in the output.

    Here's an example::

        def __repr__(self):
            return make_repr('MyClass', 'foo', name=(self.name, None))

    The output of this would be something line ``MyClass('foo',
    name='Will')``.

    """
    arguments = [repr(arg) for arg in args]
    arguments.extend(
        "{}={!r}".format(name, value)
        for name, (value, default) in sorted(kwargs.items())
        if value != default
    )
    return "{}({})".format(class_name, ", ".join(arguments))

def
_make_repr
(
class_name
,
*
args
,
**
kwargs
)
:
arguments
=
[
repr
(
arg
)
for
arg
in
args
]
arguments
.
extend
(
"{}={!r}"
.
format
(
name
,
value
)
for
name
,
(
value
,
default
)
in
sorted
(
kwargs
.
items
(
)
)
if
value
!=
default
)
return
"{}({})"
.
format
(
class_name
,
", "
.
join
(
arguments
)
)


def s3errors(path):
    """Translate S3 errors to FSErrors."""
    try:
        yield
    except ClientError as error:
        _error = error.response.get("Error", {})
        error_code = _error.get("Code", None)
        response_meta = error.response.get("ResponseMetadata", {})
        http_status = response_meta.get("HTTPStatusCode", 200)
        error_msg = _error.get("Message", None)
        if error_code == "NoSuchBucket":
            raise errors.ResourceError(path, exc=error, msg=error_msg)
        if http_status == 404:
            raise errors.ResourceNotFound(path)
        elif http_status == 403:
            raise errors.PermissionDenied(path=path, msg=error_msg)
        else:
            raise errors.OperationFailed(path=path, exc=error)
    except SSLError as error:
        raise errors.OperationFailed(path, exc=error)
    except EndpointConnectionError as error:
        raise errors.RemoteConnectionError(path, exc=error, msg="{}".format(error))

def s3errors ( path ) : try : yield except ClientError as error : _error = error . response . get ( "Error" , { } ) error_code = _error . get ( "Code" , None ) response_meta = error . response . get ( "ResponseMetadata" , { } ) http_status = response_meta . get ( "HTTPStatusCode" , 200 ) error_msg = _error . get ( "Message" , None ) if error_code == "NoSuchBucket" : raise errors . ResourceError ( path , exc = error , msg = error_msg ) if http_status == 404 : raise errors . ResourceNotFound ( path ) elif http_status == 403 : raise errors . PermissionDenied ( path = path , msg = error_msg ) else : raise errors . OperationFailed ( path = path , exc = error ) except SSLError as error : raise errors . OperationFailed ( path , exc = error ) except EndpointConnectionError as error : raise errors . RemoteConnectionError ( path , exc = error , msg = "{}" . format ( error ) )


def factory(cls, filename, mode, on_close):
        """Create a S3File backed with a temporary file."""
        _temp_file = tempfile.TemporaryFile()
        proxy = cls(_temp_file, filename, mode, on_close=on_close)
        return proxy

def factory ( cls , filename , mode , on_close ) : _temp_file = tempfile . TemporaryFile ( ) proxy = cls ( _temp_file , filename , mode , on_close = on_close ) return proxy

Create a S3File backed with a temporary file .


def gravatar_url(user_or_email, size=GRAVATAR_DEFAULT_SIZE):
    """ Builds a gravatar url from an user or email """
    if hasattr(user_or_email, 'email'):
        email = user_or_email.email
    else:
        email = user_or_email

    try:
        return escape(get_gravatar_url(email=email, size=size))
    except:
        return ''

def gravatar_url ( user_or_email , size = GRAVATAR_DEFAULT_SIZE ) : if hasattr ( user_or_email , 'email' ) : email = user_or_email . email else : email = user_or_email try : return escape ( get_gravatar_url ( email = email , size = size ) ) except : return ''


def get_gravatar_url(email, size=GRAVATAR_DEFAULT_SIZE, default=GRAVATAR_DEFAULT_IMAGE,
        rating=GRAVATAR_DEFAULT_RATING, secure=GRAVATAR_DEFAULT_SECURE):
    """
    Builds a url to a gravatar from an email address.

    :param email: The email to fetch the gravatar for
    :param size: The size (in pixels) of the gravatar to fetch
    :param default: What type of default image to use if the gravatar does not exist
    :param rating: Used to filter the allowed gravatar ratings
    :param secure: If True use https, otherwise plain http
    """
    if secure:
        url_base = GRAVATAR_SECURE_URL
    else:
        url_base = GRAVATAR_URL

    # Calculate the email hash
    email_hash = calculate_gravatar_hash(email)

    # Build querystring
    query_string = urlencode({
        's': str(size),
        'd': default,
        'r': rating,
    })

    # Build url
    url = '{base}avatar/{hash}.jpg?{qs}'.format(base=url_base,
            hash=email_hash, qs=query_string)

    return url

def get_gravatar_url ( email , size = GRAVATAR_DEFAULT_SIZE , default = GRAVATAR_DEFAULT_IMAGE , rating = GRAVATAR_DEFAULT_RATING , secure = GRAVATAR_DEFAULT_SECURE ) : if secure : url_base = GRAVATAR_SECURE_URL else : url_base = GRAVATAR_URL email_hash = calculate_gravatar_hash ( email ) query_string = urlencode ( { 's' : str ( size ) , 'd' : default , 'r' : rating , } ) url = '{base}avatar/{hash}.jpg?{qs}' . format ( base = url_base , hash = email_hash , qs = query_string ) return url

Builds a url to a gravatar from an email address .


def has_gravatar(email):
    """
    Returns True if the user has a gravatar, False if otherwise
    """
    # Request a 404 response if the gravatar does not exist
    url = get_gravatar_url(email, default=GRAVATAR_DEFAULT_IMAGE_404)

    # Verify an OK response was received
    try:
        request = Request(url)
        request.get_method = lambda: 'HEAD'
        return 200 == urlopen(request).code
    except (HTTPError, URLError):
        return False

def has_gravatar ( email ) : url = get_gravatar_url ( email , default = GRAVATAR_DEFAULT_IMAGE_404 ) try : request = Request ( url ) request . get_method = lambda : 'HEAD' return 200 == urlopen ( request ) . code except ( HTTPError , URLError ) : return False

Returns True if the user has a gravatar False if otherwise


def get_gravatar_profile_url(email, secure=GRAVATAR_DEFAULT_SECURE):
    """
    Builds a url to a gravatar profile from an email address.

    :param email: The email to fetch the gravatar for
    :param secure: If True use https, otherwise plain http
    """
    if secure:
        url_base = GRAVATAR_SECURE_URL
    else:
        url_base = GRAVATAR_URL

    # Calculate the email hash
    email_hash = calculate_gravatar_hash(email)

    # Build url
    url = '{base}{hash}'.format(base=url_base, hash=email_hash)

    return url

def get_gravatar_profile_url ( email , secure = GRAVATAR_DEFAULT_SECURE ) : if secure : url_base = GRAVATAR_SECURE_URL else : url_base = GRAVATAR_URL email_hash = calculate_gravatar_hash ( email ) url = '{base}{hash}' . format ( base = url_base , hash = email_hash ) return url

Builds a url to a gravatar profile from an email address .


def chimera_blocks(M=16, N=16, L=4):
    """
    Generator for blocks for a chimera block quotient
    """
    for x in xrange(M):
        for y in xrange(N):
            for u in (0, 1):
                yield tuple((x, y, u, k) for k in xrange(L))

def chimera_blocks ( M = 16 , N = 16 , L = 4 ) : for x in xrange ( M ) : for y in xrange ( N ) : for u in ( 0 , 1 ) : yield tuple ( ( x , y , u , k ) for k in xrange ( L ) )


def chimera_block_quotient(G, blocks):
    """
    Extract the blocks from a graph, and returns a
    block-quotient graph according to the acceptability
    functions block_good and eblock_good

    Inputs:
        G: a networkx graph
        blocks: a tuple of tuples

    """
    from networkx import Graph
    from itertools import product

    BG = Graph()
    blockid = {}
    for i, b in enumerate(blocks):
        BG.add_node(i)
        if not b or not all(G.has_node(x) for x in b):
            continue
        for q in b:
            if q in blockid:
                raise(RuntimeError, "two blocks overlap")
            blockid[q] = i

    for q, u in blockid.items():
        ublock = blocks[u]
        for p in G[q]:
            if p not in blockid:
                continue
            v = blockid[p]
            if BG.has_edge(u, v) or u == v:
                continue
            vblock = blocks[v]

            if ublock[0][2] == vblock[0][2]:
                block_edges = zip(ublock, vblock)
            else:
                block_edges = product(ublock, vblock)

            if all(G.has_edge(x, y) for x, y in block_edges):
                BG.add_edge(u, v)

    return BG

def
chimera_block_quotient
(
G
,
blocks
)
:
from
networkx
import
Graph
from
itertools
import
product
BG
=
Graph
(
)
blockid
=
{
}
for
i
,
b
in
enumerate
(
blocks
)
:
BG
.
add_node
(
i
)
if
not
b
or
not
all
(
G
.
has_node
(
x
)
for
x
in
b
)
:
continue
for
q
in
b
:
if
q
in
blockid
:
raise
(
RuntimeError
,
"two blocks overlap"
)
blockid
[
q
]
=
i
for
q
,
u
in
blockid
.
items
(
)
:
ublock
=
blocks
[
u
]
for
p
in
G
[
q
]
:
if
p
not
in
blockid
:
continue
v
=
blockid
[
p
]
if
BG
.
has_edge
(
u
,
v
)
or
u
==
v
:
continue
vblock
=
blocks
[
v
]
if
ublock
[
0
]
[
2
]
==
vblock
[
0
]
[
2
]
:
block_edges
=
zip
(
ublock
,
vblock
)
else
:
block_edges
=
product
(
ublock
,
vblock
)
if
all
(
G
.
has_edge
(
x
,
y
)
for
x
,
y
in
block_edges
)
:
BG
.
add_edge
(
u
,
v
)
return
BG

Extract the blocks from a graph and returns a block - quotient graph according to the acceptability functions block_good and eblock_good


def enumerate_resonance_smiles(smiles):
    """Return a set of resonance forms as SMILES strings, given a SMILES string.

    :param smiles: A SMILES string.
    :returns: A set containing SMILES strings for every possible resonance form.
    :rtype: set of strings.
    """
    mol = Chem.MolFromSmiles(smiles)
    #Chem.SanitizeMol(mol)  # MolFromSmiles does Sanitize by default
    mesomers = ResonanceEnumerator().enumerate(mol)
    return {Chem.MolToSmiles(m, isomericSmiles=True) for m in mesomers}

def enumerate_resonance_smiles ( smiles ) : mol = Chem . MolFromSmiles ( smiles ) mesomers = ResonanceEnumerator ( ) . enumerate ( mol ) return { Chem . MolToSmiles ( m , isomericSmiles = True ) for m in mesomers }

Return a set of resonance forms as SMILES strings given a SMILES string .


def enumerate(self, mol):
        """Enumerate all possible resonance forms and return them as a list.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: A list of all possible resonance forms of the molecule.
        :rtype: list of rdkit.Chem.rdchem.Mol
        """
        flags = 0
        if self.kekule_all:
            flags = flags | Chem.KEKULE_ALL
        if self.allow_incomplete_octets:
            flags = flags | Chem.ALLOW_INCOMPLETE_OCTETS
        if self.allow_charge_separation:
            flags = flags | Chem.ALLOW_CHARGE_SEPARATION
        if self.unconstrained_anions:
            flags = flags | Chem.UNCONSTRAINED_ANIONS
        if self.unconstrained_cations:
            flags = flags | Chem.UNCONSTRAINED_CATIONS
        results = []
        for result in Chem.ResonanceMolSupplier(mol, flags=flags, maxStructs=self.max_structures):
            # This seems necessary? ResonanceMolSupplier only does a partial sanitization
            Chem.SanitizeMol(result)
            results.append(result)
        return results

def enumerate ( self , mol ) : flags = 0 if self . kekule_all : flags = flags | Chem . KEKULE_ALL if self . allow_incomplete_octets : flags = flags | Chem . ALLOW_INCOMPLETE_OCTETS if self . allow_charge_separation : flags = flags | Chem . ALLOW_CHARGE_SEPARATION if self . unconstrained_anions : flags = flags | Chem . UNCONSTRAINED_ANIONS if self . unconstrained_cations : flags = flags | Chem . UNCONSTRAINED_CATIONS results = [ ] for result in Chem . ResonanceMolSupplier ( mol , flags = flags , maxStructs = self . max_structures ) : Chem . SanitizeMol ( result ) results . append ( result ) return results

Enumerate all possible resonance forms and return them as a list .


def normalize(self, mol):
        """Apply a series of Normalization transforms to correct functional groups and recombine charges.

        A series of transforms are applied to the molecule. For each Normalization, the transform is applied repeatedly
        until no further changes occur. If any changes occurred, we go back and start from the first Normalization
        again, in case the changes mean an earlier transform is now applicable. The molecule is returned once the entire
        series of Normalizations cause no further changes or if max_restarts (default 200) is reached.

        :param mol: The molecule to normalize.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The normalized fragment.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running Normalizer')
        # Normalize each fragment separately to get around quirky RunReactants behaviour
        fragments = []
        for fragment in Chem.GetMolFrags(mol, asMols=True):
            fragments.append(self._normalize_fragment(fragment))
        # Join normalized fragments into a single molecule again
        outmol = fragments.pop()
        for fragment in fragments:
            outmol = Chem.CombineMols(outmol, fragment)
        Chem.SanitizeMol(outmol)
        return outmol

def
normalize
(
self
,
mol
)
:
log
.
debug
(
'Running Normalizer'
)
fragments
=
[
]
for
fragment
in
Chem
.
GetMolFrags
(
mol
,
asMols
=
True
)
:
fragments
.
append
(
self
.
_normalize_fragment
(
fragment
)
)
outmol
=
fragments
.
pop
(
)
for
fragment
in
fragments
:
outmol
=
Chem
.
CombineMols
(
outmol
,
fragment
)
Chem
.
SanitizeMol
(
outmol
)
return
outmol

Apply a series of Normalization transforms to correct functional groups and recombine charges .


def _apply_transform(self, mol, rule):
        """Repeatedly apply normalization transform to molecule until no changes occur.

        It is possible for multiple products to be produced when a rule is applied. The rule is applied repeatedly to
        each of the products, until no further changes occur or after 20 attempts. If there are multiple unique products
        after the final application, the first product (sorted alphabetically by SMILES) is chosen.
        """
        mols = [mol]
        for n in six.moves.range(20):
            products = {}
            for mol in mols:
                for product in [x[0] for x in rule.RunReactants((mol,))]:
                    if Chem.SanitizeMol(product, catchErrors=True) == 0:
                        products[Chem.MolToSmiles(product, isomericSmiles=True)] = product
            if products:
                mols = [products[s] for s in sorted(products)]
            else:
                # If n == 0, the rule was not applicable and we return None
                return mols[0] if n > 0 else None

def _apply_transform ( self , mol , rule ) : mols = [ mol ] for n in six . moves . range ( 20 ) : products = { } for mol in mols : for product in [ x [ 0 ] for x in rule . RunReactants ( ( mol , ) ) ] : if Chem . SanitizeMol ( product , catchErrors = True ) == 0 : products [ Chem . MolToSmiles ( product , isomericSmiles = True ) ] = product if products : mols = [ products [ s ] for s in sorted ( products ) ] else : return mols [ 0 ] if n > 0 else None

Repeatedly apply normalization transform to molecule until no changes occur .


def canonicalize(self, mol):
        """Return a canonical tautomer by enumerating and scoring all possible tautomers.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The canonical tautomer.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        # TODO: Overload the mol parameter to pass a list of pre-enumerated tautomers
        tautomers = self._enumerate_tautomers(mol)
        if len(tautomers) == 1:
            return tautomers[0]
        # Calculate score for each tautomer
        highest = None
        for t in tautomers:
            smiles = Chem.MolToSmiles(t, isomericSmiles=True)
            log.debug('Tautomer: %s', smiles)
            score = 0
            # Add aromatic ring scores
            ssr = Chem.GetSymmSSSR(t)
            for ring in ssr:
                btypes = {t.GetBondBetweenAtoms(*pair).GetBondType() for pair in pairwise(ring)}
                elements = {t.GetAtomWithIdx(idx).GetAtomicNum() for idx in ring}
                if btypes == {BondType.AROMATIC}:
                    log.debug('Score +100 (aromatic ring)')
                    score += 100
                    if elements == {6}:
                        log.debug('Score +150 (carbocyclic aromatic ring)')
                        score += 150
            # Add SMARTS scores
            for tscore in self.scores:
                for match in t.GetSubstructMatches(tscore.smarts):
                    log.debug('Score %+d (%s)', tscore.score, tscore.name)
                    score += tscore.score
            # Add (P,S,Se,Te)-H scores
            for atom in t.GetAtoms():
                if atom.GetAtomicNum() in {15, 16, 34, 52}:
                    hs = atom.GetTotalNumHs()
                    if hs:
                        log.debug('Score %+d (%s-H bonds)', -hs, atom.GetSymbol())
                        score -= hs
            # Set as highest if score higher or if score equal and smiles comes first alphabetically
            if not highest or highest['score'] < score or (highest['score'] == score and smiles < highest['smiles']):
                log.debug('New highest tautomer: %s (%s)', smiles, score)
                highest = {'smiles': smiles, 'tautomer': t, 'score': score}
        return highest['tautomer']

Return a canonical tautomer by enumerating and scoring all possible tautomers .


def validate_smiles(smiles):
    """Return log messages for a given SMILES string using the default validations.

    Note: This is a convenience function for quickly validating a single SMILES string. It is more efficient to use
    the :class:`~molvs.validate.Validator` class directly when working with many molecules or when custom options
    are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: A list of log messages.
    :rtype: list of strings.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles)
    logs = Validator().validate(mol)
    return logs

def validate_smiles ( smiles ) : mol = Chem . MolFromSmiles ( smiles ) logs = Validator ( ) . validate ( mol ) return logs

Return log messages for a given SMILES string using the default validations .


def disconnect(self, mol):
        """Break covalent bonds between metals and organic atoms under certain conditions.

        The algorithm works as follows:

        - Disconnect N, O, F from any metal.
        - Disconnect other non-metals from transition metals + Al (but not Hg, Ga, Ge, In, Sn, As, Tl, Pb, Bi, Po).
        - For every bond broken, adjust the charges of the begin and end atoms accordingly.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The molecule with metals disconnected.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running MetalDisconnector')
        # Remove bonds that match SMARTS
        for smarts in [self._metal_nof, self._metal_non]:
            pairs = mol.GetSubstructMatches(smarts)
            rwmol = Chem.RWMol(mol)
            orders = []
            for i, j in pairs:
                # TODO: Could get the valence contributions of the bond instead of GetBondTypeAsDouble?
                orders.append(int(mol.GetBondBetweenAtoms(i, j).GetBondTypeAsDouble()))
                rwmol.RemoveBond(i, j)
            # Adjust neighbouring charges accordingly
            mol = rwmol.GetMol()
            for n, (i, j) in enumerate(pairs):
                chg = orders[n]
                atom1 = mol.GetAtomWithIdx(i)
                atom1.SetFormalCharge(atom1.GetFormalCharge() + chg)
                atom2 = mol.GetAtomWithIdx(j)
                atom2.SetFormalCharge(atom2.GetFormalCharge() - chg)
                log.info('Removed covalent bond between %s and %s', atom1.GetSymbol(), atom2.GetSymbol())
        Chem.SanitizeMol(mol)
        return mol

Break covalent bonds between metals and organic atoms under certain conditions .


def standardize_smiles(smiles):
    """Return a standardized canonical SMILES string given a SMILES string.

    Note: This is a convenience function for quickly standardizing a single SMILES string. It is more efficient to use
    the :class:`~molvs.standardize.Standardizer` class directly when working with many molecules or when custom options
    are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: The SMILES for the standardized molecule.
    :rtype: string.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    return Chem.MolToSmiles(mol, isomericSmiles=True)

def standardize_smiles ( smiles ) : mol = Chem . MolFromSmiles ( smiles , sanitize = False ) mol = Standardizer ( ) . standardize ( mol ) return Chem . MolToSmiles ( mol , isomericSmiles = True )

Return a standardized canonical SMILES string given a SMILES string .


def enumerate_tautomers_smiles(smiles):
    """Return a set of tautomers as SMILES strings, given a SMILES string.

    :param smiles: A SMILES string.
    :returns: A set containing SMILES strings for every possible tautomer.
    :rtype: set of strings.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    tautomers = TautomerEnumerator().enumerate(mol)
    return {Chem.MolToSmiles(m, isomericSmiles=True) for m in tautomers}

def enumerate_tautomers_smiles ( smiles ) : mol = Chem . MolFromSmiles ( smiles , sanitize = False ) mol = Standardizer ( ) . standardize ( mol ) tautomers = TautomerEnumerator ( ) . enumerate ( mol ) return { Chem . MolToSmiles ( m , isomericSmiles = True ) for m in tautomers }

Return a set of tautomers as SMILES strings given a SMILES string .


def canonicalize_tautomer_smiles(smiles):
    """Return a standardized canonical tautomer SMILES string given a SMILES string.

    Note: This is a convenience function for quickly standardizing and finding the canonical tautomer for a single
    SMILES string. It is more efficient to use the :class:`~molvs.standardize.Standardizer` class directly when working
    with many molecules or when custom options are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: The SMILES for the standardize canonical tautomer.
    :rtype: string.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    tautomer = TautomerCanonicalizer().canonicalize(mol)
    return Chem.MolToSmiles(tautomer, isomericSmiles=True)

def canonicalize_tautomer_smiles ( smiles ) : mol = Chem . MolFromSmiles ( smiles , sanitize = False ) mol = Standardizer ( ) . standardize ( mol ) tautomer = TautomerCanonicalizer ( ) . canonicalize ( mol ) return Chem . MolToSmiles ( tautomer , isomericSmiles = True )

Return a standardized canonical tautomer SMILES string given a SMILES string .


def standardize(self, mol):
        """Return a standardized version the given molecule.

        The standardization process consists of the following stages: RDKit
        :py:func:`~rdkit.Chem.rdmolops.RemoveHs`, RDKit :py:func:`~rdkit.Chem.rdmolops.SanitizeMol`,
        :class:`~molvs.metal.MetalDisconnector`, :class:`~molvs.normalize.Normalizer`,
        :class:`~molvs.charge.Reionizer`, RDKit :py:func:`~rdkit.Chem.rdmolops.AssignStereochemistry`.

        :param mol: The molecule to standardize.
        :type mol: rdkit.Chem.rdchem.Mol
        :returns: The standardized molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        mol = copy.deepcopy(mol)
        Chem.SanitizeMol(mol)
        mol = Chem.RemoveHs(mol)
        mol = self.disconnect_metals(mol)
        mol = self.normalize(mol)
        mol = self.reionize(mol)
        Chem.AssignStereochemistry(mol, force=True, cleanIt=True)
        # TODO: Check this removes symmetric stereocenters
        return mol

def standardize ( self , mol ) : mol = copy . deepcopy ( mol ) Chem . SanitizeMol ( mol ) mol = Chem . RemoveHs ( mol ) mol = self . disconnect_metals ( mol ) mol = self . normalize ( mol ) mol = self . reionize ( mol ) Chem . AssignStereochemistry ( mol , force = True , cleanIt = True ) return mol

Return a standardized version the given molecule .


def tautomer_parent(self, mol, skip_standardize=False):
        """Return the tautomer parent of a given molecule.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The tautomer parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        tautomer = self.canonicalize_tautomer(mol)
        tautomer = self.standardize(tautomer)
        return tautomer

def tautomer_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) tautomer = self . canonicalize_tautomer ( mol ) tautomer = self . standardize ( tautomer ) return tautomer

Return the tautomer parent of a given molecule .


def fragment_parent(self, mol, skip_standardize=False):
        """Return the fragment parent of a given molecule.

        The fragment parent is the largest organic covalent unit in the molecule.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The fragment parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        # TODO: Consider applying FragmentRemover first to remove salts, solvents?
        fragment = self.largest_fragment(mol)
        return fragment

def fragment_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) fragment = self . largest_fragment ( mol ) return fragment

Return the fragment parent of a given molecule .


def stereo_parent(self, mol, skip_standardize=False):
        """Return the stereo parent of a given molecule.

        The stereo parent has all stereochemistry information removed from tetrahedral centers and double bonds.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The stereo parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        else:
            mol = copy.deepcopy(mol)
        Chem.RemoveStereochemistry(mol)
        return mol

def stereo_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) else : mol = copy . deepcopy ( mol ) Chem . RemoveStereochemistry ( mol ) return mol

Return the stereo parent of a given molecule .


def isotope_parent(self, mol, skip_standardize=False):
        """Return the isotope parent of a given molecule.

        The isotope parent has all atoms replaced with the most abundant isotope for that element.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The isotope parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        else:
            mol = copy.deepcopy(mol)
        # Replace isotopes with common weight
        for atom in mol.GetAtoms():
            atom.SetIsotope(0)
        return mol

def isotope_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) else : mol = copy . deepcopy ( mol ) for atom in mol . GetAtoms ( ) : atom . SetIsotope ( 0 ) return mol

Return the isotope parent of a given molecule .


def charge_parent(self, mol, skip_standardize=False):
        """Return the charge parent of a given molecule.

        The charge parent is the uncharged version of the fragment parent.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The charge parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        # TODO: All ionized acids and bases should be neutralised.
        if not skip_standardize:
            mol = self.standardize(mol)
        fragment = self.fragment_parent(mol, skip_standardize=True)
        if fragment:
            uncharged = self.uncharge(fragment)
            # During final standardization, the Reionizer ensures any remaining charges are in the right places
            uncharged = self.standardize(uncharged)
            return uncharged

def charge_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) fragment = self . fragment_parent ( mol , skip_standardize = True ) if fragment : uncharged = self . uncharge ( fragment ) uncharged = self . standardize ( uncharged ) return uncharged

Return the charge parent of a given molecule .


def super_parent(self, mol, skip_standardize=False):
        """Return the super parent of a given molecule.

        THe super parent is fragment, charge, isotope, stereochemistry and tautomer insensitive. From the input
        molecule, the largest fragment is taken. This is uncharged and then isotope and stereochemistry information is
        discarded. Finally, the canonical tautomer is determined and returned.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The super parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        # We don't need to get fragment parent, because the charge parent is the largest fragment
        mol = self.charge_parent(mol, skip_standardize=True)
        mol = self.isotope_parent(mol, skip_standardize=True)
        mol = self.stereo_parent(mol, skip_standardize=True)
        mol = self.tautomer_parent(mol, skip_standardize=True)
        mol = self.standardize(mol)
        return mol

def super_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) mol = self . charge_parent ( mol , skip_standardize = True ) mol = self . isotope_parent ( mol , skip_standardize = True ) mol = self . stereo_parent ( mol , skip_standardize = True ) mol = self . tautomer_parent ( mol , skip_standardize = True ) mol = self . standardize ( mol ) return mol

Return the super parent of a given molecule .


def main():
    """Main function for molvs command line interface."""

    # Root options
    parser = MolvsParser(epilog='use "molvs <command> -h" to show help for a specific command')
    subparsers = parser.add_subparsers(title='Available commands')

    # Options common to all commands

    common_parser = MolvsParser(add_help=False)
    common_parser.add_argument('infile', nargs='?', help='input filename', type=argparse.FileType('r'), default=sys.stdin)
    common_parser.add_argument('-i', '--intype', help='input filetype', choices=FILETYPES)
    common_parser.add_argument('-:', '--smiles', help='input SMILES instead of file', metavar='<smiles>')
    common_parser.add_argument('-O', '--outfile', help='output filename', type=argparse.FileType('w'), default=sys.stdout, metavar='<outfile>')

    # Standardize options
    standardize_parser = subparsers.add_parser('standardize', help='standardize a molecule', parents=[common_parser])
    standardize_parser.add_argument('-o', '--outtype', help='output filetype', choices=FILETYPES)
    standardize_parser.set_defaults(func=standardize_main)

    # Validate options
    validate_parser = subparsers.add_parser('validate', help='validate a molecule', parents=[common_parser])
    validate_parser.set_defaults(func=validate_main)

    args = parser.parse_args()
    try:
        args.func(args)
    except Exception as e:
        sys.stderr.write('Error: %s\n\n'.encode() % e.message)
        parser.print_help()
        sys.exit(2)

Main function for molvs command line interface .


def remove(self, mol):
        """Return the molecule with specified fragments removed.

        :param mol: The molecule to remove fragments from.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The molecule with fragments removed.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running FragmentRemover')
        # Iterate FragmentPatterns and remove matching fragments
        for frag in self.fragments:
            # If nothing is left or leave_last and only one fragment, end here
            if mol.GetNumAtoms() == 0 or (self.leave_last and len(Chem.GetMolFrags(mol)) <= 1):
                break
            # Apply removal for this FragmentPattern
            removed = Chem.DeleteSubstructs(mol, frag.smarts, onlyFrags=True)
            if not mol.GetNumAtoms() == removed.GetNumAtoms():
                log.info('Removed fragment: %s', frag.name)
            if self.leave_last and removed.GetNumAtoms() == 0:
                # All the remaining fragments match this pattern - leave them all
                break
            mol = removed
        return mol

def
remove
(
self
,
mol
)
:
log
.
debug
(
'Running FragmentRemover'
)
for
frag
in
self
.
fragments
:
if
mol
.
GetNumAtoms
(
)
==
0
or
(
self
.
leave_last
and
len
(
Chem
.
GetMolFrags
(
mol
)
)
<=
1
)
:
break
removed
=
Chem
.
DeleteSubstructs
(
mol
,
frag
.
smarts
,
onlyFrags
=
True
)
if
not
mol
.
GetNumAtoms
(
)
==
removed
.
GetNumAtoms
(
)
:
log
.
info
(
'Removed fragment: %s'
,
frag
.
name
)
if
self
.
leave_last
and
removed
.
GetNumAtoms
(
)
==
0
:
break
mol
=
removed
return
mol

Return the molecule with specified fragments removed .


def choose(self, mol):
        """Return the largest covalent unit.

        The largest fragment is determined by number of atoms (including hydrogens). Ties are broken by taking the
        fragment with the higher molecular weight, and then by taking the first alphabetically by SMILES if needed.

        :param mol: The molecule to choose the largest fragment from.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The largest fragment.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running LargestFragmentChooser')
        # TODO: Alternatively allow a list of fragments to be passed as the mol parameter
        fragments = Chem.GetMolFrags(mol, asMols=True)
        largest = None
        for f in fragments:
            smiles = Chem.MolToSmiles(f, isomericSmiles=True)
            log.debug('Fragment: %s', smiles)
            organic = is_organic(f)
            if self.prefer_organic:
                # Skip this fragment if not organic and we already have an organic fragment as the largest so far
                if largest and largest['organic'] and not organic:
                    continue
                # Reset largest if it wasn't organic and this fragment is organic
                if largest and organic and not largest['organic']:
                    largest = None
            # Count atoms
            atoms = 0
            for a in f.GetAtoms():
                atoms += 1 + a.GetTotalNumHs()
            # Skip this fragment if fewer atoms than the largest
            if largest and atoms < largest['atoms']:
                continue
            # Skip this fragment if equal number of atoms but weight is lower
            weight = rdMolDescriptors.CalcExactMolWt(f)
            if largest and atoms == largest['atoms'] and weight < largest['weight']:
                continue
            # Skip this fragment if equal atoms and equal weight but smiles comes last alphabetically
            if largest and atoms == largest['atoms'] and weight == largest['weight'] and smiles > largest['smiles']:
                continue
            # Otherwise this is the largest so far
            log.debug('New largest fragment: %s (%s)', smiles, atoms)
            largest = {'smiles': smiles, 'fragment': f, 'atoms': atoms, 'weight': weight, 'organic': organic}
        return largest['fragment']

def
choose
(
self
,
mol
)
:
log
.
debug
(
'Running LargestFragmentChooser'
)
fragments
=
Chem
.
GetMolFrags
(
mol
,
asMols
=
True
)
largest
=
for
f
in
fragments
:
smiles
=
Chem
.
MolToSmiles
(
f
,
isomericSmiles
=
True
)
log
.
debug
(
'Fragment: %s'
,
smiles
)
organic
=
is_organic
(
f
)
if
self
.
prefer_organic
:
if
largest
and
largest
[
'organic'
]
and
not
organic
:
continue
if
largest
and
organic
and
not
largest
[
'organic'
]
:
largest
=
atoms
=
0
for
a
in
f
.
GetAtoms
(
)
:
atoms
+=
1
+
a
.
GetTotalNumHs
(
)
if
largest
and
atoms
<
largest
[
'atoms'
]
:
continue
weight
=
rdMolDescriptors
.
CalcExactMolWt
(
f
)
if
largest
and
atoms
==
largest
[
'atoms'
]
and
weight
<
largest
[
'weight'
]
:
continue
if
largest
and
atoms
==
largest
[
'atoms'
]
and
weight
==
largest
[
'weight'
]
and
smiles
>
largest
[
'smiles'
]
:
continue
log
.
debug
(
'New largest fragment: %s (%s)'
,
smiles
,
atoms
)
largest
=
{
'smiles'
:
smiles
,
'fragment'
:
f
,
'atoms'
:
atoms
,
'weight'
:
weight
,
'organic'
:
organic
}
return
largest
[
'fragment'
]


def integrate_ivp(u0=1.0, v0=0.0, mu=1.0, tend=10.0, dt0=1e-8, nt=0,
                  nsteps=600, t0=0.0, atol=1e-8, rtol=1e-8, plot=False,
                  savefig='None', method='bdf', dpi=100, verbose=False):
    """
    Example program integrating an IVP problem of van der Pol oscillator
    """
    f, j = get_f_and_j(mu)
    if nt > 1:
        tout = np.linspace(t0, tend, nt)
        yout, nfo = integrate_predefined(
            f, j, [u0, v0], tout, dt0, atol, rtol, nsteps=nsteps,
            check_indexing=False, method=method)
    else:
        tout, yout, nfo = integrate_adaptive(
            f, j, [u0, v0], t0, tend, dt0, atol, rtol, nsteps=nsteps,
            check_indexing=False, method=method)  # dfdt[:] also for len == 1
    if verbose:
        print(nfo)
    if plot:
        import matplotlib.pyplot as plt
        plt.plot(tout, yout[:, 1], 'g--')
        plt.plot(tout, yout[:, 0], 'k-', linewidth=2)
        if savefig == 'None':
            plt.show()
        else:
            plt.savefig(savefig, dpi=dpi)

def integrate_ivp ( u0 = 1.0 , v0 = 0.0 , mu = 1.0 , tend = 10.0 , dt0 = 1e-8 , nt = 0 , nsteps = 600 , t0 = 0.0 , atol = 1e-8 , rtol = 1e-8 , plot = False , savefig = 'None' , method = 'bdf' , dpi = 100 , verbose = False ) : f , j = get_f_and_j ( mu ) if nt > 1 : tout = np . linspace ( t0 , tend , nt ) yout , nfo = integrate_predefined ( f , j , [ u0 , v0 ] , tout , dt0 , atol , rtol , nsteps = nsteps , check_indexing = False , method = method ) else : tout , yout , nfo = integrate_adaptive ( f , j , [ u0 , v0 ] , t0 , tend , dt0 , atol , rtol , nsteps = nsteps , check_indexing = False , method = method ) if verbose : print ( nfo ) if plot : import matplotlib . pyplot as plt plt . plot ( tout , yout [ : , 1 ] , 'g--' ) plt . plot ( tout , yout [ : , 0 ] , 'k-' , linewidth = 2 ) if savefig == 'None' : plt . show ( ) else : plt . savefig ( savefig , dpi = dpi )


def get_stats(self, username='', password='', organization='llnl',
        force=True, repo_type='public'):
        """
        Retrieves the statistics from the given organization with the given
        credentials. Will not retreive data if file exists and force hasn't been
        set to True. This is to save GH API requests.
        """
        date = str(datetime.date.today())
        file_path =  ('../github_stats_output/' + date[:4] + '/' + date[:7] + '/'
            + date + '.csv')
        if force or not os.path.isfile(file_path):
            my_github.login(username, password)
            calls_beginning = self.logged_in_gh.ratelimit_remaining + 1
            print 'Rate Limit: ' + str(calls_beginning)
            my_github.get_org(organization)
            count_members = my_github.get_mems_of_org()
            count_teams = my_github.get_teams_of_org()
            my_github.repos(repo_type=repo_type, organization=organization)
            #Write JSON
            my_github.write_org_json(dict_to_write=self.members_json,
                path_ending_type='members', is_list=True)
            my_github.write_org_json(dict_to_write=
                {'singleton': self.org_retrieved.to_json()},
                path_ending_type='organization')
            my_github.write_org_json(dict_to_write=self.teams_json,
                path_ending_type='teams', is_list=True)

            my_github.write_repo_json(dict_to_write=self.repos_json,
                path_ending_type='repo')
            my_github.write_repo_json(dict_to_write=self.contributors_json,
                path_ending_type='contributors', is_list=True)
            my_github.write_repo_json(dict_to_write=self.pull_requests_json,
                path_ending_type='pull-requests', is_list=True)
            my_github.write_repo_json(dict_to_write=self.issues_json,
                path_ending_type='issues', is_list=True)
            my_github.write_repo_json(dict_to_write=self.languages_json,
                path_ending_type='languages', is_dict=True)
            my_github.write_repo_json(dict_to_write=self.commits_json,
                path_ending_type='commits', is_list=True)
            #Write CSV
            my_github.write_to_file(file_path,
                                    date,
                                    organization,
                                    count_members,
                                    count_teams)
            calls_remaining = self.logged_in_gh.ratelimit_remaining
            calls_used = calls_beginning - calls_remaining
            print ('Rate Limit Remaining: ' + str(calls_remaining) + '\nUsed '
                + str(calls_used) + ' API calls.')

def
get_stats
(
self
,
username
=
''
,
password
=
''
,
organization
=
'llnl'
,
force
=
True
,
repo_type
=
'public'
)
:
date
=
str
(
datetime
.
date
.
today
(
)
)
file_path
=
(
'../github_stats_output/'
+
date
[
:
4
]
+
'/'
+
date
[
:
7
]
+
'/'
+
date
+
'.csv'
)
if
force
or
not
os
.
path
.
isfile
(
file_path
)
:
my_github
.
login
(
username
,
password
)
calls_beginning
=
self
.
logged_in_gh
.
ratelimit_remaining
+
1
print
'Rate Limit: '
+
str
(
calls_beginning
)
my_github
.
get_org
(
organization
)
count_members
=
my_github
.
get_mems_of_org
(
)
count_teams
=
my_github
.
get_teams_of_org
(
)
my_github
.
repos
(
repo_type
=
repo_type
,
organization
=
organization
)
my_github
.
write_org_json
(
dict_to_write
=
self
.
members_json
,
path_ending_type
=
'members'
,
is_list
=
True
)
my_github
.
write_org_json
(
dict_to_write
=
{
'singleton'
:
self
.
org_retrieved
.
to_json
(
)
}
,
path_ending_type
=
'organization'
)
my_github
.
write_org_json
(
dict_to_write
=
self
.
teams_json
,
path_ending_type
=
'teams'
,
is_list
=
True
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
repos_json
,
path_ending_type
=
'repo'
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
contributors_json
,
path_ending_type
=
'contributors'
,
is_list
=
True
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
pull_requests_json
,
path_ending_type
=
'pull-requests'
,
is_list
=
True
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
issues_json
,
path_ending_type
=
'issues'
,
is_list
=
True
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
languages_json
,
path_ending_type
=
'languages'
,
is_dict
=
True
)
my_github
.
write_repo_json
(
dict_to_write
=
self
.
commits_json
,
path_ending_type
=
'commits'
,
is_list
=
True
)
my_github
.
write_to_file
(
file_path
,
date
,
organization
,
count_members
,
count_teams
)
calls_remaining
=
self
.
logged_in_gh
.
ratelimit_remaining
calls_used
=
calls_beginning
calls_remaining
print
(
'Rate Limit Remaining: '
+
str
(
calls_remaining
)
+
'\nUsed '
+
str
(
calls_used
)
+
' API calls.'
)

Retrieves the statistics from the given organization with the given credentials . Will not retreive data if file exists and force hasn t been set to True . This is to save GH API requests .


def get_mems_of_org(self):
        """
        Retrieves the number of members of the organization.
        """
        print 'Getting members.'
        counter = 0
        for member in self.org_retrieved.iter_members():
            self.members_json[member.id] = member.to_json()
            counter += 1
        return counter

def
get_mems_of_org
(
self
)
:
print
'Getting members.'
counter
=
0
for
member
in
self
.
org_retrieved
.
iter_members
(
)
:
self
.
members_json
[
member
.
id
]
=
member
.
to_json
(
)
counter
+=
1
return
counter

Retrieves the number of members of the organization .


def get_teams_of_org(self):
        """
        Retrieves the number of teams of the organization.
        """
        print 'Getting teams.'
        counter = 0
        for team in self.org_retrieved.iter_teams():
            self.teams_json[team.id] = team.to_json()
            counter += 1
        return counter

def
get_teams_of_org
(
self
)
:
print
'Getting teams.'
counter
=
0
for
team
in
self
.
org_retrieved
.
iter_teams
(
)
:
self
.
teams_json
[
team
.
id
]
=
team
.
to_json
(
)
counter
+=
1
return
counter

Retrieves the number of teams of the organization .


def repos(self, repo_type='public', organization='llnl'):
        """
        Retrieves info about the repos of the current organization.
        """
        print 'Getting repos.'
        for repo in self.org_retrieved.iter_repos(type=repo_type):
            #JSON
            json = repo.to_json()
            self.repos_json[repo.name] = json
            #CSV
            temp_repo = my_repo.My_Repo()
            temp_repo.name = repo.full_name
            self.total_repos += 1
            temp_repo.contributors = my_github.get_total_contributors(repo)
            self.total_contributors += temp_repo.contributors
            temp_repo.forks = repo.forks_count
            self.total_forks += temp_repo.forks
            temp_repo.stargazers = repo.stargazers
            self.total_stars += temp_repo.stargazers
            temp_repo.pull_requests_open, temp_repo.pull_requests_closed = \
                my_github.get_pull_reqs(repo)
            temp_repo.pull_requests = (temp_repo.pull_requests_open
                + temp_repo.pull_requests_closed)
            self.total_pull_reqs += temp_repo.pull_requests_open
            self.total_pull_reqs += temp_repo.pull_requests_closed
            self.total_pull_reqs_open += temp_repo.pull_requests_open
            self.total_pull_reqs_closed += temp_repo.pull_requests_closed
            temp_repo.open_issues = repo.open_issues_count
            self.total_open_issues += temp_repo.open_issues
            temp_repo.closed_issues = my_github.get_issues(repo, organization=organization)
            temp_repo.issues = temp_repo.closed_issues + temp_repo.open_issues
            self.total_closed_issues += temp_repo.closed_issues
            self.total_issues += temp_repo.issues
            my_github.get_languages(repo, temp_repo)
            temp_repo.readme = my_github.get_readme(repo)
            #temp_repo.license = my_github.get_license(repo)
            temp_repo.commits = self.get_commits(repo=repo, organization=organization)
            self.total_commits += temp_repo.commits
            self.all_repos.append(temp_repo)

def
repos
(
self
,
repo_type
=
'public'
,
organization
=
'llnl'
)
:
print
'Getting repos.'
for
repo
in
self
.
org_retrieved
.
iter_repos
(
type
=
repo_type
)
:
json
=
repo
.
to_json
(
)
self
.
repos_json
[
repo
.
name
]
=
json
temp_repo
=
my_repo
.
My_Repo
(
)
temp_repo
.
name
=
repo
.
full_name
self
.
total_repos
+=
1
temp_repo
.
contributors
=
my_github
.
get_total_contributors
(
repo
)
self
.
total_contributors
+=
temp_repo
.
contributors
temp_repo
.
forks
=
repo
.
forks_count
self
.
total_forks
+=
temp_repo
.
forks
temp_repo
.
stargazers
=
repo
.
stargazers
self
.
total_stars
+=
temp_repo
.
stargazers
temp_repo
.
pull_requests_open
,
temp_repo
.
pull_requests_closed
=
my_github
.
get_pull_reqs
(
repo
)
temp_repo
.
pull_requests
=
(
temp_repo
.
pull_requests_open
+
temp_repo
.
pull_requests_closed
)
self
.
total_pull_reqs
+=
temp_repo
.
pull_requests_open
self
.
total_pull_reqs
+=
temp_repo
.
pull_requests_closed
self
.
total_pull_reqs_open
+=
temp_repo
.
pull_requests_open
self
.
total_pull_reqs_closed
+=
temp_repo
.
pull_requests_closed
temp_repo
.
open_issues
=
repo
.
open_issues_count
self
.
total_open_issues
+=
temp_repo
.
open_issues
temp_repo
.
closed_issues
=
my_github
.
get_issues
(
repo
,
organization
=
organization
)
temp_repo
.
issues
=
temp_repo
.
closed_issues
+
temp_repo
.
open_issues
self
.
total_closed_issues
+=
temp_repo
.
closed_issues
self
.
total_issues
+=
temp_repo
.
issues
my_github
.
get_languages
(
repo
,
temp_repo
)
temp_repo
.
readme
=
my_github
.
get_readme
(
repo
)
temp_repo
.
commits
=
self
.
get_commits
(
repo
=
repo
,
organization
=
organization
)
self
.
total_commits
+=
temp_repo
.
commits
self
.
all_repos
.
append
(
temp_repo
)

Retrieves info about the repos of the current organization .


def get_total_contributors(self, repo):
        """
        Retrieves the number of contributors to a repo in the organization.
        Also adds to unique contributor list.
        """
        repo_contributors = 0
        for contributor in repo.iter_contributors():
            repo_contributors += 1
            self.unique_contributors[contributor.id].append(repo.name)
            self.contributors_json[repo.name].append(contributor.to_json())
        return repo_contributors

def get_total_contributors ( self , repo ) : repo_contributors = 0 for contributor in repo . iter_contributors ( ) : repo_contributors += 1 self . unique_contributors [ contributor . id ] . append ( repo . name ) self . contributors_json [ repo . name ] . append ( contributor . to_json ( ) ) return repo_contributors

Retrieves the number of contributors to a repo in the organization . Also adds to unique contributor list .


def get_pull_reqs(self, repo):
        """
        Retrieves the number of pull requests on a repo in the organization.
        """
        pull_reqs_open = 0
        pull_reqs_closed = 0
        for pull_request in repo.iter_pulls(state='all'):
            self.pull_requests_json[repo.name].append(pull_request.to_json())
            if pull_request.closed_at is not None:
                pull_reqs_closed += 1
            else:
                pull_reqs_open += 1
        return pull_reqs_open, pull_reqs_closed

def get_pull_reqs ( self , repo ) : pull_reqs_open = 0 pull_reqs_closed = 0 for pull_request in repo . iter_pulls ( state = 'all' ) : self . pull_requests_json [ repo . name ] . append ( pull_request . to_json ( ) ) if pull_request . closed_at is not None : pull_reqs_closed += 1 else : pull_reqs_open += 1 return pull_reqs_open , pull_reqs_closed

Retrieves the number of pull requests on a repo in the organization .


def get_issues(self, repo, organization='llnl'):
        """
        Retrieves the number of closed issues.
        """
        #JSON
        path = ('../github-data/' + organization + '/' + repo.name + '/issues')
        is_only_today = False
        if not os.path.exists(path): #no previous path, get all issues
            all_issues = repo.iter_issues(state='all')
            is_only_today = True
        else:
            files = os.listdir(path)
            date = str(files[-1][:-5])
            if date == str(datetime.date.today()):
                #most recent date is actually today, get previous most recent date
                if len(files) > 2:
                    date = str(files[-2][:-5])
                else:
                    #This means there is only one file, today. Retrieve every issue
                    all_issues = repo.iter_issues(state='all')
                    is_only_today = True
            if not is_only_today:#there's a previous saved JSON that's not today
                all_issues = repo.iter_issues(since=date, state='all')
        for issue in all_issues:
            self.issues_json[repo.name].append(issue.to_json())
        #CSV
        closed_issues = 0
        for issue in repo.iter_issues(state='closed'):
            if issue is not None:
                closed_issues += 1
        return closed_issues

def get_issues ( self , repo , organization = 'llnl' ) : path = ( '../github-data/' + organization + '/' + repo . name + '/issues' ) is_only_today = False if not os . path . exists ( path ) : all_issues = repo . iter_issues ( state = 'all' ) is_only_today = True else : files = os . listdir ( path ) date = str ( files [ - 1 ] [ : - 5 ] ) if date == str ( datetime . date . today ( ) ) : if len ( files ) > 2 : date = str ( files [ - 2 ] [ : - 5 ] ) else : all_issues = repo . iter_issues ( state = 'all' ) is_only_today = True if not is_only_today : all_issues = repo . iter_issues ( since = date , state = 'all' ) for issue in all_issues : self . issues_json [ repo . name ] . append ( issue . to_json ( ) ) closed_issues = 0 for issue in repo . iter_issues ( state = 'closed' ) : if issue is not None : closed_issues += 1 return closed_issues


def get_readme(self, repo):
        """
        Checks to see if the given repo has a ReadMe. MD means it has a correct
        Readme recognized by GitHub.
        """
        readme_contents = repo.readme()
        if readme_contents is not None:
            self.total_readmes += 1
            return 'MD'
        if self.search_limit >= 28:
            print 'Hit search limit. Sleeping for 60 sec.'
            time.sleep(60)
            self.search_limit = 0
        self.search_limit += 1
        search_results = self.logged_in_gh.search_code('readme'
            + 'in:path repo:' + repo.full_name)
        try:
            for result in search_results:
                path = result.path[1:]
                if '/' not in path and 'readme' in path.lower():
                    self.total_readmes += 1
                    return path
            return 'MISS'
        except (github3.models.GitHubError, StopIteration) as e:
            return 'MISS'

Checks to see if the given repo has a ReadMe . MD means it has a correct Readme recognized by GitHub .


def get_license(self, repo):
        """
        Checks to see if the given repo has a top level LICENSE file.
        """
        if self.search_limit >= 28:
            print 'Hit search limit. Sleeping for 60 sec.'
            time.sleep(60)
            self.search_limit = 0
        self.search_limit += 1
        search_results = self.logged_in_gh.search_code('license'
            + 'in:path repo:' + repo.full_name)
        try:
            for result in search_results:
                path = result.path[1:]
                if '/' not in path and 'license' in path.lower():
                    self.total_licenses += 1
                    return path
            return 'MISS'
        except (StopIteration) as e:
            return 'MISS'

Checks to see if the given repo has a top level LICENSE file .


def get_commits(self, repo, organization='llnl'):
        """
        Retrieves the number of commits to a repo in the organization. If it is
        the first time getting commits for a repo, it will get all commits and
        save them to JSON. If there are previous commits saved, it will only get
        commits that have not been saved to disk since the last date of commits.
        """
        #JSON
        path = ('../github-data/' + organization + '/' + repo.name + '/commits')
        is_only_today = False
        if not os.path.exists(path): #no previous path, get all commits
            all_commits = repo.iter_commits()
            is_only_today = True
        else:
            files = os.listdir(path)
            date = str(files[-1][:-5])
            if date == str(datetime.date.today()):
                #most recent date is actually today, get previous most recent date
                if len(files) > 2:
                    date = str(files[-2][:-5])
                else:
                    #This means there is only one file, today. Retrieve every commit
                    all_commits = repo.iter_commits()
                    is_only_today = True
            if not is_only_today:#there's a previous saved JSON that's not today
                all_commits = repo.iter_commits(since=date)
        for commit in all_commits:
            self.commits_json[repo.name].append(commit.to_json())
        #for csv
        count = 0
        for commit in repo.iter_commits():
            count += 1
        return count

def get_commits ( self , repo , organization = 'llnl' ) : path = ( '../github-data/' + organization + '/' + repo . name + '/commits' ) is_only_today = False if not os . path . exists ( path ) : all_commits = repo . iter_commits ( ) is_only_today = True else : files = os . listdir ( path ) date = str ( files [ - 1 ] [ : - 5 ] ) if date == str ( datetime . date . today ( ) ) : if len ( files ) > 2 : date = str ( files [ - 2 ] [ : - 5 ] ) else : all_commits = repo . iter_commits ( ) is_only_today = True if not is_only_today : all_commits = repo . iter_commits ( since = date ) for commit in all_commits : self . commits_json [ repo . name ] . append ( commit . to_json ( ) ) count = 0 for commit in repo . iter_commits ( ) : count += 1 return count

Retrieves the number of commits to a repo in the organization . If it is the first time getting commits for a repo it will get all commits and save them to JSON . If there are previous commits saved it will only get commits that have not been saved to disk since the last date of commits .


def write_org_json(self, date=(datetime.date.today()),
        organization='llnl',dict_to_write={}, path_ending_type='',
        is_list=False):
        """
        Writes stats from the organization to JSON.
        """
        path = ('../github-data/' + organization + '-org/'
            + path_ending_type + '/' + str(date) + '.json')
        self.checkDir(path)
        with open(path, 'w') as out_clear:#clear old data
            out_clear.close()
        with open(path, 'a') as out:
            if is_list:#used for list of items
                out.write('[')
            for item in dict_to_write:
                out.write(json.dumps(dict_to_write[item], sort_keys=True,
                    indent=4, separators=(',', ': ')) + ',')
            out.seek(-1, os.SEEK_END)#kill last comma
            out.truncate()
            if is_list:
                out.write(']')
        out.close()

def
write_org_json
(
self
,
date
=
(
datetime
.
date
.
today
(
)
)
,
organization
=
'llnl'
,
dict_to_write
=
{
}
,
path_ending_type
=
''
,
is_list
=
False
)
:
path
=
(
'../github-data/'
+
organization
+
'-org/'
+
path_ending_type
+
'/'
+
str
(
date
)
+
'.json'
)
self
.
checkDir
(
path
)
with
open
(
path
,
'w'
)
as
out_clear
:
out_clear
.
close
(
)
with
open
(
path
,
'a'
)
as
out
:
if
is_list
:
out
.
write
(
'['
)
for
item
in
dict_to_write
:
out
.
write
(
json
.
dumps
(
dict_to_write
[
item
]
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
': '
)
)
+
','
)
out
.
seek
(
1
,
os
.
SEEK_END
)
out
.
truncate
(
)
if
is_list
:
out
.
write
(
']'
)
out
.
close
(
)

Writes stats from the organization to JSON .


def write_totals(self, file_path='', date=str(datetime.date.today()),
        organization='N/A', members=0, teams=0):
        """
        Updates the total.csv file with current data.
        """

        total_exists = os.path.isfile(file_path)
        with open(file_path, 'a') as out_total:
            if not total_exists:
                out_total.write('date,organization,repos,members,teams,'
                    + 'unique_contributors,total_contributors,forks,'
                    + 'stargazers,pull_requests,open_issues,has_readme,'
                    + 'has_license,pull_requests_open,pull_requests_closed,'
                    + 'commits,id,closed_issues,issues\n')
            self.delete_last_line(date=date, file_path=file_path)
        out_total.close()
        with open(file_path, 'r') as file_read:
            row_count = sum(1 for row in file_read) - 1
        file_read.close()
        with open(file_path, 'a') as out_total:
            out_total.write(date + ',' + organization + ','
                + str(self.total_repos) + ',' + str(members) + ',' + str(teams)
                + ',' + str(len(self.unique_contributors)) + ','
                + str(self.total_contributors) + ',' + str(self.total_forks)
                + ',' + str(self.total_stars) + ',' + str(self.total_pull_reqs)
                + ',' + str(self.total_open_issues) + ','
                + str(self.total_readmes) + ',' + str(self.total_licenses) + ','
                + str(self.total_pull_reqs_open) + ','
                + str(self.total_pull_reqs_closed) + ','
                + str(self.total_commits) + ',' + str(row_count) + ','
                + str(self.total_closed_issues) + ',' + str(self.total_issues)
                + '\n')
        out_total.close()

Updates the total . csv file with current data .


def write_languages(self, file_path='',date=str(datetime.date.today())):
        """
        Updates languages.csv file with current data.
        """
        self.remove_date(file_path=file_path, date=date)
        languages_exists = os.path.isfile(file_path)
        with open(file_path, 'a') as out_languages:
            if not languages_exists:
                out_languages.write('date,language,count,size,size_log\n')
            languages_sorted = sorted(self.languages_size)
            #self.delete_last_line(date=date, file_path=file_path)
            for language in languages_sorted:
                try:
                    out_languages.write(date + ',' + language + ','
                        + str(self.languages[language]) + ','
                        + str(self.languages_size[language]) + ','
                        + str(math.log10(int(self.languages_size[language])))
                        + '\n')
                except (TypeError, KeyError) as e:
                    out_languages.write(date + ',' + language + ','
                        + str(0) + ','
                        + str(self.languages_size[language]) + ','
                        + str(math.log10(int(self.languages_size[language])))
                        + '\n')

def write_languages ( self , file_path = '' , date = str ( datetime . date . today ( ) ) ) : self . remove_date ( file_path = file_path , date = date ) languages_exists = os . path . isfile ( file_path ) with open ( file_path , 'a' ) as out_languages : if not languages_exists : out_languages . write ( 'date,language,count,size,size_log\n' ) languages_sorted = sorted ( self . languages_size ) for language in languages_sorted : try : out_languages . write ( date + ',' + language + ',' + str ( self . languages [ language ] ) + ',' + str ( self . languages_size [ language ] ) + ',' + str ( math . log10 ( int ( self . languages_size [ language ] ) ) ) + '\n' ) except ( TypeError , KeyError ) as e : out_languages . write ( date + ',' + language + ',' + str ( 0 ) + ',' + str ( self . languages_size [ language ] ) + ',' + str ( math . log10 ( int ( self . languages_size [ language ] ) ) ) + '\n' )

Updates languages . csv file with current data .


def checkDir(self, file_path=''):
        """
        Checks if a directory exists. If not, it creates one with the specified
        file_path.
        """
        if not os.path.exists(os.path.dirname(file_path)):
            try:
                os.makedirs(os.path.dirname(file_path))
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise

def checkDir ( self , file_path = '' ) : if not os . path . exists ( os . path . dirname ( file_path ) ) : try : os . makedirs ( os . path . dirname ( file_path ) ) except OSError as e : if e . errno != errno . EEXIST : raise

Checks if a directory exists . If not it creates one with the specified file_path .


def remove_date(self, file_path='', date=str(datetime.date.today())):
        """
        Removes all rows of the associated date from the given csv file.
        Defaults to today.
        """
        languages_exists = os.path.isfile(file_path)
        if languages_exists:
            with open(file_path, 'rb') as inp, open('temp.csv', 'wb') as out:
                writer = csv.writer(out)
                for row in csv.reader(inp):
                    if row[0] != date:
                        writer.writerow(row)
            inp.close()
            out.close()
            os.remove(file_path)
            os.rename("temp.csv",file_path)

def remove_date ( self , file_path = '' , date = str ( datetime . date . today ( ) ) ) : languages_exists = os . path . isfile ( file_path ) if languages_exists : with open ( file_path , 'rb' ) as inp , open ( 'temp.csv' , 'wb' ) as out : writer = csv . writer ( out ) for row in csv . reader ( inp ) : if row [ 0 ] != date : writer . writerow ( row ) inp . close ( ) out . close ( ) os . remove ( file_path ) os . rename ( "temp.csv" , file_path )

Removes all rows of the associated date from the given csv file . Defaults to today .


def gov_orgs():
    """
    Returns a list of the names of US Government GitHub organizations

    Based on: https://government.github.com/community/

    Exmample return:
        {'llnl', '18f', 'gsa', 'dhs-ncats', 'spack', ...}
    """
    us_gov_github_orgs = set()

    gov_orgs = requests.get('https://government.github.com/organizations.json').json()

    us_gov_github_orgs.update(gov_orgs['governments']['U.S. Federal'])
    us_gov_github_orgs.update(gov_orgs['governments']['U.S. Military and Intelligence'])
    us_gov_github_orgs.update(gov_orgs['research']['U.S. Research Labs'])

    return list(us_gov_github_orgs)


def create_enterprise_session(url, token=None):
    """
    Create a github3.py session for a GitHub Enterprise instance

    If token is not provided, will attempt to use the GITHUB_API_TOKEN
    environment variable if present.
    """

    gh_session = github3.enterprise_login(url=url, token=token)

    if gh_session is None:
        msg = 'Unable to connect to GitHub Enterprise (%s) with provided token.'
        raise RuntimeError(msg, url)

    return gh_session

Create a github3 . py session for a GitHub Enterprise instance


def _check_api_limits(gh_session, api_required=250, sleep_time=15):
    """
    Simplified check for API limits

    If necessary, spin in place waiting for API to reset before returning.

    See: https://developer.github.com/v3/#rate-limiting
    """
    api_rates = gh_session.rate_limit()

    api_remaining = api_rates['rate']['remaining']
    api_reset = api_rates['rate']['reset']
    logger.debug('Rate Limit - %d requests remaining', api_remaining)

    if api_remaining > api_required:
        return

    now_time = time.time()
    time_to_reset = int(api_reset - now_time)
    logger.warn('Rate Limit Depleted - Sleeping for %d seconds', time_to_reset)

    while now_time < api_reset:
        time.sleep(10)
        now_time = time.time()

    return


def connect(url='https://github.com', token=None):
    """
    Create a GitHub session for making requests
    """

    gh_session = None
    if url == 'https://github.com':
        gh_session = create_session(token)
    else:
        gh_session = create_enterprise_session(url, token)

    if gh_session is None:
        msg = 'Unable to connect to (%s) with provided token.'
        raise RuntimeError(msg, url)

    logger.info('Connected to: %s', url)

    return gh_session


def query_repos(gh_session, orgs=None, repos=None, public_only=True):
    """
    Yields GitHub3.py repo objects for provided orgs and repo names

    If orgs and repos are BOTH empty, execute special mode of getting ALL
    repositories from the GitHub Server.

    If public_only is True, will return only those repos that are marked as
    public. Set this to false to return all organizations that the session has
    permissions to access.
    """

    if orgs is None:
        orgs = []
    if repos is None:
        repos = []
    if public_only:
        privacy = 'public'
    else:
        privacy = 'all'

    _check_api_limits(gh_session, 10)

    for org_name in orgs:
        org = gh_session.organization(org_name)
        num_repos = org.public_repos_count

        _check_api_limits(gh_session, _num_requests_needed(num_repos))

        for repo in org.repositories(type=privacy):
            _check_api_limits(gh_session, 10)
            yield repo

    for repo_name in repos:
        _check_api_limits(gh_session, 10)
        org, name = repo_name.split('/')
        yield gh_session.repository(org, name)

    if not (orgs or repos):
        for repo in gh_session.all_repositories():
            yield repo

def query_repos ( gh_session , orgs = None , repos = None , public_only = True ) : if orgs is None : orgs = [ ] if repos is None : repos = [ ] if public_only : privacy = 'public' else : privacy = 'all' _check_api_limits ( gh_session , 10 ) for org_name in orgs : org = gh_session . organization ( org_name ) num_repos = org . public_repos_count _check_api_limits ( gh_session , _num_requests_needed ( num_repos ) ) for repo in org . repositories ( type = privacy ) : _check_api_limits ( gh_session , 10 ) yield repo for repo_name in repos : _check_api_limits ( gh_session , 10 ) org , name = repo_name . split ( '/' ) yield gh_session . repository ( org , name ) if not ( orgs or repos ) : for repo in gh_session . all_repositories ( ) : yield repo

Yields GitHub3 . py repo objects for provided orgs and repo names


def get_org(self, organization_name=''):
        """
        Retrieves an organization via given org name. If given
        empty string, prompts user for an org name.
        """
        self.organization_name = organization_name
        if(organization_name == ''):
            self.organization_name = raw_input('Organization: ')
        print 'Getting organization.'
        self.org_retrieved = self.logged_in_gh.organization(organization_name)

def
get_org
(
self
,
organization_name
=
''
)
:
self
.
organization_name
=
organization_name
if
(
organization_name
==
''
)
:
self
.
organization_name
=
raw_input
(
'Organization: '
)
print
'Getting organization.'
self
.
org_retrieved
=
self
.
logged_in_gh
.
organization
(
organization_name
)

Retrieves an organization via given org name . If given empty string prompts user for an org name .


def write_to_file(self, file_path='', date=(datetime.date.today()),
        organization='llnl'):
        """
        Writes stargazers data to file.
        """
        with open(file_path, 'w+') as out:
            out.write('date,organization,stargazers\n')
            sorted_stargazers = sorted(self.stargazers)#sort based on lowercase
            for star in sorted_stargazers:
                out.write(star + ',' + str(self.stargazers[star]) + '\n')
        out.close()

def write_to_file ( self , file_path = '' , date = ( datetime . date . today ( ) ) , organization = 'llnl' ) : with open ( file_path , 'w+' ) as out : out . write ( 'date,organization,stargazers\n' ) sorted_stargazers = sorted ( self . stargazers ) for star in sorted_stargazers : out . write ( star + ',' + str ( self . stargazers [ star ] ) + '\n' ) out . close ( )


def from_gitlab(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from GitLab Repository
        """
        if not isinstance(repository, gitlab.v4.objects.Project):
            raise TypeError('Repository must be a gitlab Repository object')

        project = klass()

        logger.debug(
            'GitLab: repository_id=%d path_with_namespace=%s',
            repository.id,
            repository.path_with_namespace,
        )

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.http_url_to_repo
        project['description'] = repository.description

        # TODO: Update licenses from GitLab API
        project['permissions']['licenses'] = None

        web_url = repository.web_url
        public_server = web_url.startswith('https://gitlab.com')

        if repository.visibility in ('public') and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['gitlab'] + repository.tag_list

        project['contact'] = {
            'email': '',
            'URL': web_url,
        }

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = repository.namespace['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.web_url

        api_url = repository.manager.gitlab._url
        archive_suffix = '/projects/%s/repository/archive' % repository.get_id()
        project['downloadURL'] = api_url + archive_suffix

        # project['languages'] = [l for l, _ in repository.languages()]
        # project['partners'] = []
        # project['relatedCode'] = []
        # project['reusedCode'] = []

        project['date'] = {
            'created': date_parse(repository.created_at).date().isoformat(),
            'lastModified': date_parse(repository.last_activity_at).date().isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project


def from_doecode(klass, record):
        """
        Create CodeGovProject object from DOE CODE record

        Handles crafting Code.gov Project
        """
        if not isinstance(record, dict):
            raise TypeError('`record` must be a dict')

        project = klass()

        # -- REQUIRED FIELDS --

        project['name'] = record['software_title']
        logger.debug('DOE CODE: software_title="%s"', record['software_title'])

        link = record.get('repository_link', '')
        if not link:
            link = record.get('landing_page')
            logger.warning('DOE CODE: No repositoryURL, using landing_page: %s', link)

        project['repositoryURL'] = link

        project['description'] = record['description']

        licenses = set(record['licenses'])
        licenses.discard(None)
        logger.debug('DOE CODE: licenses=%s', licenses)

        license_objects = []
        if 'Other' in licenses:
            licenses.remove('Other')
            license_objects = [{
                'name': 'Other',
                'URL': record['proprietary_url']
            }]

        if licenses:
            license_objects.extend([_license_obj(license) for license in licenses])

        project['permissions']['licenses'] = license_objects

        if record['open_source']:
            usage_type = 'openSource'
        else:
            usage_type = 'exemptByLaw'
            project['permissions']['exemptionText'] = 'This source code is restricted by patent and / or intellectual property law.'

        project['permissions']['usageType'] = usage_type

        # TODO: Compute from git repo
        project['laborHours'] = 0

        project['tags'] = ['DOE CODE']
        lab_name = record.get('lab_display_name')
        if lab_name is not None:
            project['tags'].append(lab_name)

        project['contact']['email'] = record['owner']
        # project['contact']['URL'] = ''
        # project['contact']['name'] = ''
        # project['contact']['phone'] = ''

        # -- OPTIONAL FIELDS --

        if 'version_number' in record and record['version_number']:
            project['version'] = record['version_number']

        if lab_name is not None:
            project['organization'] = lab_name

        # Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        status = record.get('ever_announced')
        if status is None:
            raise ValueError('DOE CODE: Unable to determine "ever_announced" value!')
        elif status:
            status = 'Production'
        else:
            status = 'Development'

        project['status'] = status

        vcs = None
        link = project['repositoryURL']
        if 'github.com' in link:
            vcs = 'git'
        if vcs is None:
            logger.debug('DOE CODE: Unable to determine vcs for: name="%s", repositoryURL=%s', project['name'], link)
            vcs = ''
        if vcs:
            project['vcs'] = vcs

        url = record.get('landing_page', '')
        if url:
            project['homepageURL'] = url

        # record['downloadURL'] = ''

        # self['disclaimerText'] = ''

        # self['disclaimerURL'] = ''

        if 'programming_languages' in record:
            project['languages'] = record['programming_languages']

        # self['partners'] = []
        # TODO: Look into using record['contributing_organizations']

        # self['relatedCode'] = []

        # self['reusedCode'] = []

        # date: [object] A date object describing the release.
        #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
        #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
        if 'date_record_added' in record and 'date_record_updated' in record:
            project['date'] = {
                'created': record['date_record_added'],
                # 'lastModified': '',
                'metadataLastUpdated': record['date_record_updated']
            }

        return project


def _license_obj(license):
    """
    A helper function to look up license object information

    Use names from: https://api.github.com/licenses
    """
    obj = None

    if license in ('MIT', 'MIT License'):
        obj = {
            'URL': 'https://api.github.com/licenses/mit',
            'name': 'MIT'
        }
    elif license in ('BSD 2-clause "Simplified" License'):
        obj = {
            'URL': 'https://api.github.com/licenses/bsd-2-clause',
            'name': 'BSD-2-Clause'
        }
    elif license in ('BSD 3-clause "New" or "Revised" License'):
        obj = {
            'URL': 'https://api.github.com/licenses/bsd-3-clause',
            'name': 'BSD-3-Clause'
        }
    elif license in ('Apache License 2.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/apache-2.0',
            'name': 'Apache-2.0'
        }
    elif license in ('GNU General Public License v2.1'):
        obj = {
            'URL': 'https://api.github.com/licenses/gpl-2.1',
            'name': 'GPL-2.1'
        }
    elif license in ('GNU General Public License v2.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/gpl-2.0',
            'name': 'GPL-2.0'
        }
    elif license in ('GNU Lesser General Public License v2.1'):
        obj = {
            'URL': 'https://api.github.com/licenses/lgpl-2.1',
            'name': 'LGPL-2.1'
        }
    elif license in ('GNU General Public License v3.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/gpl-3.0',
            'name': 'GPL-3.0'
        }
    elif license in ('GNU Lesser General Public License v3.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/lgpl-3.0',
            'name': 'LGPL-3.0'
        }
    elif license in ('Eclipse Public License 1.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/epl-1.0',
            'name': 'EPL-1.0',
        }
    elif license in ('Mozilla Public License 2.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/mpl-2.0',
            'name': 'MPL-2.0',
        }
    elif license in ('The Unlicense'):
        obj = {
            'URL': 'https://api.github.com/licenses/unlicense',
            'name': 'Unlicense',
        }
    elif license in ('GNU Affero General Public License v3.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/agpl-3.0',
            'name': 'AGPL-3.0',
        }
    elif license in ('Eclipse Public License 2.0'):
        obj = {
            'URL': 'https://api.github.com/licenses/epl-2.0',
            'name': 'EPL-2.0',
        }

    if obj is None:
        logger.warn('I dont understand the license: %s', license)
        raise ValueError('Aborting!')

    return obj


def get_traffic(self):
        """
        Retrieves the traffic for the repositories of the given organization.
        """
        print 'Getting traffic.'
        #Uses the developer API. Note this could change.
        headers = {'Accept': 'application/vnd.github.spiderman-preview', 'Authorization': 'token ' + self.token}
        headers_release = {'Authorization': 'token ' + self.token}
        for repo in self.org_retrieved.iter_repos(type='public'):
            url = ('https://api.github.com/repos/' + self.organization_name
                + '/' + repo.name)
            self.get_referrers(url=url, headers=headers, repo_name=repo.name)
            self.get_paths(url=url, headers=headers)
            self.get_data(url=url, headers=headers, dict_to_store=self.views,
                type='views', repo_name=repo.name)
            self.get_data(url=url, headers=headers, dict_to_store=self.clones,
                type='clones', repo_name=repo.name)
            self.get_releases(url=url, headers=headers_release, repo_name=repo.name)

Retrieves the traffic for the repositories of the given organization .


def get_releases(self, url='', headers={}, repo_name=''):
        """
        Retrieves the releases for the given repo in JSON.
        """
        url_releases = (url + '/releases')
        r = requests.get(url_releases, headers=headers)
        self.releases_json[repo_name] = r.json()

def get_releases ( self , url = '' , headers = { } , repo_name = '' ) : url_releases = ( url + '/releases' ) r = requests . get ( url_releases , headers = headers ) self . releases_json [ repo_name ] = r . json ( )

Retrieves the releases for the given repo in JSON .


def get_referrers(self, url='', headers={}, repo_name=''):
        """
        Retrieves the total referrers and unique referrers of all repos in json
        and then stores it in a dict.
        """
        #JSON
        url_referrers = (url + '/traffic/popular/referrers')
        r1 = requests.get(url_referrers, headers=headers)
        referrers_json = r1.json()
        self.referrers_json[repo_name] = referrers_json
        #CSV
        for referrer in referrers_json:
            ref_name = referrer['referrer']
            try:
                tuple_in = (referrer['count'], referrer['uniques'])#curr vals
                tuple = (self.referrers[ref_name][0] + tuple_in[0],#cal new vals
                    self.referrers[ref_name][1] + tuple_in[1])
                self.referrers[ref_name] = tuple#record new vals
            except KeyError:
                tuple = self.referrers[ref_name] = (referrer['count'],
                    referrer['uniques'])
                self.referrers_lower[ref_name.lower()] = ref_name

def get_referrers ( self , url = '' , headers = { } , repo_name = '' ) : url_referrers = ( url + '/traffic/popular/referrers' ) r1 = requests . get ( url_referrers , headers = headers ) referrers_json = r1 . json ( ) self . referrers_json [ repo_name ] = referrers_json for referrer in referrers_json : ref_name = referrer [ 'referrer' ] try : tuple_in = ( referrer [ 'count' ] , referrer [ 'uniques' ] ) tuple = ( self . referrers [ ref_name ] [ 0 ] + tuple_in [ 0 ] , self . referrers [ ref_name ] [ 1 ] + tuple_in [ 1 ] ) self . referrers [ ref_name ] = tuple except KeyError : tuple = self . referrers [ ref_name ] = ( referrer [ 'count' ] , referrer [ 'uniques' ] ) self . referrers_lower [ ref_name . lower ( ) ] = ref_name

Retrieves the total referrers and unique referrers of all repos in json and then stores it in a dict .


def get_data(self, url='',headers={}, date=str(datetime.date.today()),
        dict_to_store={}, type='', repo_name=''):
        """
        Retrieves data from json and stores it in the supplied dict. Accepts
        'clones' or 'views' as type.
        """
        #JSON
        url = (url + '/traffic/' + type)
        r3 = requests.get(url, headers=headers)
        json = r3.json()
        if type == 'views':
            self.views_json[repo_name] = json
        elif type == 'clones':
            self.clones_json[repo_name] = json
        #CSV
        for day in json[type]:
            timestamp_seconds = day['timestamp']/1000
            try:
                date_timestamp = datetime.datetime.utcfromtimestamp(
                    timestamp_seconds).strftime('%Y-%m-%d')
                #do not add todays date, some views might not be recorded yet
                if date_timestamp != date:
                    tuple_in = (day['count'], day['uniques'])
                    tuple = (dict_to_store[timestamp_seconds][0] + tuple_in[0],
                        dict_to_store[timestamp_seconds][1] + tuple_in[1])
                    dict_to_store[timestamp_seconds] = tuple
            except KeyError:
                tuple = dict_to_store[timestamp_seconds] = (day['count'],
                    day['uniques'])

def get_data ( self , url = '' , headers = { } , date = str ( datetime . date . today ( ) ) , dict_to_store = { } , type = '' , repo_name = '' ) : url = ( url + '/traffic/' + type ) r3 = requests . get ( url , headers = headers ) json = r3 . json ( ) if type == 'views' : self . views_json [ repo_name ] = json elif type == 'clones' : self . clones_json [ repo_name ] = json for day in json [ type ] : timestamp_seconds = day [ 'timestamp' ] / 1000 try : date_timestamp = datetime . datetime . utcfromtimestamp ( timestamp_seconds ) . strftime ( '%Y-%m-%d' ) if date_timestamp != date : tuple_in = ( day [ 'count' ] , day [ 'uniques' ] ) tuple = ( dict_to_store [ timestamp_seconds ] [ 0 ] + tuple_in [ 0 ] , dict_to_store [ timestamp_seconds ] [ 1 ] + tuple_in [ 1 ] ) dict_to_store [ timestamp_seconds ] = tuple except KeyError : tuple = dict_to_store [ timestamp_seconds ] = ( day [ 'count' ] , day [ 'uniques' ] )

Retrieves data from json and stores it in the supplied dict . Accepts clones or views as type .


def write_json(self, date=(datetime.date.today()),
        organization='llnl',dict_to_write={}, path_ending_type=''):
        """
        Writes all traffic data to file in JSON form.
        """
        for repo in dict_to_write:
            if len(dict_to_write[repo]) != 0:#don't need to write out empty lists
                path = ('../github-data/' + organization + '/' + repo + '/' +
                     path_ending_type + '/' + str(date) + '.json')
                self.checkDir(path)
                with open(path, 'w') as out:
                    out.write(json.dumps(dict_to_write[repo], sort_keys=True,
                        indent=4, separators=(',', ': ')))
                out.close()

def
write_json
(
self
,
date
=
(
datetime
.
date
.
today
(
)
)
,
organization
=
'llnl'
,
dict_to_write
=
{
}
,
path_ending_type
=
''
)
:
for
repo
in
dict_to_write
:
if
len
(
dict_to_write
[
repo
]
)
!=
0
:
path
=
(
'../github-data/'
+
organization
+
'/'
+
repo
+
'/'
+
path_ending_type
+
'/'
+
str
(
date
)
+
'.json'
)
self
.
checkDir
(
path
)
with
open
(
path
,
'w'
)
as
out
:
out
.
write
(
json
.
dumps
(
dict_to_write
[
repo
]
,
sort_keys
=
True
,
indent
=
4
,
separators
=
(
','
,
': '
)
)
)
out
.
close
(
)

Writes all traffic data to file in JSON form .


def write_to_file(self, referrers_file_path='', views_file_path='',
        clones_file_path='', date=(datetime.date.today()), organization='llnl',
        views_row_count=0, clones_row_count=0):
        """
        Writes all traffic data to file.
        """
        self.write_referrers_to_file(file_path=referrers_file_path)
        self.write_data_to_file(file_path=views_file_path,
            dict_to_write=self.views, name='views',
            row_count=views_row_count)
        self.write_data_to_file(file_path=clones_file_path,
            dict_to_write=self.clones, name='clones',
            row_count=clones_row_count)

def write_to_file ( self , referrers_file_path = '' , views_file_path = '' , clones_file_path = '' , date = ( datetime . date . today ( ) ) , organization = 'llnl' , views_row_count = 0 , clones_row_count = 0 ) : self . write_referrers_to_file ( file_path = referrers_file_path ) self . write_data_to_file ( file_path = views_file_path , dict_to_write = self . views , name = 'views' , row_count = views_row_count ) self . write_data_to_file ( file_path = clones_file_path , dict_to_write = self . clones , name = 'clones' , row_count = clones_row_count )


def check_data_redundancy(self, file_path='', dict_to_check={}):
        """
        Checks the given csv file against the json data scraped for the given
        dict. It will remove all data retrieved that has already been recorded
        so we don't write redundant data to file. Returns count of rows from
        file.
        """
        count = 0
        exists = os.path.isfile(file_path)
        previous_dates = {}
        if exists:
            with open(file_path, 'r') as input:
                input.readline()#skip header line
                for row in csv.reader(input):
                    timestamp = calendar.timegm(time.strptime(row[0],
                        '%Y-%m-%d'))
                    if timestamp in dict_to_check:#our date is already recorded
                        del dict_to_check[timestamp]
                    #calc current id max
                    count += 1
            input.close()
        return count

def check_data_redundancy ( self , file_path = '' , dict_to_check = { } ) : count = 0 exists = os . path . isfile ( file_path ) previous_dates = { } if exists : with open ( file_path , 'r' ) as input : input . readline ( ) for row in csv . reader ( input ) : timestamp = calendar . timegm ( time . strptime ( row [ 0 ] , '%Y-%m-%d' ) ) if timestamp in dict_to_check : del dict_to_check [ timestamp ] count += 1 input . close ( ) return count

Checks the given csv file against the json data scraped for the given dict . It will remove all data retrieved that has already been recorded so we don t write redundant data to file . Returns count of rows from file .


def write_data_to_file(self, file_path='', date=str(datetime.date.today()),
        organization='llnl',dict_to_write={}, name='', row_count=0):
        """
        Writes given dict to file.
        """
        exists = os.path.isfile(file_path)
        with open(file_path, 'a') as out:
            if not exists:
                out.write('date,organization,' + name + ',unique_' + name
                + ',id\n')
            sorted_dict = sorted(dict_to_write)
            for day in sorted_dict:
                day_formatted = datetime.datetime.utcfromtimestamp(
                    day ).strftime('%Y-%m-%d')
                out.write(day_formatted + ',' + organization + ','
                    + str(dict_to_write[day][0]) + ',' + str(dict_to_write[day][1])
                    + ',' + str(row_count) + '\n')
                row_count += 1

def write_data_to_file ( self , file_path = '' , date = str ( datetime . date . today ( ) ) , organization = 'llnl' , dict_to_write = { } , name = '' , row_count = 0 ) : exists = os . path . isfile ( file_path ) with open ( file_path , 'a' ) as out : if not exists : out . write ( 'date,organization,' + name + ',unique_' + name + ',id\n' ) sorted_dict = sorted ( dict_to_write ) for day in sorted_dict : day_formatted = datetime . datetime . utcfromtimestamp ( day ) . strftime ( '%Y-%m-%d' ) out . write ( day_formatted + ',' + organization + ',' + str ( dict_to_write [ day ] [ 0 ] ) + ',' + str ( dict_to_write [ day ] [ 1 ] ) + ',' + str ( row_count ) + '\n' ) row_count += 1


def write_referrers_to_file(self, file_path='',
        date=str(datetime.date.today()), organization='llnl'):
        """
        Writes the referrers data to file.
        """
        self.remove_date(file_path=file_path, date=date)
        referrers_exists = os.path.isfile(file_path)
        with open(file_path, 'a') as out:
            if not referrers_exists:
                out.write('date,organization,referrer,count,count_log,uniques,'
                    + 'uniques_logged\n')
            sorted_referrers = sorted(self.referrers_lower)#sort based on lowercase
            for referrer in sorted_referrers:
                ref_name = self.referrers_lower[referrer]#grab real name from
                count = self.referrers[ref_name][0]
                uniques = self.referrers[ref_name][1]
                if count == 1:#so we don't display 0 for count of 1
                    count = 1.5
                if uniques == 1:
                    uniques = 1.5
                count_logged = math.log(count)
                uniques_logged = math.log(uniques)
                out.write(date + ',' + organization + ','
                + ref_name + ',' + str(count) + ',' + str(count_logged) + ','
                + str(uniques) + ',' + str(uniques_logged) + '\n')
        out.close()


def process_json(filename):
    """
    Converts a DOE CODE .json file into DOE CODE projects
    Yields DOE CODE records from a DOE CODE .json file
    """

    logger.debug('Processing DOE CODE json: %s', filename)

    doecode_json = json.load(open(filename))

    for record in doecode_json['records']:
        yield record

def
process_json
(
filename
)
:
logger
.
debug
(
'Processing DOE CODE json: %s'
,
filename
)
doecode_json
=
json
.
load
(
open
(
filename
)
)
for
record
in
doecode_json
[
'records'
]
:
yield
record

Converts a DOE CODE . json file into DOE CODE projects Yields DOE CODE records from a DOE CODE . json file


def process_url(url, key):
    """
    Yields DOE CODE records from a DOE CODE .json URL response
    Converts a DOE CODE API .json URL response into DOE CODE projects
    """

    logger.debug('Fetching DOE CODE JSON: %s', url)

    if key is None:
        raise ValueError('DOE CODE API Key value is missing!')

    response = requests.get(url, headers={"Authorization": "Basic " + key})
    doecode_json = response.json()

    for record in doecode_json['records']:
        yield record

def
process_url
(
url
,
key
)
:
logger
.
debug
(
'Fetching DOE CODE JSON: %s'
,
url
)
if
key
is
:
raise
ValueError
(
'DOE CODE API Key value is missing!'
)
response
=
requests
.
get
(
url
,
headers
=
{
"Authorization"
:
"Basic "
+
key
}
)
doecode_json
=
response
.
json
(
)
for
record
in
doecode_json
[
'records'
]
:
yield
record

Yields DOE CODE records from a DOE CODE . json URL response Converts a DOE CODE API . json URL response into DOE CODE projects


def process(filename=None, url=None, key=None):
    """
    Yeilds DOE CODE records based on provided input sources

    param:
        filename (str): Path to a DOE CODE .json file
        url (str): URL for a DOE CODE server json file
        key (str): API Key for connecting to DOE CODE server
    """

    if filename is not None:
        yield from process_json(filename)
    elif url and key:
        yield from process_url(url, key)

def process ( filename = None , url = None , key = None ) : if filename is not None : yield from process_json ( filename ) elif url and key : yield from process_url ( url , key )