def F1(self, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the F1 values for a set of thresholds for the models explored.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where
        the keys are "train", "valid", and "xval".

        :param List thresholds: If None, then the thresholds in this set of metrics will be used.
        :param bool train: If True, return the F1 value for the training data.
        :param bool valid: If True, return the F1 value for the validation data.
        :param bool xval: If True, return the F1 value for each of the cross-validated splits.
        :returns: Dictionary of model keys to F1 values
        """
        return {model.model_id: model.F1(thresholds, train, valid, xval) for model in
                self.models}

def F1 ( self , thresholds = None , train = False , valid = False , xval = False ) : return { model . model_id : model . F1 ( thresholds , train , valid , xval ) for model in self . models }

Get the F1 values for a set of thresholds for the models explored .


def varimp(self, use_pandas=False):
        """
        Return the Importance of components associcated with a pca model.

        use_pandas: ``bool``  (default: ``False``).
        """
        model = self._model_json["output"]
        if "importance" in list(model.keys()) and model["importance"]:
            vals = model["importance"].cell_values
            header = model["importance"].col_header
            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have importances of components.")

Return the Importance of components associcated with a pca model .


def proj_archetypes(self, test_data, reverse_transform=False):
        """
        Convert archetypes of the model into original feature space.

        :param H2OFrame test_data: The dataset upon which the model was trained.
        :param bool reverse_transform: Whether the transformation of the training data during model-building
            should be reversed on the projected archetypes.

        :returns: model archetypes projected back into the original training data's feature space.
        """
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"project_archetypes": True, "reverse_transform": reverse_transform})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])

Convert archetypes of the model into original feature space .


def screeplot(self, type="barplot", **kwargs):
        """
        Produce the scree plot.

        Library ``matplotlib`` is required for this function.

        :param str type: either ``"barplot"`` or ``"lines"``.
        """
        # check for matplotlib. exit if absent.
        is_server = kwargs.pop("server")
        if kwargs:
            raise ValueError("Unknown arguments %s to screeplot()" % ", ".join(kwargs.keys()))
        try:
            import matplotlib
            if is_server: matplotlib.use('Agg', warn=False)
            import matplotlib.pyplot as plt
        except ImportError:
            print("matplotlib is required for this function!")
            return

        variances = [s ** 2 for s in self._model_json['output']['importance'].cell_values[0][1:]]
        plt.xlabel('Components')
        plt.ylabel('Variances')
        plt.title('Scree Plot')
        plt.xticks(list(range(1, len(variances) + 1)))
        if type == "barplot":
            plt.bar(list(range(1, len(variances) + 1)), variances)
        elif type == "lines":
            plt.plot(list(range(1, len(variances) + 1)), variances, 'b--')
        if not is_server: plt.show()


def translate_name(name):
    """
    Convert names with underscores into camelcase.

    For example:
        "num_rows" => "numRows"
        "very_long_json_name" => "veryLongJsonName"
        "build_GBM_model" => "buildGbmModel"
        "KEY" => "key"
        "middle___underscores" => "middleUnderscores"
        "_exclude_fields" => "_excludeFields" (retain initial/trailing underscores)
        "__http_status__" => "__httpStatus__"

    :param name: name to be converted
    """
    parts = name.split("_")
    i = 0
    while parts[i] == "":
        parts[i] = "_"
        i += 1
    parts[i] = parts[i].lower()
    for j in range(i + 1, len(parts)):
        parts[j] = parts[j].capitalize()
    i = len(parts) - 1
    while parts[i] == "":
        parts[i] = "_"
        i -= 1
    return "".join(parts)

def translate_name ( name ) : parts = name . split ( "_" ) i = 0 while parts [ i ] == "" : parts [ i ] = "_" i += 1 parts [ i ] = parts [ i ] . lower ( ) for j in range ( i + 1 , len ( parts ) ) : parts [ j ] = parts [ j ] . capitalize ( ) i = len ( parts ) - 1 while parts [ i ] == "" : parts [ i ] = "_" i -= 1 return "" . join ( parts )

Convert names with underscores into camelcase .


def dedent(ind, text):
    """
    Dedent text to the specific indentation level.

    :param ind: common indentation level for the resulting text (number of spaces to append to every line)
    :param text: text that should be transformed.
    :return: ``text`` with all common indentation removed, and then the specified amount of indentation added.
    """
    text2 = textwrap.dedent(text)
    if ind == 0:
        return text2
    indent_str = " " * ind
    return "\n".join(indent_str + line for line in text2.split("\n"))

def
dedent
(
ind
,
text
)
:
text2
=
textwrap
.
dedent
(
text
)
if
ind
==
0
:
return
text2
indent_str
=
" "
*
ind
return
"\n"
.
join
(
indent_str
+
line
for
line
in
text2
.
split
(
"\n"
)
)

Dedent text to the specific indentation level .


def extractRunInto(javaLogText):
    """
    This function will extract the various operation time for GLRM model building iterations.

    :param javaLogText:
    :return:
    """
    global g_initialXY
    global g_reguarlize_Y
    global g_regularize_X_objective
    global g_updateX
    global g_updateY
    global g_objective
    global g_stepsize
    global g_history

    if os.path.isfile(javaLogText):

        run_result = dict()
        run_result["total time (ms)"] = []
        run_result["initialXY (ms)"] = []
        run_result["regularize Y (ms)"] = []
        run_result["regularize X and objective (ms)"] = []
        run_result["update X (ms)"] = []
        run_result["update Y (ms)"] = []
        run_result["objective (ms)"] = []
        run_result["step size (ms)"] = []
        run_result["update history (ms)"] = []

        total_run_time = -1
        val = 0.0
        with open(javaLogText, 'r') as thefile:   # go into tempfile and grab test run info
            for each_line in thefile:
                temp_string = each_line.split()

                if len(temp_string) > 0:
                    val = temp_string[-1].replace('\\','')

                if g_initialXY in each_line:    # start of a new file
                    if total_run_time > 0:  # update total run time
                        run_result["total time (ms)"].append(total_run_time)
                        total_run_time = 0.0
                    else:
                        total_run_time = 0.0

                    run_result["initialXY (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_reguarlize_Y in each_line:
                    run_result["regularize Y (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_regularize_X_objective in each_line:
                    run_result["regularize X and objective (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_updateX in each_line:
                    run_result["update X (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_updateY in each_line:
                    run_result["update Y (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_objective in each_line:
                    run_result["objective (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_stepsize in each_line:
                    run_result["step size (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

                if g_history in each_line:
                    run_result["update history (ms)"].append(float(val))
                    total_run_time = total_run_time+float(val)

        run_result["total time (ms)"].append(total_run_time)    # save the last one
        print("Run result summary: \n {0}".format(run_result))

    else:
        print("Cannot find your java log file.  Nothing is done.\n")

This function will extract the various operation time for GLRM model building iterations .


def main(argv):
    """
    Main program.  Take user input, parse it and call other functions to execute the commands
    and extract run summary and store run result in json file

    @return: none
    """
    global g_test_root_dir
    global g_temp_filename

    if len(argv) < 2:
        print("invoke this script as python extractGLRMRuntimeJavaLog.py javatextlog.\n")
        sys.exit(1)
    else:   # we may be in business
        javaLogText = argv[1]         # filename while java log is stored

        print("your java text is {0}".format(javaLogText))
        extractRunInto(javaLogText)

Main program . Take user input parse it and call other functions to execute the commands and extract run summary and store run result in json file


def close(self):
        """
        Close an existing connection; once closed it cannot be used again.

        Strictly speaking it is not necessary to close all connection that you opened -- we have several mechanisms
        in place that will do so automatically (__del__(), __exit__() and atexit() handlers), however there is also
        no good reason to make this method private.
        """
        if self._session_id:
            try:
                # If the server gone bad, we don't want to wait forever...
                if self._timeout is None: self._timeout = 1
                self.request("DELETE /4/sessions/%s" % self._session_id)
                self._print("H2O session %s closed." % self._session_id)
            except Exception:
                pass
            self._session_id = None
        self._stage = -1

def
close
(
self
)
:
if
self
.
_session_id
:
try
:
if
self
.
_timeout
is
:
self
.
_timeout
=
1
self
.
request
(
"DELETE /4/sessions/%s"
%
self
.
_session_id
)
self
.
_print
(
"H2O session %s closed."
%
self
.
_session_id
)
except
Exception
:
pass
self
.
_session_id
=
self
.
_stage
=
1

Close an existing connection ; once closed it cannot be used again .


def session_id(self):
        """
        Return the session id of the current connection.

        The session id is issued (through an API request) the first time it is requested, but no sooner. This is
        because generating a session id puts it into the DKV on the server, which effectively locks the cluster. Once
        issued, the session id will stay the same until the connection is closed.
        """
        if self._session_id is None:
            req = self.request("POST /4/sessions")
            self._session_id = req.get("session_key") or req.get("session_id")
        return CallableString(self._session_id)

def
session_id
(
self
)
:
if
self
.
_session_id
is
:
req
=
self
.
request
(
"POST /4/sessions"
)
self
.
_session_id
=
req
.
get
(
"session_key"
)
or
req
.
get
(
"session_id"
)
return
CallableString
(
self
.
_session_id
)

Return the session id of the current connection .


def start_logging(self, dest=None):
        """
        Start logging all API requests to the provided destination.

        :param dest: Where to write the log: either a filename (str), or an open file handle (file). If not given,
            then a new temporary file will be created.
        """
        assert_is_type(dest, None, str, type(sys.stdout))
        if dest is None:
            dest = os.path.join(tempfile.mkdtemp(), "h2o-connection.log")
        self._print("Now logging all API requests to file %r" % dest)
        self._is_logging = True
        self._logging_dest = dest

Start logging all API requests to the provided destination .


def _prepare_data_payload(data):
        """
        Make a copy of the `data` object, preparing it to be sent to the server.

        The data will be sent via x-www-form-urlencoded or multipart/form-data mechanisms. Both of them work with
        plain lists of key/value pairs, so this method converts the data into such format.
        """
        if not data: return None
        res = {}
        for key, value in viewitems(data):
            if value is None: continue  # don't send args set to None so backend defaults take precedence
            if isinstance(value, list):
                value = stringify_list(value)
            elif isinstance(value, dict):
                if "__meta" in value and value["__meta"]["schema_name"].endswith("KeyV3"):
                    value = value["name"]
                else:
                    value = stringify_dict(value)
            else:
                value = str(value)
            res[key] = value
        return res

def _prepare_data_payload ( data ) : if not data : return None res = { } for key , value in viewitems ( data ) : if value is None : continue if isinstance ( value , list ) : value = stringify_list ( value ) elif isinstance ( value , dict ) : if "__meta" in value and value [ "__meta" ] [ "schema_name" ] . endswith ( "KeyV3" ) : value = value [ "name" ] else : value = stringify_dict ( value ) else : value = str ( value ) res [ key ] = value return res

Make a copy of the data object preparing it to be sent to the server .


def _prepare_file_payload(filename):
        """
        Prepare `filename` to be sent to the server.

        The "preparation" consists of creating a data structure suitable
        for passing to requests.request().
        """
        if not filename: return None
        absfilename = os.path.abspath(filename)
        if not os.path.exists(absfilename):
            raise H2OValueError("File %s does not exist" % filename, skip_frames=1)
        return {os.path.basename(absfilename): open(absfilename, "rb")}

def
_prepare_file_payload
(
filename
)
:
if
not
filename
:
return
absfilename
=
os
.
path
.
abspath
(
filename
)
if
not
os
.
path
.
exists
(
absfilename
)
:
raise
H2OValueError
(
"File %s does not exist"
%
filename
,
skip_frames
=
1
)
return
{
os
.
path
.
basename
(
absfilename
)
:
open
(
absfilename
,
"rb"
)
}

Prepare filename to be sent to the server .


def _log_start_transaction(self, endpoint, data, json, files, params):
        """Log the beginning of an API request."""
        # TODO: add information about the caller, i.e. which module + line of code called the .request() method
        #       This can be done by fetching current traceback and then traversing it until we find the request function
        self._requests_counter += 1
        if not self._is_logging: return
        msg = "\n---- %d --------------------------------------------------------\n" % self._requests_counter
        msg += "[%s] %s\n" % (time.strftime("%H:%M:%S"), endpoint)
        if params is not None: msg += "     params: {%s}\n" % ", ".join("%s:%s" % item for item in viewitems(params))
        if data is not None:   msg += "     body: {%s}\n" % ", ".join("%s:%s" % item for item in viewitems(data))
        if json is not None:
            import json as j
            msg += "     json: %s\n" % j.dumps(json)
        if files is not None:  msg += "     file: %s\n" % ", ".join(f.name for f in viewvalues(files))
        self._log_message(msg + "\n")


def _log_end_transaction(self, start_time, response):
        """Log response from an API request."""
        if not self._is_logging: return
        elapsed_time = int((time.time() - start_time) * 1000)
        msg = "<<< HTTP %d %s   (%d ms)\n" % (response.status_code, response.reason, elapsed_time)
        if "Content-Type" in response.headers:
            msg += "    Content-Type: %s\n" % response.headers["Content-Type"]
        msg += response.text
        self._log_message(msg + "\n\n")

def
_log_end_transaction
(
self
,
start_time
,
response
)
:
if
not
self
.
_is_logging
:
return
elapsed_time
=
int
(
(
time
.
time
(
)
start_time
)
*
1000
)
msg
=
"<<< HTTP %d %s   (%d ms)\n"
%
(
response
.
status_code
,
response
.
reason
,
elapsed_time
)
if
"Content-Type"
in
response
.
headers
:
msg
+=
"    Content-Type: %s\n"
%
response
.
headers
[
"Content-Type"
]
msg
+=
response
.
text
self
.
_log_message
(
msg
+
"\n\n"
)


def _log_message(self, msg):
        """
        Log the message `msg` to the destination `self._logging_dest`.

        If this destination is a file name, then we append the message to the file and then close the file
        immediately. If the destination is an open file handle, then we simply write the message there and do not
        attempt to close it.
        """
        if is_type(self._logging_dest, str):
            with open(self._logging_dest, "at", encoding="utf-8") as f:
                f.write(msg)
        else:
            self._logging_dest.write(msg)

def _log_message ( self , msg ) : if is_type ( self . _logging_dest , str ) : with open ( self . _logging_dest , "at" , encoding = "utf-8" ) as f : f . write ( msg ) else : self . _logging_dest . write ( msg )

Log the message msg to the destination self . _logging_dest .


def _process_response(response, save_to):
        """
        Given a response object, prepare it to be handed over to the external caller.

        Preparation steps include:
           * detect if the response has error status, and convert it to an appropriate exception;
           * detect Content-Type, and based on that either parse the response as JSON or return as plain text.
        """
        status_code = response.status_code
        if status_code == 200 and save_to:
            if save_to.startswith("~"): save_to = os.path.expanduser(save_to)
            if os.path.isdir(save_to) or save_to.endswith(os.path.sep):
                dirname = os.path.abspath(save_to)
                filename = H2OConnection._find_file_name(response)
            else:
                dirname, filename = os.path.split(os.path.abspath(save_to))
            fullname = os.path.join(dirname, filename)
            try:
                if not os.path.exists(dirname):
                    os.makedirs(dirname)
                with open(fullname, "wb") as f:
                    for chunk in response.iter_content(chunk_size=65536):
                        if chunk:  # Empty chunks may occasionally happen
                            f.write(chunk)
            except OSError as e:
                raise H2OValueError("Cannot write to file %s: %s" % (fullname, e))
            return fullname

        content_type = response.headers.get("Content-Type", "")
        if ";" in content_type:  # Remove a ";charset=..." part
            content_type = content_type[:content_type.index(";")]

        # Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is.
        if content_type == "application/json":
            try:
                data = response.json(object_pairs_hook=H2OResponse)
            except (JSONDecodeError, requests.exceptions.ContentDecodingError) as e:
                raise H2OServerError("Malformed JSON from server (%s):\n%s" % (str(e), response.text))
        else:
            data = response.text

        # Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content")
        if status_code in {200, 201, 202, 204}:
            return data

        # Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed")
        if status_code in {400, 404, 412} and isinstance(data, (H2OErrorV3, H2OModelBuilderErrorV3)):
            raise H2OResponseError(data)

        # Server errors (notably 500 = "Server Error")
        # Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server
        # did not provide the correct status code.
        raise H2OServerError("HTTP %d %s:\n%r" % (status_code, response.reason, data))

def
_process_response
(
response
,
save_to
)
:
status_code
=
response
.
status_code
if
status_code
==
200
and
save_to
:
if
save_to
.
startswith
(
"~"
)
:
save_to
=
os
.
path
.
expanduser
(
save_to
)
if
os
.
path
.
isdir
(
save_to
)
or
save_to
.
endswith
(
os
.
path
.
sep
)
:
dirname
=
os
.
path
.
abspath
(
save_to
)
filename
=
H2OConnection
.
_find_file_name
(
response
)
else
:
dirname
,
filename
=
os
.
path
.
split
(
os
.
path
.
abspath
(
save_to
)
)
fullname
=
os
.
path
.
join
(
dirname
,
filename
)
try
:
if
not
os
.
path
.
exists
(
dirname
)
:
os
.
makedirs
(
dirname
)
with
open
(
fullname
,
"wb"
)
as
f
:
for
chunk
in
response
.
iter_content
(
chunk_size
=
65536
)
:
if
chunk
:
f
.
write
(
chunk
)
except
OSError
as
e
:
raise
H2OValueError
(
"Cannot write to file %s: %s"
%
(
fullname
,
e
)
)
return
fullname
content_type
=
response
.
headers
.
get
(
"Content-Type"
,
""
)
if
";"
in
content_type
:
content_type
=
content_type
[
:
content_type
.
index
(
";"
)
]
if
content_type
==
"application/json"
:
try
:
data
=
response
.
json
(
object_pairs_hook
=
H2OResponse
)
except
(
JSONDecodeError
,
requests
.
exceptions
.
ContentDecodingError
)
as
e
:
raise
H2OServerError
(
"Malformed JSON from server (%s):\n%s"
%
(
str
(
e
)
,
response
.
text
)
)
else
:
data
=
response
.
text
if
status_code
in
{
200
,
201
,
202
,
204
}
:
return
data
if
status_code
in
{
400
,
404
,
412
}
and
isinstance
(
data
,
(
H2OErrorV3
,
H2OModelBuilderErrorV3
)
)
:
raise
H2OResponseError
(
data
)
raise
H2OServerError
(
"HTTP %d %s:\n%r"
%
(
status_code
,
response
.
reason
,
data
)
)

Given a response object prepare it to be handed over to the external caller .


def _print(self, msg, flush=False, end="\n"):
        """Helper function to print connection status messages when in verbose mode."""
        if self._verbose:
            print2(msg, end=end, flush=flush)

def _print ( self , msg , flush = False , end = "\n" ) : if self . _verbose : print2 ( msg , end = end , flush = flush )

Helper function to print connection status messages when in verbose mode .


def get_automl(project_name):
    """
    Retrieve information about an AutoML instance.

    :param str project_name:  A string indicating the project_name of the automl instance to retrieve.
    :returns: A dictionary containing the project_name, leader model, and leaderboard.
    """
    automl_json = h2o.api("GET /99/AutoML/%s" % project_name)
    project_name = automl_json["project_name"]
    leaderboard_list = [key["name"] for key in automl_json['leaderboard']['models']]

    if leaderboard_list is not None and len(leaderboard_list) > 0:
        leader_id = leaderboard_list[0]
    else:
        leader_id = None

    leader = h2o.get_model(leader_id)
    # Intentionally mask the progress bar here since showing multiple progress bars is confusing to users.
    # If any failure happens, revert back to user's original setting for progress and display the error message.
    is_progress = H2OJob.__PROGRESS_BAR__
    h2o.no_progress()
    try:
        # Parse leaderboard H2OTwoDimTable & return as an H2OFrame
        leaderboard = h2o.H2OFrame(
            automl_json["leaderboard_table"].cell_values,
            column_names=automl_json["leaderboard_table"].col_header)
    except Exception as ex:
        raise ex
    finally:
        if is_progress is True:
            h2o.show_progress()

    leaderboard = leaderboard[1:]
    automl_dict = {'project_name': project_name, "leader": leader, "leaderboard": leaderboard}
    return automl_dict

def
get_automl
(
project_name
)
:
automl_json
=
h2o
.
api
(
"GET /99/AutoML/%s"
%
project_name
)
project_name
=
automl_json
[
"project_name"
]
leaderboard_list
=
[
key
[
"name"
]
for
key
in
automl_json
[
'leaderboard'
]
[
'models'
]
]
if
leaderboard_list
is
not
and
len
(
leaderboard_list
)
>
0
:
leader_id
=
leaderboard_list
[
0
]
else
:
leader_id
=
leader
=
h2o
.
get_model
(
leader_id
)
is_progress
=
H2OJob
.
__PROGRESS_BAR__
h2o
.
no_progress
(
)
try
:
leaderboard
=
h2o
.
H2OFrame
(
automl_json
[
"leaderboard_table"
]
.
cell_values
,
column_names
=
automl_json
[
"leaderboard_table"
]
.
col_header
)
except
Exception
as
ex
:
raise
ex
finally
:
if
is_progress
is
True
:
h2o
.
show_progress
(
)
leaderboard
=
leaderboard
[
1
:
]
automl_dict
=
{
'project_name'
:
project_name
,
"leader"
:
leader
,
"leaderboard"
:
leaderboard
}
return
automl_dict

Retrieve information about an AutoML instance .


def download_pojo(self, path="", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the POJO for the leader model in AutoML to the directory specified by path.

        If path is an empty string, then dump the output to screen.

        :param path:  An absolute path to the directory where POJO should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the POJO file written.
        """

        return h2o.download_pojo(self.leader, path, get_jar=get_genmodel_jar, jar_name=genmodel_name)

def download_pojo ( self , path = "" , get_genmodel_jar = False , genmodel_name = "" ) : return h2o . download_pojo ( self . leader , path , get_jar = get_genmodel_jar , jar_name = genmodel_name )

Download the POJO for the leader model in AutoML to the directory specified by path .


def download_mojo(self, path=".", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the leader model in AutoML in MOJO format.

        :param path: the path where MOJO file should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the MOJO file written.
        """

        return ModelBase.download_mojo(self.leader, path, get_genmodel_jar, genmodel_name)

def download_mojo ( self , path = "." , get_genmodel_jar = False , genmodel_name = "" ) : return ModelBase . download_mojo ( self . leader , path , get_genmodel_jar , genmodel_name )

Download the leader model in AutoML in MOJO format .


def fit(self, X, y=None, **params):
        """
        Fit this object by computing the means and standard deviations used by the transform method.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: Ignored
        :returns: This H2OScaler instance
        """
        if isinstance(self.parms["center"], (tuple, list)): self._means = self.parms["center"]
        if isinstance(self.parms["scale"], (tuple, list)): self._stds = self.parms["scale"]
        if self.means is None and self.parms["center"]:
            self._means = X.mean(return_frame=True).getrow()
        else:
            self._means = False
        if self.stds is None and self.parms["scale"]:
            self._stds = X.sd()
        else:
            self._stds = False
        return self

def fit ( self , X , y = None , ** params ) : if isinstance ( self . parms [ "center" ] , ( tuple , list ) ) : self . _means = self . parms [ "center" ] if isinstance ( self . parms [ "scale" ] , ( tuple , list ) ) : self . _stds = self . parms [ "scale" ] if self . means is None and self . parms [ "center" ] : self . _means = X . mean ( return_frame = True ) . getrow ( ) else : self . _means = False if self . stds is None and self . parms [ "scale" ] : self . _stds = X . sd ( ) else : self . _stds = False return self

Fit this object by computing the means and standard deviations used by the transform method .


def transform(self, X, y=None, **params):
        """
        Scale an H2OFrame with the fitted means and standard deviations.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: (Ignored)
        :returns: A scaled H2OFrame.
        """
        return X.scale(self.means, self.stds)

def transform ( self , X , y = None , ** params ) : return X . scale ( self . means , self . stds )

Scale an H2OFrame with the fitted means and standard deviations .


def inverse_transform(self, X, y=None, **params):
        """
        Undo the scale transformation.

        :param X: An H2OFrame; may contain NAs and/or categoricals.
        :param y: None (Ignored)
        :param params: (Ignored)
        :returns: An H2OFrame
        """
        for i in range(X.ncol):
            X[i] = self.means[i] + self.stds[i] * X[i]
        return X

def inverse_transform ( self , X , y = None , ** params ) : for i in range ( X . ncol ) : X [ i ] = self . means [ i ] + self . stds [ i ] * X [ i ] return X


def extract_true_string(string_content):
    """
    remove extra characters before the actual string we are
    looking for.  The Jenkins console output is encoded using utf-8.  However, the stupid
    redirect function can only encode using ASCII.  I have googled for half a day with no
    results to how to resolve the issue.  Hence, we are going to the heat and just manually
    get rid of the junk.

    Parameters
    ----------

    string_content :  str
        contains a line read in from jenkins console

    :return: str: contains the content of the line after the string '[0m'

    """

    startL,found,endL = string_content.partition('[0m')

    if found:
        return endL
    else:
        return string_content

def extract_true_string ( string_content ) : startL , found , endL = string_content . partition ( '[0m' ) if found : return endL else : return string_content

remove extra characters before the actual string we are looking for . The Jenkins console output is encoded using utf - 8 . However the stupid redirect function can only encode using ASCII . I have googled for half a day with no results to how to resolve the issue . Hence we are going to the heat and just manually get rid of the junk .


def find_node_name(each_line,temp_func_list):
    """
    Find the slave machine where a Jenkins job was executed on.  It will save this
    information in g_failed_test_info_dict.  In addition, it will
    delete this particular function handle off the temp_func_list as we do not need
    to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_node_name
    global g_failed_test_info_dict

    if g_node_name in each_line:
        temp_strings = each_line.split()
        [start,found,endstr] = each_line.partition(g_node_name)

        if found:
            temp_strings = endstr.split()
            g_failed_test_info_dict["6.node_name"] = extract_true_string(temp_strings[1])
            temp_func_list.remove(find_node_name)

    return True

def find_node_name ( each_line , temp_func_list ) : global g_node_name global g_failed_test_info_dict if g_node_name in each_line : temp_strings = each_line . split ( ) [ start , found , endstr ] = each_line . partition ( g_node_name ) if found : temp_strings = endstr . split ( ) g_failed_test_info_dict [ "6.node_name" ] = extract_true_string ( temp_strings [ 1 ] ) temp_func_list . remove ( find_node_name ) return True

Find the slave machine where a Jenkins job was executed on . It will save this information in g_failed_test_info_dict . In addition it will delete this particular function handle off the temp_func_list as we do not need to perform this action again .


def find_git_hash_branch(each_line,temp_func_list):
    """
    Find the git hash and branch info that  a Jenkins job was taken from.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_git_hash_branch
    global g_failed_test_info_dict

    if g_git_hash_branch in each_line:
        [start,found,endstr] = each_line.partition(g_git_hash_branch)
        temp_strings = endstr.strip().split()

        if len(temp_strings) > 1:
            g_failed_test_info_dict["4.git_hash"] = temp_strings[0]
            g_failed_test_info_dict["5.git_branch"] = temp_strings[1]

        temp_func_list.remove(find_git_hash_branch)

    return True

def find_git_hash_branch ( each_line , temp_func_list ) : global g_git_hash_branch global g_failed_test_info_dict if g_git_hash_branch in each_line : [ start , found , endstr ] = each_line . partition ( g_git_hash_branch ) temp_strings = endstr . strip ( ) . split ( ) if len ( temp_strings ) > 1 : g_failed_test_info_dict [ "4.git_hash" ] = temp_strings [ 0 ] g_failed_test_info_dict [ "5.git_branch" ] = temp_strings [ 1 ] temp_func_list . remove ( find_git_hash_branch ) return True

Find the git hash and branch info that a Jenkins job was taken from . It will save this information in g_failed_test_info_dict . In addition it will delete this particular function handle off the temp_func_list as we do not need to perform this action again .


def find_build_timeout(each_line,temp_func_list):
    """
    Find if a Jenkins job has taken too long to finish and was killed.  It will save this
    information in g_failed_test_info_dict.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
"""
    global g_build_timeout
    global g_failed_test_info_dict
    global g_failure_occurred

    if g_build_timeout in each_line:
        g_failed_test_info_dict["8.build_timeout"] = 'Yes'
        g_failure_occurred = True
        return False    # build timeout was found, no need to continue mining the console text
    else:
        return True

def find_build_timeout ( each_line , temp_func_list ) : global g_build_timeout global g_failed_test_info_dict global g_failure_occurred if g_build_timeout in each_line : g_failed_test_info_dict [ "8.build_timeout" ] = 'Yes' g_failure_occurred = True return False else : return True

Find if a Jenkins job has taken too long to finish and was killed . It will save this information in g_failed_test_info_dict .


def find_build_failure(each_line,temp_func_list):
    """
    Find if a Jenkins job has failed to build.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_build_success
    global g_build_success_tests
    global g_failed_test_info_dict
    global g_failure_occurred
    global g_build_failed_message

    for ind in range(0,len(g_build_failed_message)):
        if g_build_failed_message[ind] in each_line.lower():
            if ((ind == 0) and (len(g_failed_jobs) > 0)):
                continue
            else:
                g_failure_occurred = True
                g_failed_test_info_dict["7.build_failure"] = 'Yes'
                temp_func_list.remove(find_build_failure)
                return False

    return True

def find_build_failure ( each_line , temp_func_list ) : global g_build_success global g_build_success_tests global g_failed_test_info_dict global g_failure_occurred global g_build_failed_message for ind in range ( 0 , len ( g_build_failed_message ) ) : if g_build_failed_message [ ind ] in each_line . lower ( ) : if ( ( ind == 0 ) and ( len ( g_failed_jobs ) > 0 ) ) : continue else : g_failure_occurred = True g_failed_test_info_dict [ "7.build_failure" ] = 'Yes' temp_func_list . remove ( find_build_failure ) return False return True

Find if a Jenkins job has failed to build . It will save this information in g_failed_test_info_dict . In addition it will delete this particular function handle off the temp_func_list as we do not need to perform this action again .


def find_build_id(each_line,temp_func_list):
    """
    Find the build id of a jenkins job.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_before_java_file
    global g_java_filenames
    global g_build_id_text
    global g_jenkins_url
    global g_output_filename
    global g_output_pickle_filename

    if g_build_id_text in each_line:
        [startStr,found,endStr] = each_line.partition(g_build_id_text)
        g_failed_test_info_dict["2.build_id"] = endStr.strip()

        temp_func_list.remove(find_build_id)
        g_jenkins_url = os.path.join('http://',g_jenkins_url,'view',g_view_name,'job',g_failed_test_info_dict["1.jobName"],g_failed_test_info_dict["2.build_id"],'artifact')

    return True

def find_build_id ( each_line , temp_func_list ) : global g_before_java_file global g_java_filenames global g_build_id_text global g_jenkins_url global g_output_filename global g_output_pickle_filename if g_build_id_text in each_line : [ startStr , found , endStr ] = each_line . partition ( g_build_id_text ) g_failed_test_info_dict [ "2.build_id" ] = endStr . strip ( ) temp_func_list . remove ( find_build_id ) g_jenkins_url = os . path . join ( 'http://' , g_jenkins_url , 'view' , g_view_name , 'job' , g_failed_test_info_dict [ "1.jobName" ] , g_failed_test_info_dict [ "2.build_id" ] , 'artifact' ) return True

Find the build id of a jenkins job . It will save this information in g_failed_test_info_dict . In addition it will delete this particular function handle off the temp_func_list as we do not need to perform this action again .


def extract_job_build_url(url_string):
    """
    From user input, grab the jenkins job name and saved it in g_failed_test_info_dict.
    In addition, it will grab the jenkins url and the view name into g_jenkins_url, and
    g_view_name.

    Parameters
    ----------
    url_string :  str
        contains information on the jenkins job whose console output we are interested in.

    :return: none
    """
    global g_failed_test_info_dict
    global g_jenkins_url
    global g_view_name

    tempString = url_string.strip('/').split('/')

    if len(tempString) < 6:
        print "Illegal URL resource address.\n"
        sys.exit(1)

    g_failed_test_info_dict["1.jobName"] = tempString[6]

    g_jenkins_url = tempString[2]
    g_view_name = tempString[4]

def
extract_job_build_url
(
url_string
)
:
global
g_failed_test_info_dict
global
g_jenkins_url
global
g_view_name
tempString
=
url_string
.
strip
(
'/'
)
.
split
(
'/'
)
if
len
(
tempString
)
<
6
:
print
"Illegal URL resource address.\n"
sys
.
exit
(
1
)
g_failed_test_info_dict
[
"1.jobName"
]
=
tempString
[
6
]
g_jenkins_url
=
tempString
[
2
]
g_view_name
=
tempString
[
4
]

From user input grab the jenkins job name and saved it in g_failed_test_info_dict . In addition it will grab the jenkins url and the view name into g_jenkins_url and g_view_name .


def grab_java_message():
    """scan through the java output text and extract the bad java messages that may or may not happened when
    unit tests are run.  It will not record any bad java messages that are stored in g_ok_java_messages.

    :return: none
    """

    global g_temp_filename
    global g_current_testname
    global g_java_start_text
    global g_ok_java_messages
    global g_java_general_bad_messages  # store bad java messages not associated with running a unit test
    global g_java_general_bad_message_types
    global g_failure_occurred
    global g_java_message_type
    global g_all_java_message_type
    global g_toContinue

    java_messages = []      # store all bad java messages associated with running a unit test
    java_message_types = [] # store all bad java message types associated with running a unit test

    if os.path.isfile(g_temp_filename): # open temp file containing content of some java_*_0.out.txt
        java_file = open(g_temp_filename,'r')

        g_toContinue = False    # denote if a multi-line message starts

        tempMessage = ""
        messageType = ""

        for each_line in java_file:

            if (g_java_start_text in each_line):
                startStr,found,endStr = each_line.partition(g_java_start_text)

                if len(found) > 0:
                    if len(g_current_testname) > 0: # a new unit test is being started.  Save old info and move on
                        associate_test_with_java(g_current_testname,java_messages,java_message_types)

                    g_current_testname = endStr.strip() # record the test name

                    java_messages = []
                    java_message_types = []

            temp_strings = each_line.strip().split()

            if (len(temp_strings) >= 6) and (temp_strings[5] in g_all_java_message_type):
                if g_toContinue == True:    # at the end of last message fragment
                    addJavaMessages(tempMessage,messageType,java_messages,java_message_types)
                    tempMessage = ""
                    messageType = ""

                # start of new message fragment
                g_toContinue = False
            else: # non standard output.  Continuation of last java message, add it to bad java message list
                if g_toContinue:

                    tempMessage += each_line    # add more java message here
                    # if len(g_current_testname) == 0:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)
                    # else:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)

            if ((len(temp_strings) > 5) and (temp_strings[5] in g_java_message_type)):  # find a bad java message
                startStr,found,endStr = each_line.partition(temp_strings[5])    # can be WARN,ERRR,FATAL,TRACE

                if found and (len(endStr.strip()) > 0):
                    tempMessage += endStr
                    messageType = temp_strings[5]
#                    if (tempMessage not in g_ok_java_messages["general"]):  # found new bad messages that cannot be ignored
                    g_toContinue = True

                        # add tempMessage to bad java message list
#                        addJavaMessages(tempMessage,temp_strings[5],java_messages,java_message_types)
        java_file.close()

def grab_java_message ( ) : global g_temp_filename global g_current_testname global g_java_start_text global g_ok_java_messages global g_java_general_bad_messages global g_java_general_bad_message_types global g_failure_occurred global g_java_message_type global g_all_java_message_type global g_toContinue java_messages = [ ] java_message_types = [ ] if os . path . isfile ( g_temp_filename ) : java_file = open ( g_temp_filename , 'r' ) g_toContinue = False tempMessage = "" messageType = "" for each_line in java_file : if ( g_java_start_text in each_line ) : startStr , found , endStr = each_line . partition ( g_java_start_text ) if len ( found ) > 0 : if len ( g_current_testname ) > 0 : associate_test_with_java ( g_current_testname , java_messages , java_message_types ) g_current_testname = endStr . strip ( ) java_messages = [ ] java_message_types = [ ] temp_strings = each_line . strip ( ) . split ( ) if ( len ( temp_strings ) >= 6 ) and ( temp_strings [ 5 ] in g_all_java_message_type ) : if g_toContinue == True : addJavaMessages ( tempMessage , messageType , java_messages , java_message_types ) tempMessage = "" messageType = "" g_toContinue = False else : if g_toContinue : tempMessage += each_line if ( ( len ( temp_strings ) > 5 ) and ( temp_strings [ 5 ] in g_java_message_type ) ) : startStr , found , endStr = each_line . partition ( temp_strings [ 5 ] ) if found and ( len ( endStr . strip ( ) ) > 0 ) : tempMessage += endStr messageType = temp_strings [ 5 ] g_toContinue = True java_file . close ( )

scan through the java output text and extract the bad java messages that may or may not happened when unit tests are run . It will not record any bad java messages that are stored in g_ok_java_messages .


def save_dict():
    """
    Save the log scraping results into logs denoted by g_output_filename_failed_tests and
    g_output_filename_passed_tests.

    :return: none
    """

    global g_test_root_dir
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failed_test_info_dict

    # some build can fail really early that no buid id info is stored in the console text.
    if "2.build_id" not in g_failed_test_info_dict.keys():
        g_failed_test_info_dict["2.build_id"] = "unknown"

    build_id = g_failed_test_info_dict["2.build_id"]

    g_output_filename_failed_tests = g_output_filename_failed_tests+'_build_'+build_id+'_failed_tests.log'
    g_output_filename_passed_tests = g_output_filename_passed_tests+'_build_'+build_id+'_passed_tests.log'
    g_output_pickle_filename = g_output_pickle_filename+'_build_'+build_id+'.pickle'

    allKeys = sorted(g_failed_test_info_dict.keys())

    # write out the jenkins job info into log files.
    with open(g_output_pickle_filename,'wb') as test_file:
        pickle.dump(g_failed_test_info_dict,test_file)

    # write out the failure report as text into a text file
    text_file_failed_tests = open(g_output_filename_failed_tests,'w')
    text_file_passed_tests = None
    allKeys = sorted(g_failed_test_info_dict.keys())
    write_passed_tests = False

    if ("passed_tests_info *********" in allKeys):
        text_file_passed_tests = open(g_output_filename_passed_tests,'w')
        write_passed_tests = True

    for keyName in allKeys:
        val = g_failed_test_info_dict[keyName]
        if isinstance(val,list):    # writing one of the job lists
            if (len(val) == 3):     # it is a message for a test
                if keyName == "failed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_failed_tests)

                if keyName == "passed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_passed_tests)
            elif (len(val) == 2):                   # it is a general bad java message
                write_java_message(keyName,val,text_file_failed_tests)
                if write_passed_tests:
                    write_java_message(keyName,val,text_file_passed_tests)
        else:
            write_general_build_message(keyName,val,text_file_failed_tests)
            if write_passed_tests:
                write_general_build_message(keyName,val,text_file_passed_tests)

    text_file_failed_tests.close()
    if write_passed_tests:
        text_file_passed_tests.close()

def
save_dict
(
)
:
global
g_test_root_dir
global
g_output_filename_failed_tests
global
g_output_filename_passed_tests
global
g_output_pickle_filename
global
g_failed_test_info_dict
if
"2.build_id"
not
in
g_failed_test_info_dict
.
keys
(
)
:
g_failed_test_info_dict
[
"2.build_id"
]
=
"unknown"
build_id
=
g_failed_test_info_dict
[
"2.build_id"
]
g_output_filename_failed_tests
=
g_output_filename_failed_tests
+
'_build_'
+
build_id
+
'_failed_tests.log'
g_output_filename_passed_tests
=
g_output_filename_passed_tests
+
'_build_'
+
build_id
+
'_passed_tests.log'
g_output_pickle_filename
=
g_output_pickle_filename
+
'_build_'
+
build_id
+
'.pickle'
allKeys
=
sorted
(
g_failed_test_info_dict
.
keys
(
)
)
with
open
(
g_output_pickle_filename
,
'wb'
)
as
test_file
:
pickle
.
dump
(
g_failed_test_info_dict
,
test_file
)
text_file_failed_tests
=
open
(
g_output_filename_failed_tests
,
'w'
)
text_file_passed_tests
=
allKeys
=
sorted
(
g_failed_test_info_dict
.
keys
(
)
)
write_passed_tests
=
False
if
(
"passed_tests_info *********"
in
allKeys
)
:
text_file_passed_tests
=
open
(
g_output_filename_passed_tests
,
'w'
)
write_passed_tests
=
True
for
keyName
in
allKeys
:
val
=
g_failed_test_info_dict
[
keyName
]
if
isinstance
(
val
,
list
)
:
if
(
len
(
val
)
==
3
)
:
if
keyName
==
"failed_tests_info *********"
:
write_test_java_message
(
keyName
,
val
,
text_file_failed_tests
)
if
keyName
==
"passed_tests_info *********"
:
write_test_java_message
(
keyName
,
val
,
text_file_passed_tests
)
elif
(
len
(
val
)
==
2
)
:
write_java_message
(
keyName
,
val
,
text_file_failed_tests
)
if
write_passed_tests
:
write_java_message
(
keyName
,
val
,
text_file_passed_tests
)
else
:
write_general_build_message
(
keyName
,
val
,
text_file_failed_tests
)
if
write_passed_tests
:
write_general_build_message
(
keyName
,
val
,
text_file_passed_tests
)
text_file_failed_tests
.
close
(
)
if
write_passed_tests
:
text_file_passed_tests
.
close
(
)

Save the log scraping results into logs denoted by g_output_filename_failed_tests and g_output_filename_passed_tests .


def update_summary_file():
    """
    Concatecate all log file into a summary text file to be sent to users
    at the end of a daily log scraping.

    :return: none
    """
    global g_summary_text_filename
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests

    with open(g_summary_text_filename,'a') as tempfile:
        write_file_content(tempfile,g_output_filename_failed_tests)
        write_file_content(tempfile,g_output_filename_passed_tests)

def update_summary_file ( ) : global g_summary_text_filename global g_output_filename_failed_tests global g_output_filename_passed_tests with open ( g_summary_text_filename , 'a' ) as tempfile : write_file_content ( tempfile , g_output_filename_failed_tests ) write_file_content ( tempfile , g_output_filename_passed_tests )

Concatecate all log file into a summary text file to be sent to users at the end of a daily log scraping .


def write_file_content(fhandle,file2read):
    """
    Write one log file into the summary text file.

    Parameters
    ----------
    fhandle :  Python file handle
        file handle to the summary text file
    file2read : Python file handle
        file handle to log file where we want to add its content to the summary text file.

    :return: none
    """
    if os.path.isfile(file2read):

        # write summary of failed tests logs
        with open(file2read,'r') as tfile:
            fhandle.write('============ Content of '+ file2read)
            fhandle.write('\n')
            fhandle.write(tfile.read())
            fhandle.write('\n\n')

def
write_file_content
(
fhandle
,
file2read
)
:
if
os
.
path
.
isfile
(
file2read
)
:
with
open
(
file2read
,
'r'
)
as
tfile
:
fhandle
.
write
(
'============ Content of '
+
file2read
)
fhandle
.
write
(
'\n'
)
fhandle
.
write
(
tfile
.
read
(
)
)
fhandle
.
write
(
'\n\n'
)

Write one log file into the summary text file .


def write_java_message(key,val,text_file):
    """
    Loop through all java messages that are not associated with a unit test and
    write them into a log file.

    Parameters
    ----------
    key :  str
        9.general_bad_java_messages
    val : list of list of str
        contains the bad java messages and the message types.

    :return: none
    """

    text_file.write(key)
    text_file.write('\n')

    if (len(val[0]) > 0) and (len(val) >= 3):
        for index in range(len(val[0])):
            text_file.write("Java Message Type: ")
            text_file.write(val[1][index])
            text_file.write('\n')

            text_file.write("Java Message: ")

            for jmess in val[2][index]:
                text_file.write(jmess)
                text_file.write('\n')

        text_file.write('\n \n')

def
write_java_message
(
key
,
val
,
text_file
)
:
text_file
.
write
(
key
)
text_file
.
write
(
'\n'
)
if
(
len
(
val
[
0
]
)
>
0
)
and
(
len
(
val
)
>=
3
)
:
for
index
in
range
(
len
(
val
[
0
]
)
)
:
text_file
.
write
(
"Java Message Type: "
)
text_file
.
write
(
val
[
1
]
[
index
]
)
text_file
.
write
(
'\n'
)
text_file
.
write
(
"Java Message: "
)
for
jmess
in
val
[
2
]
[
index
]
:
text_file
.
write
(
jmess
)
text_file
.
write
(
'\n'
)
text_file
.
write
(
'\n \n'
)

Loop through all java messages that are not associated with a unit test and write them into a log file .


def load_java_messages_to_ignore():
    """
    Load in pickle file that contains dict structure with bad java messages to ignore per unit test
    or for all cases.  The ignored bad java info is stored in g_ok_java_messages dict.

    :return:
    """
    global g_ok_java_messages
    global g_java_message_pickle_filename

    if os.path.isfile(g_java_message_pickle_filename):
        with open(g_java_message_pickle_filename,'rb') as tfile:
            g_ok_java_messages = pickle.load(tfile)
    else:
        g_ok_java_messages["general"] = []

def load_java_messages_to_ignore ( ) : global g_ok_java_messages global g_java_message_pickle_filename if os . path . isfile ( g_java_message_pickle_filename ) : with open ( g_java_message_pickle_filename , 'rb' ) as tfile : g_ok_java_messages = pickle . load ( tfile ) else : g_ok_java_messages [ "general" ] = [ ]

Load in pickle file that contains dict structure with bad java messages to ignore per unit test or for all cases . The ignored bad java info is stored in g_ok_java_messages dict .


def normalize_enum_constant(s):
    """Return enum constant `s` converted to a canonical snake-case."""
    if s.islower(): return s
    if s.isupper(): return s.lower()
    return "".join(ch if ch.islower() else "_" + ch.lower() for ch in s).strip("_")

def normalize_enum_constant ( s ) : if s . islower ( ) : return s if s . isupper ( ) : return s . lower ( ) return "" . join ( ch if ch . islower ( ) else "_" + ch . lower ( ) for ch in s ) . strip ( "_" )

Return enum constant s converted to a canonical snake - case .


def find_synonyms(self, word, count=20):
        """
        Find synonyms using a word2vec model.

        :param str word: A single word to find synonyms for.
        :param int count: The first "count" synonyms will be returned.

        :returns: the approximate reconstruction of the training data.
        """
        j = h2o.api("GET /3/Word2VecSynonyms", data={'model': self.model_id, 'word': word, 'count': count})
        return OrderedDict(sorted(zip(j['synonyms'], j['scores']), key=lambda t: t[1], reverse=True))

def
find_synonyms
(
self
,
word
,
count
=
20
)
:
j
=
h2o
.
api
(
"GET /3/Word2VecSynonyms"
,
data
=
{
'model'
:
self
.
model_id
,
'word'
:
word
,
'count'
:
count
}
)
return
OrderedDict
(
sorted
(
zip
(
j
[
'synonyms'
]
,
j
[
'scores'
]
)
,
key
=
lambda
t
:
t
[
1
]
,
reverse
=
True
)
)


def poll(self, verbose_model_scoring_history = False):
        """
        Wait until the job finishes.

        This method will continuously query the server about the status of the job, until the job reaches a
        completion. During this time we will display (in stdout) a progress bar with % completion status.
        """
        try:
            hidden = not H2OJob.__PROGRESS_BAR__
            pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
            if verbose_model_scoring_history:
                pb.execute(self._refresh_job_status, print_verbose_info=lambda x: self._print_verbose_info() if int(x * 10) % 5 == 0  else " ")
            else:
                pb.execute(self._refresh_job_status)
        except StopIteration as e:
            if str(e) == "cancelled":
                h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
                self.status = "CANCELLED"
            # Potentially we may want to re-raise the exception here

        assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
            "Polling finished while the job has status %s" % self.status
        if self.warnings:
            for w in self.warnings:
                warnings.warn(w)

        # check if failed... and politely print relevant message
        if self.status == "CANCELLED":
            raise H2OJobCancelled("Job<%s> was cancelled by the user." % self.job_key)
        if self.status == "FAILED":
            if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
                raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
                                       "\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
            else:
                raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))

        return self


def to_pojo(self, pojo_name="", path="", get_jar=True):
        """
        Convert the munging operations performed on H2OFrame into a POJO.

        :param pojo_name:  (str) Name of POJO
        :param path:  (str) path of POJO.
        :param get_jar: (bool) Whether to also download the h2o-genmodel.jar file needed to compile the POJO
        :return: None
        """
        assert_is_type(pojo_name, str)
        assert_is_type(path, str)
        assert_is_type(get_jar, bool)
        if pojo_name == "":
            pojo_name = "AssemblyPOJO_" + str(uuid.uuid4())
        java = h2o.api("GET /99/Assembly.java/%s/%s" % (self.id, pojo_name))
        file_path = path + "/" + pojo_name + ".java"
        if path == "":
            print(java)
        else:
            with open(file_path, 'w', encoding="utf-8") as f:
                f.write(java)  # this had better be utf-8 ?
        if get_jar and path != "":
            h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))

def
to_pojo
(
self
,
pojo_name
=
""
,
path
=
""
,
get_jar
=
True
)
:
assert_is_type
(
pojo_name
,
str
)
assert_is_type
(
path
,
str
)
assert_is_type
(
get_jar
,
bool
)
if
pojo_name
==
""
:
pojo_name
=
"AssemblyPOJO_"
+
str
(
uuid
.
uuid4
(
)
)
java
=
h2o
.
api
(
"GET /99/Assembly.java/%s/%s"
%
(
self
.
id
,
pojo_name
)
)
file_path
=
path
+
"/"
+
pojo_name
+
".java"
if
path
==
""
:
print
(
java
)
else
:
with
open
(
file_path
,
'w'
,
encoding
=
"utf-8"
)
as
f
:
f
.
write
(
java
)
if
get_jar
and
path
!=
""
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
"h2o-genmodel.jar"
)
)

Convert the munging operations performed on H2OFrame into a POJO .


def fit(self, fr):
        """
        To perform the munging operations on a frame specified in steps on the frame fr.

        :param fr: H2OFrame where munging operations are to be performed on.
        :return: H2OFrame after munging operations are completed.
        """
        assert_is_type(fr, H2OFrame)
        steps = "[%s]" % ",".join(quoted(step[1].to_rest(step[0]).replace('"', "'")) for step in self.steps)
        j = h2o.api("POST /99/Assembly", data={"steps": steps, "frame": fr.frame_id})
        self.id = j["assembly"]["name"]
        return H2OFrame.get_frame(j["result"]["name"])

def
fit
(
self
,
fr
)
:
assert_is_type
(
fr
,
H2OFrame
)
steps
=
"[%s]"
%
","
.
join
(
quoted
(
step
[
1
]
.
to_rest
(
step
[
0
]
)
.
replace
(
'"'
,
"'"
)
)
for
step
in
self
.
steps
)
j
=
h2o
.
api
(
"POST /99/Assembly"
,
data
=
{
"steps"
:
steps
,
"frame"
:
fr
.
frame_id
}
)
self
.
id
=
j
[
"assembly"
]
[
"name"
]
return
H2OFrame
.
get_frame
(
j
[
"result"
]
[
"name"
]
)

To perform the munging operations on a frame specified in steps on the frame fr .


def percentileOnSortedList(N, percent, key=lambda x:x, interpolate='mean'):
    # 5 ways of resolving fractional
    # floor, ceil, funky, linear, mean
    interpolateChoices = ['floor', 'ceil', 'funky', 'linear', 'mean']
    if interpolate not in interpolateChoices:
        print "Bad choice for interpolate:", interpolate
        print "Supported choices:", interpolateChoices
    """
    Find the percentile of a list of values.

    @parameter N - is a list of values. Note N MUST BE already sorted.
    @parameter percent - a float value from 0.0 to 1.0.
    @parameter key - optional key function to compute value from each element of N.

    @return - the percentile of the values
    """
    if N is None:
        return None
    k = (len(N)-1) * percent
    f = int(math.floor(k))
    c = int(math.ceil(k))

    if f == c:
        d = key(N[f])
        msg = "aligned:"

    elif interpolate=='floor':
        d = key(N[f])
        msg = "fractional with floor:"

    elif interpolate=='ceil':
        d = key(N[c])
        msg = "fractional with ceil:"

    elif interpolate=='funky':
        d0 = key(N[f]) * (c-k)
        d1 = key(N[c]) * (k-f)
        d = d0+d1
        msg = "fractional with Tung(floor and ceil) :"

    elif interpolate=='linear':
        assert (c-f)==1
        assert (k>=f) and (k<=c)
        pctDiff = k-f
        dDiff = pctDiff * (key(N[c]) - key(N[f]))
        d = key(N[f] + dDiff)
        msg = "fractional %s with linear(floor and ceil):" % pctDiff

    elif interpolate=='mean':
        d = (key(N[c]) + key(N[f])) / 2.0
        msg = "fractional with mean(floor and ceil):"

    # print 3 around the floored k, for eyeballing when we're close
    flooredK = int(f)
    # print the 3 around the median
    if flooredK > 0:
        print "prior->", key(N[flooredK-1]), " "
    else:
        print "prior->", "<bof>"
    print "floor->", key(N[flooredK]), " ", msg, 'result:', d, "f:", f, "len(N):", len(N)
    if flooredK+1 < len(N):
        print " ceil->", key(N[flooredK+1]), "c:", c
    else:
        print " ceil-> <eof>", "c:", c

    return d

Find the percentile of a list of values .


def default_params(self):
        """Dictionary of the default parameters of the model."""
        params = {}
        for p in self.parms:
            params[p] = self.parms[p]["default_value"]
        return params

def default_params ( self ) : params = { } for p in self . parms : params [ p ] = self . parms [ p ] [ "default_value" ] return params

Dictionary of the default parameters of the model .


def actual_params(self):
        """Dictionary of actual parameters of the model."""
        params_to_select = {"model_id": "name",
                            "response_column": "column_name",
                            "training_frame": "name",
                            "validation_frame": "name"}
        params = {}
        for p in self.parms:
            if p in params_to_select.keys():
                params[p] = self.parms[p]["actual_value"].get(params_to_select[p], None)
            else:
                params[p] = self.parms[p]["actual_value"]
        return params

def actual_params ( self ) : params_to_select = { "model_id" : "name" , "response_column" : "column_name" , "training_frame" : "name" , "validation_frame" : "name" } params = { } for p in self . parms : if p in params_to_select . keys ( ) : params [ p ] = self . parms [ p ] [ "actual_value" ] . get ( params_to_select [ p ] , None ) else : params [ p ] = self . parms [ p ] [ "actual_value" ] return params

Dictionary of actual parameters of the model .


def deepfeatures(self, test_data, layer):
        """
        Return hidden layer details.

        :param test_data: Data to create a feature space on
        :param layer: 0 index hidden layer
        """
        if test_data is None: raise ValueError("Must specify test data")
        if str(layer).isdigit():
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer": layer}), "deepfeatures")
        else:
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer_name": layer}), "deepfeatures")
        j.poll()
        return h2o.get_frame(j.dest_key)


def scoring_history(self):
        """
        Retrieve Model Score History.

        :returns: The score history as an H2OTwoDimTable or a Pandas DataFrame.
        """
        model = self._model_json["output"]
        if "scoring_history" in model and model["scoring_history"] is not None:
            return model["scoring_history"].as_data_frame()
        print("No score history for this model")

def
scoring_history
(
self
)
:
model
=
self
.
_model_json
[
"output"
]
if
"scoring_history"
in
model
and
model
[
"scoring_history"
]
is
not
:
return
model
[
"scoring_history"
]
.
as_data_frame
(
)
print
(
"No score history for this model"
)


def show(self):
        """Print innards of model, without regards to type."""
        if self._future:
            self._job.poll_once()
            return
        if self._model_json is None:
            print("No model trained yet")
            return
        if self.model_id is None:
            print("This H2OEstimator has been removed.")
            return
        model = self._model_json["output"]
        print("Model Details")
        print("=============")

        print(self.__class__.__name__, ": ", self._model_json["algo_full_name"])
        print("Model Key: ", self._id)

        self.summary()

        print()
        # training metrics
        tm = model["training_metrics"]
        if tm: tm.show()
        vm = model["validation_metrics"]
        if vm: vm.show()
        xm = model["cross_validation_metrics"]
        if xm: xm.show()
        xms = model["cross_validation_metrics_summary"]
        if xms: xms.show()

        if "scoring_history" in model and model["scoring_history"]:
            model["scoring_history"].show()
        if "variable_importances" in model and model["variable_importances"]:
            model["variable_importances"].show()

def
show
(
self
)
:
if
self
.
_future
:
self
.
_job
.
poll_once
(
)
return
if
self
.
_model_json
is
:
print
(
"No model trained yet"
)
return
if
self
.
model_id
is
:
print
(
"This H2OEstimator has been removed."
)
return
model
=
self
.
_model_json
[
"output"
]
print
(
"Model Details"
)
print
(
"============="
)
print
(
self
.
__class__
.
__name__
,
": "
,
self
.
_model_json
[
"algo_full_name"
]
)
print
(
"Model Key: "
,
self
.
_id
)
self
.
summary
(
)
print
(
)
tm
=
model
[
"training_metrics"
]
if
tm
:
tm
.
show
(
)
vm
=
model
[
"validation_metrics"
]
if
vm
:
vm
.
show
(
)
xm
=
model
[
"cross_validation_metrics"
]
if
xm
:
xm
.
show
(
)
xms
=
model
[
"cross_validation_metrics_summary"
]
if
xms
:
xms
.
show
(
)
if
"scoring_history"
in
model
and
model
[
"scoring_history"
]
:
model
[
"scoring_history"
]
.
show
(
)
if
"variable_importances"
in
model
and
model
[
"variable_importances"
]
:
model
[
"variable_importances"
]
.
show
(
)

Print innards of model without regards to type .


def varimp(self, use_pandas=False):
        """
        Pretty print the variable importances, or return them in a list.

        :param use_pandas: If True, then the variable importances will be returned as a pandas data frame.

        :returns: A list or Pandas DataFrame.
        """
        model = self._model_json["output"]
        if self.algo=='glm' or "variable_importances" in list(model.keys()) and model["variable_importances"]:
            if self.algo=='glm':
                tempvals = model["standardized_coefficient_magnitudes"].cell_values
                maxVal = 0
                sum=0
                for item in tempvals:
                    sum=sum+item[1]
                    if item[1]>maxVal:
                        maxVal = item[1]
                vals = []
                for item in tempvals:
                    tempT = (item[0], item[1], item[1]/maxVal, item[1]/sum)
                    vals.append(tempT)
                header = ["variable", "relative_importance", "scaled_importance", "percentage"]
            else:
                vals = model["variable_importances"].cell_values
                header = model["variable_importances"].col_header

            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have variable importances")

Pretty print the variable importances or return them in a list .


def residual_degrees_of_freedom(self, train=False, valid=False, xval=False):
        """
        Retreive the residual degress of freedom if this model has the attribute, or None otherwise.

        :param bool train: Get the residual dof for the training set. If both train and valid are False, then train
            is selected by default.
        :param bool valid: Get the residual dof for the validation set. If both train and valid are True, then train
            is selected by default.

        :returns: Return the residual dof, or None if it is not present.
        """
        if xval: raise H2OValueError("Cross-validation metrics are not available.")
        if not train and not valid: train = True
        if train and valid:         train = True
        if train:
            return self._model_json["output"]["training_metrics"].residual_degrees_of_freedom()
        else:
            return self._model_json["output"]["validation_metrics"].residual_degrees_of_freedom()

Retreive the residual degress of freedom if this model has the attribute or None otherwise .


def coef(self):
        """
        Return the coefficients which can be applied to the non-standardized data.

        Note: standardize = True by default, if set to False then coef() return the coefficients which are fit directly.
        """
        tbl = self._model_json["output"]["coefficients_table"]
        if tbl is None:
            return None
        return {name: coef for name, coef in zip(tbl["names"], tbl["coefficients"])}

def coef ( self ) : tbl = self . _model_json [ "output" ] [ "coefficients_table" ] if tbl is None : return None return { name : coef for name , coef in zip ( tbl [ "names" ] , tbl [ "coefficients" ] ) }

Return the coefficients which can be applied to the non - standardized data .


def download_pojo(self, path="", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the POJO for this model to the directory specified by path.

        If path is an empty string, then dump the output to screen.

        :param path:  An absolute path to the directory where POJO should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the POJO file written.
        """
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)
        path = path.rstrip("/")
        return h2o.download_pojo(self, path, get_jar=get_genmodel_jar, jar_name=genmodel_name)

def download_pojo ( self , path = "" , get_genmodel_jar = False , genmodel_name = "" ) : assert_is_type ( path , str ) assert_is_type ( get_genmodel_jar , bool ) path = path . rstrip ( "/" ) return h2o . download_pojo ( self , path , get_jar = get_genmodel_jar , jar_name = genmodel_name )

Download the POJO for this model to the directory specified by path .


def download_mojo(self, path=".", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the model in MOJO format.

        :param path: the path where MOJO file should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the MOJO file written.
        """
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)

        if not self.have_mojo:
            raise H2OValueError("Export to MOJO not supported")

        if get_genmodel_jar:
            if genmodel_name == "":
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))
            else:
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, genmodel_name))
        return h2o.api("GET /3/Models/%s/mojo" % self.model_id, save_to=path)

def
download_mojo
(
self
,
path
=
"."
,
get_genmodel_jar
=
False
,
genmodel_name
=
""
)
:
assert_is_type
(
path
,
str
)
assert_is_type
(
get_genmodel_jar
,
bool
)
if
not
self
.
have_mojo
:
raise
H2OValueError
(
"Export to MOJO not supported"
)
if
get_genmodel_jar
:
if
genmodel_name
==
""
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
"h2o-genmodel.jar"
)
)
else
:
h2o
.
api
(
"GET /3/h2o-genmodel.jar"
,
save_to
=
os
.
path
.
join
(
path
,
genmodel_name
)
)
return
h2o
.
api
(
"GET /3/Models/%s/mojo"
%
self
.
model_id
,
save_to
=
path
)


def save_model_details(self, path="", force=False):
        """
        Save Model Details of an H2O Model in JSON Format to disk.

        :param model: The model object to save.
        :param path: a path to save the model details at (hdfs, s3, local)
        :param force: if True overwrite destination directory in case it exists, or throw exception if set to False.

        :returns str: the path of the saved model details
        """
        assert_is_type(path, str)
        assert_is_type(force, bool)
        path = os.path.join(os.getcwd() if path == "" else path, self.model_id + ".json")
        return h2o.api("GET /99/Models/%s/json" % self.model_id, data={"dir": path, "force": force})["dir"]

def
save_model_details
(
self
,
path
=
""
,
force
=
False
)
:
assert_is_type
(
path
,
str
)
assert_is_type
(
force
,
bool
)
path
=
os
.
path
.
join
(
os
.
getcwd
(
)
if
path
==
""
else
path
,
self
.
model_id
+
".json"
)
return
h2o
.
api
(
"GET /99/Models/%s/json"
%
self
.
model_id
,
data
=
{
"dir"
:
path
,
"force"
:
force
}
)
[
"dir"
]

Save Model Details of an H2O Model in JSON Format to disk .


def _check_targets(y_actual, y_predicted):
        """Check that y_actual and y_predicted have the same length.

        :param H2OFrame y_actual:
        :param H2OFrame y_predicted:

        :returns: None
        """
        if len(y_actual) != len(y_predicted):
            raise ValueError("Row mismatch: [{},{}]".format(len(y_actual), len(y_predicted)))

def
_check_targets
(
y_actual
,
y_predicted
)
:
if
len
(
y_actual
)
!=
len
(
y_predicted
)
:
raise
ValueError
(
"Row mismatch: [{},{}]"
.
format
(
len
(
y_actual
)
,
len
(
y_predicted
)
)
)

Check that y_actual and y_predicted have the same length .


def cross_validation_models(self):
        """
        Obtain a list of cross-validation models.

        :returns: list of H2OModel objects.
        """
        cvmodels = self._model_json["output"]["cross_validation_models"]
        if cvmodels is None: return None
        m = []
        for p in cvmodels: m.append(h2o.get_model(p["name"]))
        return m

def cross_validation_models ( self ) : cvmodels = self . _model_json [ "output" ] [ "cross_validation_models" ] if cvmodels is None : return None m = [ ] for p in cvmodels : m . append ( h2o . get_model ( p [ "name" ] ) ) return m

Obtain a list of cross - validation models .


def gbm(interactive=True, echo=True, testing=False):
    """GBM model demo."""

    def demo_body(go):
        """
        Demo of H2O's Gradient Boosting estimator.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2OGradientBoostingEstimator
        prostate_gbm = H2OGradientBoostingEstimator(distribution="bernoulli", ntrees=10, max_depth=8,
                                                    min_rows=10, learn_rate=0.2)
        prostate_gbm.train(x=["AGE", "RACE", "PSA", "VOL", "GLEASON"],
                           y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_gbm.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_gbm.predict(test)
        predictions.show()

        go()
        # Fetch a tree, print number of tree nodes, show root node description
        from h2o.tree import H2OTree, H2ONode
        tree = H2OTree(prostate_gbm, 0, "0")
        len(tree)
        tree.left_children
        tree.right_children
        tree.root_node.show()

        go()
        # Show default performance metrics
        performance = prostate_gbm.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)

def gbm ( interactive = True , echo = True , testing = False ) : def demo_body ( go ) : go ( ) h2o . init ( ) go ( ) prostate = h2o . load_dataset ( "prostate" ) go ( ) prostate . describe ( ) go ( ) train , test = prostate . split_frame ( ratios = [ 0.70 ] ) go ( ) train [ "CAPSULE" ] = train [ "CAPSULE" ] . asfactor ( ) test [ "CAPSULE" ] = test [ "CAPSULE" ] . asfactor ( ) go ( ) from h2o . estimators import H2OGradientBoostingEstimator prostate_gbm = H2OGradientBoostingEstimator ( distribution = "bernoulli" , ntrees = 10 , max_depth = 8 , min_rows = 10 , learn_rate = 0.2 ) prostate_gbm . train ( x = [ "AGE" , "RACE" , "PSA" , "VOL" , "GLEASON" ] , y = "CAPSULE" , training_frame = train ) go ( ) prostate_gbm . show ( ) go ( ) predictions = prostate_gbm . predict ( test ) predictions . show ( ) go ( ) from h2o . tree import H2OTree , H2ONode tree = H2OTree ( prostate_gbm , 0 , "0" ) len ( tree ) tree . left_children tree . right_children tree . root_node . show ( ) go ( ) performance = prostate_gbm . model_performance ( test ) performance . show ( ) _run_demo ( demo_body , interactive , echo , testing )


def deeplearning(interactive=True, echo=True, testing=False):
    """Deep Learning model demo."""

    def demo_body(go):
        """
        Demo of H2O's Deep Learning model.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2ODeepLearningEstimator
        prostate_dl = H2ODeepLearningEstimator(activation="Tanh", hidden=[10, 10, 10], epochs=10000)
        prostate_dl.train(x=list(set(prostate.col_names) - {"ID", "CAPSULE"}),
                          y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_dl.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_dl.predict(test)
        predictions.show()

        go()
        # Show default performance metrics
        performance = prostate_dl.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)

def deeplearning ( interactive = True , echo = True , testing = False ) : def demo_body ( go ) : go ( ) h2o . init ( ) go ( ) prostate = h2o . load_dataset ( "prostate" ) go ( ) prostate . describe ( ) go ( ) train , test = prostate . split_frame ( ratios = [ 0.70 ] ) go ( ) train [ "CAPSULE" ] = train [ "CAPSULE" ] . asfactor ( ) test [ "CAPSULE" ] = test [ "CAPSULE" ] . asfactor ( ) go ( ) from h2o . estimators import H2ODeepLearningEstimator prostate_dl = H2ODeepLearningEstimator ( activation = "Tanh" , hidden = [ 10 , 10 , 10 ] , epochs = 10000 ) prostate_dl . train ( x = list ( set ( prostate . col_names ) - { "ID" , "CAPSULE" } ) , y = "CAPSULE" , training_frame = train ) go ( ) prostate_dl . show ( ) go ( ) predictions = prostate_dl . predict ( test ) predictions . show ( ) go ( ) performance = prostate_dl . model_performance ( test ) performance . show ( ) _run_demo ( demo_body , interactive , echo , testing )


def glm(interactive=True, echo=True, testing=False):
    """GLM model demo."""

    def demo_body(go):
        """
        Demo of H2O's Generalized Linear Estimator.

        This demo uploads a dataset to h2o, parses it, and shows a description.
        Then it divides the dataset into training and test sets, builds a GLM
        from the training set, and makes predictions for the test set.
        Finally, default performance metrics are displayed.
        """
        go()
        # Connect to H2O
        h2o.init()

        go()
        # Upload the prostate dataset that comes included in the h2o python package
        prostate = h2o.load_dataset("prostate")

        go()
        # Print a description of the prostate data
        prostate.describe()

        go()
        # Randomly split the dataset into ~70/30, training/test sets
        train, test = prostate.split_frame(ratios=[0.70])

        go()
        # Convert the response columns to factors (for binary classification problems)
        train["CAPSULE"] = train["CAPSULE"].asfactor()
        test["CAPSULE"] = test["CAPSULE"].asfactor()

        go()
        # Build a (classification) GLM
        from h2o.estimators import H2OGeneralizedLinearEstimator
        prostate_glm = H2OGeneralizedLinearEstimator(family="binomial", alpha=[0.5])
        prostate_glm.train(x=["AGE", "RACE", "PSA", "VOL", "GLEASON"],
                           y="CAPSULE", training_frame=train)

        go()
        # Show the model
        prostate_glm.show()

        go()
        # Predict on the test set and show the first ten predictions
        predictions = prostate_glm.predict(test)
        predictions.show()

        go()
        # Show default performance metrics
        performance = prostate_glm.model_performance(test)
        performance.show()

    # Execute:
    _run_demo(demo_body, interactive, echo, testing)

def glm ( interactive = True , echo = True , testing = False ) : def demo_body ( go ) : go ( ) h2o . init ( ) go ( ) prostate = h2o . load_dataset ( "prostate" ) go ( ) prostate . describe ( ) go ( ) train , test = prostate . split_frame ( ratios = [ 0.70 ] ) go ( ) train [ "CAPSULE" ] = train [ "CAPSULE" ] . asfactor ( ) test [ "CAPSULE" ] = test [ "CAPSULE" ] . asfactor ( ) go ( ) from h2o . estimators import H2OGeneralizedLinearEstimator prostate_glm = H2OGeneralizedLinearEstimator ( family = "binomial" , alpha = [ 0.5 ] ) prostate_glm . train ( x = [ "AGE" , "RACE" , "PSA" , "VOL" , "GLEASON" ] , y = "CAPSULE" , training_frame = train ) go ( ) prostate_glm . show ( ) go ( ) predictions = prostate_glm . predict ( test ) predictions . show ( ) go ( ) performance = prostate_glm . model_performance ( test ) performance . show ( ) _run_demo ( demo_body , interactive , echo , testing )


def _wait_for_keypress():
    """
    Wait for a key press on the console and return it.

    Borrowed from http://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key
    """
    result = None
    if os.name == "nt":
        # noinspection PyUnresolvedReferences
        import msvcrt
        result = msvcrt.getch()
    else:
        import termios
        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        try:
            result = sys.stdin.read(1)
        except IOError:
            pass
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)

    return result

def _wait_for_keypress ( ) : result = None if os . name == "nt" : import msvcrt result = msvcrt . getch ( ) else : import termios fd = sys . stdin . fileno ( ) oldterm = termios . tcgetattr ( fd ) newattr = termios . tcgetattr ( fd ) newattr [ 3 ] = newattr [ 3 ] & ~ termios . ICANON & ~ termios . ECHO termios . tcsetattr ( fd , termios . TCSANOW , newattr ) try : result = sys . stdin . read ( 1 ) except IOError : pass finally : termios . tcsetattr ( fd , termios . TCSAFLUSH , oldterm ) return result

Wait for a key press on the console and return it .


def as_data_frame(self):
        """Convert to a python 'data frame'."""
        if can_use_pandas():
            import pandas
            pandas.options.display.max_colwidth = 70
            return pandas.DataFrame(self._cell_values, columns=self._col_header)
        return self

def as_data_frame ( self ) : if can_use_pandas ( ) : import pandas pandas . options . display . max_colwidth = 70 return pandas . DataFrame ( self . _cell_values , columns = self . _col_header ) return self


def show(self, header=True):
        """Print the contents of this table."""
        # if h2o.can_use_pandas():
        #  import pandas
        #  pandas.options.display.max_rows = 20
        #  print pandas.DataFrame(self._cell_values,columns=self._col_header)
        #  return
        if header and self._table_header:
            print(self._table_header + ":", end=' ')
            if self._table_description: print(self._table_description)
        print()
        table = copy.deepcopy(self._cell_values)
        nr = 0
        if _is_list_of_lists(table): nr = len(
            table)  # only set if we truly have multiple rows... not just one long row :)
        if nr > 20:  # create a truncated view of the table, first/last 5 rows
            trunc_table = []
            trunc_table += [v for v in table[:5]]
            trunc_table.append(["---"] * len(table[0]))
            trunc_table += [v for v in table[(nr - 5):]]
            table = trunc_table

        H2ODisplay(table, self._col_header, numalign="left", stralign="left")
        if nr > 20 and can_use_pandas(): print('\nSee the whole table with table.as_data_frame()')


def start(jar_path=None, nthreads=-1, enable_assertions=True, max_mem_size=None, min_mem_size=None,
              ice_root=None, log_dir=None, log_level=None, port="54321+", name=None, extra_classpath=None,
              verbose=True, jvm_custom_args=None, bind_to_localhost=True):
        """
        Start new H2O server on the local machine.

        :param jar_path: Path to the h2o.jar executable. If not given, then we will search for h2o.jar in the
            locations returned by `._jar_paths()`.
        :param nthreads: Number of threads in the thread pool. This should be related to the number of CPUs used.
            -1 means use all CPUs on the host. A positive integer specifies the number of CPUs directly.
        :param enable_assertions: If True, pass `-ea` option to the JVM.
        :param max_mem_size: Maximum heap size (jvm option Xmx), in bytes.
        :param min_mem_size: Minimum heap size (jvm option Xms), in bytes.
        :param log_dir: Directory for H2O logs to be stored if a new instance is started. Default directory is determined
        by H2O internally.
        :param log_level: The logger level for H2O if a new instance is started.
        :param ice_root: A directory where H2O stores its temporary files. Default location is determined by
            tempfile.mkdtemp().
        :param port: Port where to start the new server. This could be either an integer, or a string of the form
            "DDDDD+", indicating that the server should start looking for an open port starting from DDDDD and up.
        :param name: name of the h2o cluster to be started
        :param extra_classpath List of paths to libraries that should be included on the Java classpath.
        :param verbose: If True, then connection info will be printed to the stdout.
        :param jvm_custom_args Custom, user-defined arguments for the JVM H2O is instantiated in
        :param bind_to_localhost A flag indicating whether access to the H2O instance should be restricted to the local
            machine (default) or if it can be reached from other computers on the network.
            Only applicable when H2O is started from the Python client.

        :returns: a new H2OLocalServer instance
        """
        assert_is_type(jar_path, None, str)
        assert_is_type(port, None, int, str)
        assert_is_type(name, None, str)
        assert_is_type(nthreads, -1, BoundInt(1, 4096))
        assert_is_type(enable_assertions, bool)
        assert_is_type(min_mem_size, None, int)
        assert_is_type(max_mem_size, None, BoundInt(1 << 25))
        assert_is_type(log_dir, str, None)
        assert_is_type(log_level, str, None)
        assert_satisfies(log_level, log_level in [None, "TRACE", "DEBUG", "INFO", "WARN", "ERRR", "FATA"])
        assert_is_type(ice_root, None, I(str, os.path.isdir))
        assert_is_type(extra_classpath, None, [str])
        assert_is_type(jvm_custom_args, list, None)
        assert_is_type(bind_to_localhost, bool)
        if jar_path:
            assert_satisfies(jar_path, jar_path.endswith("h2o.jar"))

        if min_mem_size is not None and max_mem_size is not None and min_mem_size > max_mem_size:
            raise H2OValueError("`min_mem_size`=%d is larger than the `max_mem_size`=%d" % (min_mem_size, max_mem_size))
        if port is None: port = "54321+"
        baseport = None
        # TODO: get rid of this port gimmick and have 2 separate parameters.
        if is_type(port, str):
            if port.isdigit():
                port = int(port)
            else:
                if not(port[-1] == "+" and port[:-1].isdigit()):
                    raise H2OValueError("`port` should be of the form 'DDDD+', where D is a digit. Got: %s" % port)
                baseport = int(port[:-1])
                port = 0

        hs = H2OLocalServer()
        hs._verbose = bool(verbose)
        hs._jar_path = hs._find_jar(jar_path)
        hs._extra_classpath = extra_classpath
        hs._ice_root = ice_root
        hs._name = name
        if not ice_root:
            hs._ice_root = tempfile.mkdtemp()
            hs._tempdir = hs._ice_root

        if verbose: print("Attempting to start a local H2O server...")
        hs._launch_server(port=port, baseport=baseport, nthreads=int(nthreads), ea=enable_assertions,
                          mmax=max_mem_size, mmin=min_mem_size, jvm_custom_args=jvm_custom_args,
                          bind_to_localhost=bind_to_localhost, log_dir=log_dir, log_level=log_level)
        if verbose: print("  Server is running at %s://%s:%d" % (hs.scheme, hs.ip, hs.port))
        atexit.register(lambda: hs.shutdown())
        return hs

Start new H2O server on the local machine .


def _find_jar(self, path0=None):
        """
        Return the location of an h2o.jar executable.

        :param path0: Explicitly given h2o.jar path. If provided, then we will simply check whether the file is there,
            otherwise we will search for an executable in locations returned by ._jar_paths().

        :raises H2OStartupError: if no h2o.jar executable can be found.
        """
        jar_paths = [path0] if path0 else self._jar_paths()
        searched_paths = []
        for jp in jar_paths:
            searched_paths.append(jp)
            if os.path.exists(jp):
                return jp
        raise H2OStartupError("Cannot start local server: h2o.jar not found. Paths searched:\n" +
                              "".join("    %s\n" % s for s in searched_paths))

Return the location of an h2o . jar executable .


def _jar_paths():
        """Produce potential paths for an h2o.jar executable."""

        # PUBDEV-3534 hook to use arbitrary h2o.jar
        own_jar = os.getenv("H2O_JAR_PATH", "")
        if own_jar != "":
            if not os.path.isfile(own_jar):
                raise H2OStartupError("Environment variable H2O_JAR_PATH is set to '%d' but file does not exists, unset environment variable or provide valid path to h2o.jar file." % own_jar)
            yield own_jar

        # Check if running from an h2o-3 src folder (or any subfolder), in which case use the freshly-built h2o.jar
        cwd_chunks = os.path.abspath(".").split(os.path.sep)
        for i in range(len(cwd_chunks), 0, -1):
            if cwd_chunks[i - 1] == "h2o-3":
                yield os.path.sep.join(cwd_chunks[:i] + ["build", "h2o.jar"])
        # Then check the backend/bin folder:
        # (the following works assuming this code is located in h2o/backend/server.py file)
        backend_dir = os.path.split(os.path.realpath(__file__))[0]
        yield os.path.join(backend_dir, "bin", "h2o.jar")

        # Then try several old locations where h2o.jar might have been installed
        prefix1 = prefix2 = sys.prefix
        # On Unix-like systems Python typically gets installed into /Library/... or /System/Library/... If one of
        # those paths is sys.prefix, then we also build its counterpart.
        if prefix1.startswith(os.path.sep + "Library"):
            prefix2 = os.path.join("", "System", prefix1)
        elif prefix1.startswith(os.path.sep + "System"):
            prefix2 = prefix1[len(os.path.join("", "System")):]
        yield os.path.join(prefix1, "h2o_jar", "h2o.jar")
        yield os.path.join(os.path.abspath(os.sep), "usr", "local", "h2o_jar", "h2o.jar")
        yield os.path.join(prefix1, "local", "h2o_jar", "h2o.jar")
        yield os.path.join(get_config_var("userbase"), "h2o_jar", "h2o.jar")
        yield os.path.join(prefix2, "h2o_jar", "h2o.jar")

Produce potential paths for an h2o . jar executable .


def hit_ratio_table(self, train=False, valid=False, xval=False):
        """
        Retrieve the Hit Ratios.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param train: If train is True, then return the hit ratio value for the training data.
        :param valid: If valid is True, then return the hit ratio value for the validation data.
        :param xval:  If xval is True, then return the hit ratio value for the cross validation data.
        :return: The hit ratio for this regression model.
        """
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in zip(list(tm.keys()), list(tm.values())): m[k] = None if v is None else v.hit_ratio_table()
        return list(m.values())[0] if len(m) == 1 else m

def hit_ratio_table ( self , train = False , valid = False , xval = False ) : tm = ModelBase . _get_metrics ( self , train , valid , xval ) m = { } for k , v in zip ( list ( tm . keys ( ) ) , list ( tm . values ( ) ) ) : m [ k ] = None if v is None else v . hit_ratio_table ( ) return list ( m . values ( ) ) [ 0 ] if len ( m ) == 1 else m


def csv_dict_writer(f, fieldnames, **kwargs):
    """Equivalent of csv.DictWriter, but allows `delimiter` to be a unicode string on Py2."""
    import csv
    if "delimiter" in kwargs:
        kwargs["delimiter"] = str(kwargs["delimiter"])
    return csv.DictWriter(f, fieldnames, **kwargs)

def csv_dict_writer ( f , fieldnames , ** kwargs ) : import csv if "delimiter" in kwargs : kwargs [ "delimiter" ] = str ( kwargs [ "delimiter" ] ) return csv . DictWriter ( f , fieldnames , ** kwargs )

Equivalent of csv . DictWriter but allows delimiter to be a unicode string on Py2 .