def fit(self, X, y=None, **params):
        """
        Fit an H2O model as part of a scikit-learn pipeline or grid search.

        A warning will be issued if a caller other than sklearn attempts to use this method.

        :param H2OFrame X: An H2OFrame consisting of the predictor variables.
        :param H2OFrame y: An H2OFrame consisting of the response variable.
        :param params: Extra arguments.
        :returns: The current instance of H2OEstimator for method chaining.
        """
        stk = inspect.stack()[1:]
        warn = True
        for s in stk:
            mod = inspect.getmodule(s[0])
            if mod:
                warn = "sklearn" not in mod.__name__
                if not warn: break
        if warn:
            warnings.warn("\n\n\t`fit` is not recommended outside of the sklearn framework. Use `train` instead.",
                          UserWarning, stacklevel=2)
        training_frame = X.cbind(y) if y is not None else X
        x = X.names
        y = y.names[0] if y is not None else None
        self.train(x, y, training_frame, **params)
        return self

Fit an H2O model as part of a scikit - learn pipeline or grid search .


def get_params(self, deep=True):
        """
        Obtain parameters for this estimator.

        Used primarily for sklearn Pipelines and sklearn grid search.

        :param deep: If True, return parameters of all sub-objects that are estimators.

        :returns: A dict of parameters
        """
        out = dict()
        for key, value in self.parms.items():
            if deep and isinstance(value, H2OEstimator):
                deep_items = list(value.get_params().items())
                out.update((key + "__" + k, val) for k, val in deep_items)
            out[key] = value
        return out

def get_params ( self , deep = True ) : out = dict ( ) for key , value in self . parms . items ( ) : if deep and isinstance ( value , H2OEstimator ) : deep_items = list ( value . get_params ( ) . items ( ) ) out . update ( ( key + "__" + k , val ) for k , val in deep_items ) out [ key ] = value return out


def signal_handler(signum, stackframe):
    """Helper function to handle caught signals."""
    global g_runner
    global g_handling_signal

    if g_handling_signal:
        # Don't do this recursively.
        return
    g_handling_signal = True

    print("")
    print("----------------------------------------------------------------------")
    print("")
    print("SIGNAL CAUGHT (" + str(signum) + ").  TEARING DOWN CLOUDS.")
    print("")
    print("----------------------------------------------------------------------")
    g_runner.terminate()

Helper function to handle caught signals .


def wipe_output_dir():
    """Clear the output directory."""
    print("Wiping output directory.")
    try:
        if os.path.exists(g_output_dir):
            shutil.rmtree(str(g_output_dir))
    except OSError as e:
        print("ERROR: Removing output directory %s failed: " % g_output_dir)
        print("       (errno {0}): {1}".format(e.errno, e.strerror))
        print("")
        sys.exit(1)


def remove_sandbox(parent_dir, dir_name):
    """
    This function is written to remove sandbox directories if they exist under the
    parent_dir.

    :param parent_dir: string denoting full parent directory path
    :param dir_name: string denoting directory path which could be a sandbox
    :return: None
    """
    if "Rsandbox" in dir_name:
        rsandbox_dir = os.path.join(parent_dir, dir_name)
        try:
            if sys.platform == "win32":
                os.system(r'C:/cygwin64/bin/rm.exe -r -f "{0}"'.format(rsandbox_dir))
            else:
                shutil.rmtree(rsandbox_dir)
        except OSError as e:
            print("")
            print("ERROR: Removing RSandbox directory failed: " + rsandbox_dir)
            print("       (errno {0}): {1}".format(e.errno, e.strerror))
            print("")
            sys.exit(1)

This function is written to remove sandbox directories if they exist under the parent_dir .


def scrape_port_from_stdout(self):
        """
        Look at the stdout log and figure out which port the JVM chose.

        If successful, port number is stored in self.port; otherwise the
        program is terminated. This call is blocking, and will wait for
        up to 30s for the server to start up.
        """
        regex = re.compile(r"Open H2O Flow in your web browser: https?://([^:]+):(\d+)")
        retries_left = 30
        while retries_left and not self.terminated:
            with open(self.output_file_name, "r") as f:
                for line in f:
                    mm = re.search(regex, line)
                    if mm is not None:
                        self.port = mm.group(2)
                        print("H2O cloud %d node %d listening on port %s\n    with output file %s" %
                              (self.cloud_num, self.node_num, self.port, self.output_file_name))
                        return
            if self.terminated: break
            retries_left -= 1
            time.sleep(1)

        if self.terminated: return
        print("\nERROR: Too many retries starting cloud %d.\nCheck the output log %s.\n" %
              (self.cloud_num, self.output_file_name))
        sys.exit(1)

Look at the stdout log and figure out which port the JVM chose .


def scrape_cloudsize_from_stdout(self, nodes_per_cloud):
        """
        Look at the stdout log and wait until the cluster of proper size is formed.
        This call is blocking.
        Exit if this fails.

        :param nodes_per_cloud:
        :return none
        """
        retries = 60
        while retries > 0:
            if self.terminated: return
            f = open(self.output_file_name, "r")
            s = f.readline()
            while len(s) > 0:
                if self.terminated: return
                match_groups = re.search(r"Cloud of size (\d+) formed", s)
                if match_groups is not None:
                    size = match_groups.group(1)
                    if size is not None:
                        size = int(size)
                        if size == nodes_per_cloud:
                            f.close()
                            return

                s = f.readline()

            f.close()
            retries -= 1
            if self.terminated: return
            time.sleep(1)

        print("")
        print("ERROR: Too many retries starting cloud.")
        print("")
        sys.exit(1)

Look at the stdout log and wait until the cluster of proper size is formed . This call is blocking . Exit if this fails .


def stop(self):
        """
        Normal node shutdown.
        Ignore failures for now.

        :return none
        """
        if self.pid > 0:
            print("Killing JVM with PID {}".format(self.pid))
            try:
                self.child.terminate()
                self.child.wait()
            except OSError:
                pass
            self.pid = -1

def
stop
(
self
)
:
if
self
.
pid
>
0
:
print
(
"Killing JVM with PID {}"
.
format
(
self
.
pid
)
)
try
:
self
.
child
.
terminate
(
)
self
.
child
.
wait
(
)
except
OSError
:
pass
self
.
pid
=
1

Normal node shutdown . Ignore failures for now .


def stop(self):
        """
        Normal cluster shutdown.

        :return none
        """
        for node in self.nodes:
            node.stop()

        for node in self.client_nodes:
            node.stop()

def stop ( self ) : for node in self . nodes : node . stop ( ) for node in self . client_nodes : node . stop ( )

Normal cluster shutdown.

        :return none


def get_ip(self):
        """ Return an ip to use to talk to this cluster. """
        if len(self.client_nodes) > 0:
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_ip()

def get_ip ( self ) : if len ( self . client_nodes ) > 0 : node = self . client_nodes [ 0 ] else : node = self . nodes [ 0 ] return node . get_ip ( )

Return an ip to use to talk to this cluster .


def get_port(self):
        """ Return a port to use to talk to this cluster. """
        if len(self.client_nodes) > 0:
            node = self.client_nodes[0]
        else:
            node = self.nodes[0]
        return node.get_port()

def get_port ( self ) : if len ( self . client_nodes ) > 0 : node = self . client_nodes [ 0 ] else : node = self . nodes [ 0 ] return node . get_port ( )

Return a port to use to talk to this cluster .


def roc(self, train=False, valid=False, xval=False):
        """
        Return the coordinates of the ROC curve for a given set of data.

        The coordinates are two-tuples containing the false positive rates as a list and true positive rates as a list.
        If all are False (default), then return is the training data. If more than one ROC
        curve is requested, the data is returned as a dictionary of two-tuples.

        :param bool train: If True, return the ROC value for the training data.
        :param bool valid: If True, return the ROC value for the validation data.
        :param bool xval: If True, return the ROC value for each of the cross-validated splits.

        :returns: The ROC values for the specified key(s).
        """
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in viewitems(tm):

            if v is not None:
                m[k] = (v.fprs, v.tprs)
        return list(m.values())[0] if len(m) == 1 else m

def roc ( self , train = False , valid = False , xval = False ) : tm = ModelBase . _get_metrics ( self , train , valid , xval ) m = { } for k , v in viewitems ( tm ) : if v is not None : m [ k ] = ( v . fprs , v . tprs ) return list ( m . values ( ) ) [ 0 ] if len ( m ) == 1 else m

Return the coordinates of the ROC curve for a given set of data .


def _determine_vec_size(self):
        """
        Determines vec_size for a pre-trained model after basic model verification.
        """
        first_column = self.pre_trained.types[self.pre_trained.columns[0]]

        if first_column != 'string':
            raise H2OValueError("First column of given pre_trained model %s is required to be a String",
                                self.pre_trained.frame_id)

        if list(self.pre_trained.types.values()).count('string') > 1:
            raise H2OValueError("There are multiple columns in given pre_trained model %s with a String type.",
                                self.pre_trained.frame_id)

        self.vec_size = self.pre_trained.dim[1] - 1;

Determines vec_size for a pre - trained model after basic model verification .


def h2o_mean_absolute_error(y_actual, y_predicted, weights=None):
    """
    Mean absolute error regression loss.

    :param y_actual: H2OFrame of actual response.
    :param y_predicted: H2OFrame of predicted response.
    :param weights: (Optional) sample weights
    :returns: mean absolute error loss (best is 0.0).
    """
    ModelBase._check_targets(y_actual, y_predicted)
    return _colmean((y_predicted - y_actual).abs())

def h2o_mean_absolute_error ( y_actual , y_predicted , weights = None ) : ModelBase . _check_targets ( y_actual , y_predicted ) return _colmean ( ( y_predicted - y_actual ) . abs ( ) )


def h2o_mean_squared_error(y_actual, y_predicted, weights=None):
    """
    Mean squared error regression loss

    :param y_actual: H2OFrame of actual response.
    :param y_predicted: H2OFrame of predicted response.
    :param weights: (Optional) sample weights
    :returns: mean squared error loss (best is 0.0).
    """
    ModelBase._check_targets(y_actual, y_predicted)
    return _colmean((y_predicted - y_actual) ** 2)


def h2o_median_absolute_error(y_actual, y_predicted):
    """
    Median absolute error regression loss

    :param y_actual: H2OFrame of actual response.
    :param y_predicted: H2OFrame of predicted response.
    :returns: median absolute error loss (best is 0.0)
    """
    ModelBase._check_targets(y_actual, y_predicted)
    return (y_predicted - y_actual).abs().median()

def h2o_median_absolute_error ( y_actual , y_predicted ) : ModelBase . _check_targets ( y_actual , y_predicted ) return ( y_predicted - y_actual ) . abs ( ) . median ( )


def h2o_explained_variance_score(y_actual, y_predicted, weights=None):
    """
    Explained variance regression score function.

    :param y_actual: H2OFrame of actual response.
    :param y_predicted: H2OFrame of predicted response.
    :param weights: (Optional) sample weights
    :returns: the explained variance score.
    """
    ModelBase._check_targets(y_actual, y_predicted)

    _, numerator = _mean_var(y_actual - y_predicted, weights)
    _, denominator = _mean_var(y_actual, weights)
    if denominator == 0.0:
        return 1. if numerator == 0 else 0.  # 0/0 => 1, otherwise, 0
    return 1 - numerator / denominator

def h2o_explained_variance_score ( y_actual , y_predicted , weights = None ) : ModelBase . _check_targets ( y_actual , y_predicted ) _ , numerator = _mean_var ( y_actual - y_predicted , weights ) _ , denominator = _mean_var ( y_actual , weights ) if denominator == 0.0 : return 1. if numerator == 0 else 0. return 1 - numerator / denominator

Explained variance regression score function .


def assert_is_type(var, *types, **kwargs):
    """
    Assert that the argument has the specified type.

    This function is used to check that the type of the argument is correct, otherwises it raises an H2OTypeError.
    See more details in the module's help.

    :param var: variable to check
    :param types: the expected types
    :param kwargs:
        message: override the error message
        skip_frames: how many local frames to skip when printing out the error.

    :raises H2OTypeError: if the argument is not of the desired type.
    """
    assert types, "The list of expected types was not provided"
    expected_type = types[0] if len(types) == 1 else U(*types)
    if _check_type(var, expected_type): return

    # Type check failed => Create a nice error message
    assert set(kwargs).issubset({"message", "skip_frames"}), "Unexpected keyword arguments: %r" % kwargs
    message = kwargs.get("message", None)
    skip_frames = kwargs.get("skip_frames", 1)
    args = _retrieve_assert_arguments()
    vname = args[0]
    etn = _get_type_name(expected_type, dump=", ".join(args[1:]))
    vtn = _get_type_name(type(var))
    raise H2OTypeError(var_name=vname, var_value=var, var_type_name=vtn, exp_type_name=etn, message=message,
                       skip_frames=skip_frames)

Assert that the argument has the specified type .


def assert_matches(v, regex):
    """
    Assert that string variable matches the provided regular expression.

    :param v: variable to check.
    :param regex: regular expression to check against (can be either a string, or compiled regexp).
    """
    m = re.match(regex, v)
    if m is None:
        vn = _retrieve_assert_arguments()[0]
        message = "Argument `{var}` (= {val!r}) did not match /{regex}/".format(var=vn, regex=regex, val=v)
        raise H2OValueError(message, var_name=vn, skip_frames=1)
    return m

Assert that string variable matches the provided regular expression .


def assert_satisfies(v, cond, message=None):
    """
    Assert that variable satisfies the provided condition.

    :param v: variable to check. Its value is only used for error reporting.
    :param bool cond: condition that must be satisfied. Should be somehow related to the variable ``v``.
    :param message: message string to use instead of the default.
    """
    if not cond:
        vname, vexpr = _retrieve_assert_arguments()
        if not message:
            message = "Argument `{var}` (= {val!r}) does not satisfy the condition {expr}" \
                      .format(var=vname, val=v, expr=vexpr)
        raise H2OValueError(message=message, var_name=vname, skip_frames=1)

Assert that variable satisfies the provided condition .


def _retrieve_assert_arguments():
    """
    Magic variable name retrieval.

    This function is designed as a helper for assert_is_type() function. Typically such assertion is used like this::

        assert_is_type(num_threads, int)

    If the variable `num_threads` turns out to be non-integer, we would like to raise an exception such as

        H2OTypeError("`num_threads` is expected to be integer, but got <str>")

    and in order to compose an error message like that, we need to know that the variables that was passed to
    assert_is_type() carries a name "num_threads". Naturally, the variable itself knows nothing about that.

    This is where this function comes in: we walk up the stack trace until the first frame outside of this
    file, find the original line that called the assert_is_type() function, and extract the variable name from
    that line. This is slightly fragile, in particular we assume that only one assert_is_type statement can be per line,
    or that this statement does not spill over multiple lines, etc.
    """
    try:
        raise RuntimeError("Catch me!")
    except RuntimeError:
        # Walk up the stacktrace until we are outside of this file
        tb = sys.exc_info()[2]
        assert tb.tb_frame.f_code.co_name == "_retrieve_assert_arguments"
        this_filename = tb.tb_frame.f_code.co_filename
        fr = tb.tb_frame
        while fr is not None and fr.f_code.co_filename == this_filename:
            fr = fr.f_back

        # Read the source file and tokenize it, extracting the expressions.
        try:
            with io.open(fr.f_code.co_filename, "r", encoding="utf-8") as f:
                # Skip initial lines that are irrelevant
                for i in range(fr.f_lineno - 1): next(f)
                # Create tokenizer
                g = tokenize.generate_tokens(f.readline)
                step = 0
                args_tokens = []
                level = 0
                for ttt in g:
                    if step == 0:
                        if ttt[0] != tokenize.NAME: continue
                        if not ttt[1].startswith("assert_"): continue
                        step = 1
                    elif step == 1:
                        assert ttt[0] == tokenize.OP and ttt[1] == "("
                        args_tokens.append([])
                        step = 2
                    elif step == 2:
                        if level == 0 and ttt[0] == tokenize.OP and ttt[1] == ",":
                            args_tokens.append([])
                        elif level == 0 and ttt[0] == tokenize.OP and ttt[1] == ")":
                            break
                        else:
                            if ttt[0] == tokenize.OP and ttt[1] in "([{": level += 1
                            if ttt[0] == tokenize.OP and ttt[1] in ")]}": level -= 1
                            assert level >= 0, "Parse error: parentheses level became negative"
                            args_tokens[-1].append(ttt)
                args = [tokenize.untokenize(at).strip().replace("\n", " ") for at in args_tokens]
                return args
        except IOError:
            return "arg",


def _check_type(var, vtype):
    """
    Return True if the variable is of the specified type, and False otherwise.

    :param var: variable to check
    :param vtype: expected variable's type
    """
    if vtype is None:
        return var is None
    if isinstance(vtype, _primitive_type):
        return var == vtype
    if vtype is str:
        return isinstance(var, _str_type)
    if vtype is int:
        return isinstance(var, _int_type)
    if vtype is numeric:
        return isinstance(var, _num_type)
    if isinstance(vtype, MagicType):
        return vtype.check(var)
    if isinstance(vtype, type):
        # ``vtype`` is a name of the class, or a built-in type such as "list", "tuple", etc
        return isinstance(var, vtype)
    if isinstance(vtype, list):
        # ``vtype`` is a list literal
        elem_type = U(*vtype)
        return isinstance(var, list) and all(_check_type(item, elem_type) for item in var)
    if isinstance(vtype, set):
        # ``vtype`` is a set literal
        elem_type = U(*vtype)
        return isinstance(var, set) and all(_check_type(item, elem_type) for item in var)
    if isinstance(vtype, tuple):
        # ``vtype`` is a tuple literal
        return (isinstance(var, tuple) and len(vtype) == len(var) and
                all(_check_type(var[i], vtype[i]) for i in range(len(vtype))))
    if isinstance(vtype, dict):
        # ``vtype`` is a dict literal
        ttkv = U(*viewitems(vtype))
        return isinstance(var, dict) and all(_check_type(kv, ttkv) for kv in viewitems(var))
    if isinstance(vtype, (FunctionType, BuiltinFunctionType)):
        return vtype(var)
    raise RuntimeError("Ivalid type %r in _check_type()" % vtype)

def
_check_type
(
var
,
vtype
)
:
if
vtype
is
:
return
var
is
if
isinstance
(
vtype
,
_primitive_type
)
:
return
var
==
vtype
if
vtype
is
str
:
return
isinstance
(
var
,
_str_type
)
if
vtype
is
int
:
return
isinstance
(
var
,
_int_type
)
if
vtype
is
numeric
:
return
isinstance
(
var
,
_num_type
)
if
isinstance
(
vtype
,
MagicType
)
:
return
vtype
.
check
(
var
)
if
isinstance
(
vtype
,
type
)
:
return
isinstance
(
var
,
vtype
)
if
isinstance
(
vtype
,
list
)
:
elem_type
=
U
(
*
vtype
)
return
isinstance
(
var
,
list
)
and
all
(
_check_type
(
item
,
elem_type
)
for
item
in
var
)
if
isinstance
(
vtype
,
set
)
:
elem_type
=
U
(
*
vtype
)
return
isinstance
(
var
,
set
)
and
all
(
_check_type
(
item
,
elem_type
)
for
item
in
var
)
if
isinstance
(
vtype
,
tuple
)
:
return
(
isinstance
(
var
,
tuple
)
and
len
(
vtype
)
==
len
(
var
)
and
all
(
_check_type
(
var
[
i
]
,
vtype
[
i
]
)
for
i
in
range
(
len
(
vtype
)
)
)
)
if
isinstance
(
vtype
,
dict
)
:
ttkv
=
U
(
*
viewitems
(
vtype
)
)
return
isinstance
(
var
,
dict
)
and
all
(
_check_type
(
kv
,
ttkv
)
for
kv
in
viewitems
(
var
)
)
if
isinstance
(
vtype
,
(
FunctionType
,
BuiltinFunctionType
)
)
:
return
vtype
(
var
)
raise
RuntimeError
(
"Ivalid type %r in _check_type()"
%
vtype
)

Return True if the variable is of the specified type and False otherwise .


def _get_type_name(vtype, dump=None):
    """
    Return the name of the provided type.

        _get_type_name(int) == "integer"
        _get_type_name(str) == "string"
        _get_type_name(tuple) == "tuple"
        _get_type_name(Exception) == "Exception"
        _get_type_name(U(int, float, bool)) == "integer|float|bool"
        _get_type_name(U(H2OFrame, None)) == "?H2OFrame"
    """
    if vtype is None:
        return "None"
    if vtype is str:
        return "string"
    if vtype is int:
        return "integer"
    if vtype is numeric:
        return "numeric"
    if is_type(vtype, str):
        return '"%s"' % repr(vtype)[1:-1]
    if is_type(vtype, int):
        return str(vtype)
    if isinstance(vtype, MagicType):
        return vtype.name(dump)
    if isinstance(vtype, type):
        return vtype.__name__
    if isinstance(vtype, list):
        return "list(%s)" % _get_type_name(U(*vtype), dump)
    if isinstance(vtype, set):
        return "set(%s)" % _get_type_name(U(*vtype), dump)
    if isinstance(vtype, tuple):
        return "(%s)" % ", ".join(_get_type_name(item, dump) for item in vtype)
    if isinstance(vtype, dict):
        return "dict(%s)" % ", ".join("%s: %s" % (_get_type_name(tk, dump), _get_type_name(tv, dump))
                                      for tk, tv in viewitems(vtype))
    if isinstance(vtype, (FunctionType, BuiltinFunctionType)):
        if vtype.__name__ == "<lambda>":
            return _get_lambda_source_code(vtype, dump)
        else:
            return vtype.__name__
    raise RuntimeError("Unexpected `vtype`: %r" % vtype)

def
_get_type_name
(
vtype
,
dump
=
)
:
if
vtype
is
:
return
"None"
if
vtype
is
str
:
return
"string"
if
vtype
is
int
:
return
"integer"
if
vtype
is
numeric
:
return
"numeric"
if
is_type
(
vtype
,
str
)
:
return
'"%s"'
%
repr
(
vtype
)
[
1
:
1
]
if
is_type
(
vtype
,
int
)
:
return
str
(
vtype
)
if
isinstance
(
vtype
,
MagicType
)
:
return
vtype
.
name
(
dump
)
if
isinstance
(
vtype
,
type
)
:
return
vtype
.
__name__
if
isinstance
(
vtype
,
list
)
:
return
"list(%s)"
%
_get_type_name
(
U
(
*
vtype
)
,
dump
)
if
isinstance
(
vtype
,
set
)
:
return
"set(%s)"
%
_get_type_name
(
U
(
*
vtype
)
,
dump
)
if
isinstance
(
vtype
,
tuple
)
:
return
"(%s)"
%
", "
.
join
(
_get_type_name
(
item
,
dump
)
for
item
in
vtype
)
if
isinstance
(
vtype
,
dict
)
:
return
"dict(%s)"
%
", "
.
join
(
"%s: %s"
%
(
_get_type_name
(
tk
,
dump
)
,
_get_type_name
(
tv
,
dump
)
)
for
tk
,
tv
in
viewitems
(
vtype
)
)
if
isinstance
(
vtype
,
(
FunctionType
,
BuiltinFunctionType
)
)
:
if
vtype
.
__name__
==
"<lambda>"
:
return
_get_lambda_source_code
(
vtype
,
dump
)
else
:
return
vtype
.
__name__
raise
RuntimeError
(
"Unexpected `vtype`: %r"
%
vtype
)


def _get_lambda_source_code(lambda_fn, src):
    """Attempt to find the source code of the ``lambda_fn`` within the string ``src``."""
    def gen_lambdas():
        def gen():
            yield src + "\n"

        g = gen()
        step = 0
        tokens = []
        for tok in tokenize.generate_tokens(getattr(g, "next", getattr(g, "__next__", None))):
            if step == 0:
                if tok[0] == tokenize.NAME and tok[1] == "lambda":
                    step = 1
                    tokens = [tok]
                    level = 0
            elif step == 1:
                if tok[0] == tokenize.NAME:
                    tokens.append(tok)
                    step = 2
                else:
                    step = 0
            elif step == 2:
                if tok[0] == tokenize.OP and tok[1] == ":":
                    tokens.append(tok)
                    step = 3
                else:
                    step = 0
            elif step == 3:
                if level == 0 and (tok[0] == tokenize.OP and tok[1] in ",)" or tok[0] == tokenize.ENDMARKER):
                    yield tokenize.untokenize(tokens).strip()
                    step = 0
                else:
                    tokens.append(tok)
                    if tok[0] == tokenize.OP:
                        if tok[1] in "[({": level += 1
                        if tok[1] in "])}": level -= 1
        assert not tokens

    actual_code = lambda_fn.__code__.co_code
    for lambda_src in gen_lambdas():
        try:
            fn = eval(lambda_src, globals(), locals())
            if fn.__code__.co_code == actual_code:
                return lambda_src.split(":", 1)[1].strip()
        except Exception:
            pass
    return "<lambda>"

def _get_lambda_source_code ( lambda_fn , src ) : def gen_lambdas ( ) : def gen ( ) : yield src + "\n" g = gen ( ) step = 0 tokens = [ ] for tok in tokenize . generate_tokens ( getattr ( g , "next" , getattr ( g , "__next__" , None ) ) ) : if step == 0 : if tok [ 0 ] == tokenize . NAME and tok [ 1 ] == "lambda" : step = 1 tokens = [ tok ] level = 0 elif step == 1 : if tok [ 0 ] == tokenize . NAME : tokens . append ( tok ) step = 2 else : step = 0 elif step == 2 : if tok [ 0 ] == tokenize . OP and tok [ 1 ] == ":" : tokens . append ( tok ) step = 3 else : step = 0 elif step == 3 : if level == 0 and ( tok [ 0 ] == tokenize . OP and tok [ 1 ] in ",)" or tok [ 0 ] == tokenize . ENDMARKER ) : yield tokenize . untokenize ( tokens ) . strip ( ) step = 0 else : tokens . append ( tok ) if tok [ 0 ] == tokenize . OP : if tok [ 1 ] in "[({" : level += 1 if tok [ 1 ] in "])}" : level -= 1 assert not tokens actual_code = lambda_fn . __code__ . co_code for lambda_src in gen_lambdas ( ) : try : fn = eval ( lambda_src , globals ( ) , locals ( ) ) if fn . __code__ . co_code == actual_code : return lambda_src . split ( ":" , 1 ) [ 1 ] . strip ( ) except Exception : pass return "<lambda>"

Attempt to find the source code of the lambda_fn within the string src .


def check(self, var):
        """Return True if the variable does not match any of the types, and False otherwise."""
        return not any(_check_type(var, tt) for tt in self._types)

def check ( self , var ) : return not any ( _check_type ( var , tt ) for tt in self . _types )

Return True if the variable does not match any of the types and False otherwise .


def check(self, var):
        """Check whether the provided value is a valid enum constant."""
        if not isinstance(var, _str_type): return False
        return _enum_mangle(var) in self._consts

def check ( self , var ) : if not isinstance ( var , _str_type ) : return False return _enum_mangle ( var ) in self . _consts

Check whether the provided value is a valid enum constant .


def get_config():
        """Retrieve the config as a dictionary of key-value pairs."""
        self = H2OConfigReader._get_instance()
        if not self._config_loaded:
            self._read_config()
        return self._config

def get_config ( ) : self = H2OConfigReader . _get_instance ( ) if not self . _config_loaded : self . _read_config ( ) return self . _config

Retrieve the config as a dictionary of key - value pairs .


def _read_config(self):
        """Find and parse config file, storing all variables in ``self._config``."""
        self._config_loaded = True
        conf = []
        for f in self._candidate_log_files():
            if os.path.isfile(f):
                self._logger.info("Reading config file %s" % f)
                section_rx = re.compile(r"^\[(\w+)\]$")
                keyvalue_rx = re.compile(r"^(\w+:)?([\w.]+)\s*=(.*)$")
                with io.open(f, "rt", encoding="utf-8") as config_file:
                    section_name = None
                    for lineno, line in enumerate(config_file):
                        line = line.strip()
                        if line == "" or line.startswith("#"): continue
                        m1 = section_rx.match(line)
                        if m1:
                            section_name = m1.group(1)
                            continue
                        m2 = keyvalue_rx.match(line)
                        if m2:
                            lng = m2.group(1)
                            key = m2.group(2)
                            val = m2.group(3).strip()
                            if lng and lng.lower() != "py:": continue
                            if section_name:
                                key = section_name + "." + key
                            if key in H2OConfigReader._allowed_config_keys:
                                conf.append((key, val))
                            else:
                                self._logger.error("Key %s is not a valid config key" % key)
                            continue
                        self._logger.error("Syntax error in config file line %d: %s" % (lineno, line))
                self._config = dict(conf)
                return

Find and parse config file storing all variables in self . _config .


def _candidate_log_files():
        """Return possible locations for the .h2oconfig file, one at a time."""
        # Search for .h2oconfig in the current directory and all parent directories
        relpath = ".h2oconfig"
        prevpath = None
        while True:
            abspath = os.path.abspath(relpath)
            if abspath == prevpath: break
            prevpath = abspath
            relpath = "../" + relpath
            yield abspath
        # Also check if .h2oconfig exists in the user's directory
        yield os.path.expanduser("~/.h2oconfig")

def _candidate_log_files ( ) : relpath = ".h2oconfig" prevpath = None while True : abspath = os . path . abspath ( relpath ) if abspath == prevpath : break prevpath = abspath relpath = "../" + relpath yield abspath yield os . path . expanduser ( "~/.h2oconfig" )

Return possible locations for the . h2oconfig file one at a time .


def execute(self, progress_fn, print_verbose_info=None):
        """
        Start the progress bar, and return only when the progress reaches 100%.

        :param progress_fn: the executor function (or a generator). This function should take no arguments
            and return either a single number -- the current progress level, or a tuple (progress level, delay),
            where delay is the time interval for when the progress should be checked again. This function may at
            any point raise the ``StopIteration(message)`` exception, which will interrupt the progress bar,
            display the ``message`` in red font, and then re-raise the exception.
        :raises StopIteration: if the job is interrupted. The reason for interruption is provided in the exception's
            message. The message will say "cancelled" if the job was interrupted by the user by pressing Ctrl+C.
        """
        assert_is_type(progress_fn, FunctionType, GeneratorType, MethodType)
        if isinstance(progress_fn, GeneratorType):
            # Convert generator to a regular function
            progress_fn = (lambda g: lambda: next(g))(progress_fn)

        # Initialize the execution context
        self._next_poll_time = 0
        self._t0 = time.time()
        self._x0 = 0
        self._v0 = 0.01  # corresponds to 100s completion time
        self._ve = 0.01

        progress = 0
        status = None  # Status message in case the job gets interrupted.
        try:
            while True:
                # We attempt to synchronize all helper functions, ensuring that each of them has the same idea
                # for what the current time moment is. Otherwise we could have some corner cases when one method
                # says that something must happen right now, while the other already sees that moment in the past.
                now = time.time()

                # Query the progress level, but only if it's time already
                if self._next_poll_time <= now:
                    res = progress_fn()  # may raise StopIteration
                    assert_is_type(res, (numeric, numeric), numeric)
                    if not isinstance(res, tuple):
                        res = (res, -1)
                    # Progress querying could have taken some time, so update the current time moment
                    now = time.time()
                    self._store_model_progress(res, now)
                    self._recalculate_model_parameters(now)

                # Render the widget regardless of whether it's too early or not
                progress = min(self._compute_progress_at_time(now)[0], 1)
                if progress == 1 and self._get_real_progress() >= 1:
                    # Do not exit until both the model and the actual progress reach 100% mark.
                    break
                result = self._widget.render(progress)
                assert_is_type(result, RenderResult)
                time0 = result.next_time
                time1 = self._get_time_at_progress(result.next_progress)
                next_render_time = min(time0, time1)
                self._draw(result.rendered)

                # Wait until the next rendering/querying cycle
                wait_time = min(next_render_time, self._next_poll_time) - now
                if wait_time > 0:
                    time.sleep(wait_time)
                    if print_verbose_info is not None:
                        print_verbose_info(progress)
        except KeyboardInterrupt:
            # If the user presses Ctrl+C, we interrupt the progress bar.
            status = "cancelled"
        except StopIteration as e:
            # If the generator raises StopIteration before reaching 100%, then the progress display will
            # reamin incomplete.
            status = str(e)

        # Do one final rendering before we exit
        result = self._widget.render(progress=progress, status=status)
        self._draw(result.rendered, final=True)

        if status == "cancelled":
            # Re-raise the exception, to inform the upstream caller that something unexpected happened.
            raise StopIteration(status)

def execute ( self , progress_fn , print_verbose_info = None ) : assert_is_type ( progress_fn , FunctionType , GeneratorType , MethodType ) if isinstance ( progress_fn , GeneratorType ) : progress_fn = ( lambda g : lambda : next ( g ) ) ( progress_fn ) self . _next_poll_time = 0 self . _t0 = time . time ( ) self . _x0 = 0 self . _v0 = 0.01 self . _ve = 0.01 progress = 0 status = None try : while True : now = time . time ( ) if self . _next_poll_time <= now : res = progress_fn ( ) assert_is_type ( res , ( numeric , numeric ) , numeric ) if not isinstance ( res , tuple ) : res = ( res , - 1 ) now = time . time ( ) self . _store_model_progress ( res , now ) self . _recalculate_model_parameters ( now ) progress = min ( self . _compute_progress_at_time ( now ) [ 0 ] , 1 ) if progress == 1 and self . _get_real_progress ( ) >= 1 : break result = self . _widget . render ( progress ) assert_is_type ( result , RenderResult ) time0 = result . next_time time1 = self . _get_time_at_progress ( result . next_progress ) next_render_time = min ( time0 , time1 ) self . _draw ( result . rendered ) wait_time = min ( next_render_time , self . _next_poll_time ) - now if wait_time > 0 : time . sleep ( wait_time ) if print_verbose_info is not None : print_verbose_info ( progress ) except KeyboardInterrupt : status = "cancelled" except StopIteration as e : status = str ( e ) result = self . _widget . render ( progress = progress , status = status ) self . _draw ( result . rendered , final = True ) if status == "cancelled" : raise StopIteration ( status )

Start the progress bar and return only when the progress reaches 100% .


def _store_model_progress(self, res, now):
        """
        Save the current model progress into ``self._progress_data``, and update ``self._next_poll_time``.

        :param res: tuple (progress level, poll delay).
        :param now: current timestamp.
        """
        raw_progress, delay = res
        raw_progress = clamp(raw_progress, 0, self._maxval)
        self._progress_data.append((now, raw_progress))

        if delay < 0:
            # calculation of ``_guess_next_poll_interval()`` should be done only *after* we pushed the fresh data to
            # ``self._progress_data``.
            delay = self._guess_next_poll_interval()
        self._next_poll_time = now + clamp(delay, self.MIN_PROGRESS_CHECK_INTERVAL, self.MAX_PROGRESS_CHECK_INTERVAL)

def _store_model_progress ( self , res , now ) : raw_progress , delay = res raw_progress = clamp ( raw_progress , 0 , self . _maxval ) self . _progress_data . append ( ( now , raw_progress ) ) if delay < 0 : delay = self . _guess_next_poll_interval ( ) self . _next_poll_time = now + clamp ( delay , self . MIN_PROGRESS_CHECK_INTERVAL , self . MAX_PROGRESS_CHECK_INTERVAL )

Save the current model progress into self . _progress_data and update self . _next_poll_time .


def _recalculate_model_parameters(self, now):
        """Compute t0, x0, v0, ve."""
        time_until_end = self._estimate_progress_completion_time(now) - now
        assert time_until_end >= 0, "Estimated progress completion cannot be in the past."
        x_real = self._get_real_progress()
        if x_real == 1:
            t0, x0, v0, ve = now, 1, 0, 0
        else:
            x0, v0 = self._compute_progress_at_time(now)
            t0 = now
            if x0 >= 1:
                # On rare occasion, the model's progress may have reached 100% by ``now``. This can happen if
                # (1) the progress is close to 100% initially and has high speed, (2) on the previous call we
                # estimated that the process completion time will be right after the next poll time, and (3)
                # the polling itself took so much time that the process effectively "overshoot".
                # If this happens, then we adjust x0, v0 to the previous valid data checkpoint.
                t0, x0, v0 = self._t0, self._x0, self._v0
                time_until_end += now - t0
            z = self.BETA * time_until_end
            max_speed = (1 - x_real**2) / self.FINISH_DELAY
            ve = v0 + (self.BETA * (1 - x0) - v0 * z) / (z - 1 + math.exp(-z))
            if ve < 0:
                # Current speed is too high -- reduce v0 (violate non-smoothness of speed)
                v0 = self.BETA * (1 - x0) / (1 - math.exp(-z))
                ve = 0
            if ve > max_speed:
                # Current speed is too low: finish later, but do not allow ``ve`` to be higher than ``max_speed``
                ve = max_speed
        self._t0, self._x0, self._v0, self._ve = t0, x0, v0, ve


def _estimate_progress_completion_time(self, now):
        """
        Estimate the moment when the underlying process is expected to reach completion.

        This function should only return future times. Also this function is not allowed to return time moments less
        than self._next_poll_time if the actual progress is below 100% (this is because we won't know that the
        process have finished until we poll the external progress function).
        """
        assert self._next_poll_time >= now
        tlast, wlast = self._progress_data[-1]
        # If reached 100%, make sure that we finish as soon as possible, but maybe not immediately
        if wlast == self._maxval:
            current_completion_time = (1 - self._x0) / self._v0 + self._t0
            return clamp(current_completion_time, now, now + self.FINISH_DELAY)

        # Calculate the approximate speed of the raw progress based on recent data
        tacc, wacc = 0, 0
        factor = self.GAMMA
        for t, x in self._progress_data[-2::-1]:
            tacc += factor * (tlast - t)
            wacc += factor * (wlast - x)
            factor *= self.GAMMA
            if factor < 1e-2: break

        # If there was no progress at all, then just assume it's 5 minutes from now
        if wacc == 0: return now + 300

        # Estimate the completion time assuming linear progress
        t_estimate = tlast + tacc * (self._maxval - wlast) / wacc

        # Adjust the estimate if it looks like it may happen too soon
        if t_estimate <= self._next_poll_time:
            t_estimate = self._next_poll_time + self.FINISH_DELAY

        return t_estimate

def _estimate_progress_completion_time ( self , now ) : assert self . _next_poll_time >= now tlast , wlast = self . _progress_data [ - 1 ] if wlast == self . _maxval : current_completion_time = ( 1 - self . _x0 ) / self . _v0 + self . _t0 return clamp ( current_completion_time , now , now + self . FINISH_DELAY ) tacc , wacc = 0 , 0 factor = self . GAMMA for t , x in self . _progress_data [ - 2 : : - 1 ] : tacc += factor * ( tlast - t ) wacc += factor * ( wlast - x ) factor *= self . GAMMA if factor < 1e-2 : break if wacc == 0 : return now + 300 t_estimate = tlast + tacc * ( self . _maxval - wlast ) / wacc if t_estimate <= self . _next_poll_time : t_estimate = self . _next_poll_time + self . FINISH_DELAY return t_estimate

Estimate the moment when the underlying process is expected to reach completion .


def _guess_next_poll_interval(self):
        """
        Determine when to query the progress status next.

        This function is used if the external progress function did not return time interval for when it should be
        queried next.
        """
        time_elapsed = self._progress_data[-1][0] - self._progress_data[0][0]
        real_progress = self._get_real_progress()
        return min(0.2 * time_elapsed, 0.5 + (1 - real_progress)**0.5)

def _guess_next_poll_interval ( self ) : time_elapsed = self . _progress_data [ - 1 ] [ 0 ] - self . _progress_data [ 0 ] [ 0 ] real_progress = self . _get_real_progress ( ) return min ( 0.2 * time_elapsed , 0.5 + ( 1 - real_progress ) ** 0.5 )

Determine when to query the progress status next .


def _compute_progress_at_time(self, t):
        """
        Calculate the modelled progress state for the given time moment.

        :returns: tuple (x, v) of the progress level and progress speed.
        """
        t0, x0, v0, ve = self._t0, self._x0, self._v0, self._ve
        z = (v0 - ve) * math.exp(-self.BETA * (t - t0))
        vt = ve + z
        xt = clamp(x0 + ve * (t - t0) + (v0 - ve - z) / self.BETA, 0, 1)
        return xt, vt

def _compute_progress_at_time ( self , t ) : t0 , x0 , v0 , ve = self . _t0 , self . _x0 , self . _v0 , self . _ve z = ( v0 - ve ) * math . exp ( - self . BETA * ( t - t0 ) ) vt = ve + z xt = clamp ( x0 + ve * ( t - t0 ) + ( v0 - ve - z ) / self . BETA , 0 , 1 ) return xt , vt

Calculate the modelled progress state for the given time moment .


def _get_time_at_progress(self, x_target):
        """
        Return the projected time when progress level `x_target` will be reached.

        Since the underlying progress model is nonlinear, we need to do use Newton method to find a numerical solution
        to the equation x(t) = x_target.
        """
        t, x, v = self._t0, self._x0, self._v0
        # The convergence should be achieved in just few iterations, however in unlikely situation that it doesn't
        # we don't want to loop forever...
        for _ in range(20):
            if v == 0: return 1e20
            # make time prediction assuming the progress will continue at a linear speed ``v``
            t += (x_target - x) / v
            # calculate the actual progress at that time
            x, v = self._compute_progress_at_time(t)
            # iterate until convergence
            if abs(x - x_target) < 1e-3: return t
        return time.time() + 100

def _get_time_at_progress ( self , x_target ) : t , x , v = self . _t0 , self . _x0 , self . _v0 for _ in range ( 20 ) : if v == 0 : return 1e20 t += ( x_target - x ) / v x , v = self . _compute_progress_at_time ( t ) if abs ( x - x_target ) < 1e-3 : return t return time . time ( ) + 100

Return the projected time when progress level x_target will be reached .


def _draw(self, txt, final=False):
        """Print the rendered string to the stdout."""
        if not self._file_mode:
            # If the user presses Ctrl+C this ensures we still start writing from the beginning of the line
            sys.stdout.write("\r")
        sys.stdout.write(txt)
        if final and not isinstance(self._widget, _HiddenWidget):
            sys.stdout.write("\n")
        else:
            if not self._file_mode:
                sys.stdout.write("\r")
            sys.stdout.flush()

def _draw ( self , txt , final = False ) : if not self . _file_mode : sys . stdout . write ( "\r" ) sys . stdout . write ( txt ) if final and not isinstance ( self . _widget , _HiddenWidget ) : sys . stdout . write ( "\n" ) else : if not self . _file_mode : sys . stdout . write ( "\r" ) sys . stdout . flush ( )

Print the rendered string to the stdout .


def _compute_widget_sizes(self):
        """Initial rendering stage, done in order to compute widths of all widgets."""
        wl = [0] * len(self._widgets)
        flex_count = 0

        # First render all non-flexible widgets
        for i, widget in enumerate(self._widgets):
            if isinstance(widget, ProgressBarFlexibleWidget):
                flex_count += 1
            else:
                wl[i] = widget.render(1).length

        remaining_width = self._width - sum(wl)
        remaining_width -= len(self._widgets) - 1  # account for 1-space interval between widgets
        if remaining_width < 10 * flex_count:
            if self._file_mode:
                remaining_width = 10 * flex_count
            else:
                # The window is too small to accomodate the widget: try to split it into several lines, otherwise
                # switch to "file mode". If we don't do this, then rendering the widget will cause it to wrap, and
                # then when we use \r to go to the beginning of the line, only part of the widget will be overwritten,
                # which means we'll have many (possibly hundreds) of progress bar lines in the end.
                widget0 = self._widgets[0]
                if isinstance(widget0, PBWString) and remaining_width + widget0.render(0).length >= 10 * flex_count:
                    remaining_width += widget0.render(0).length + 1
                    self._to_render = widget0.render(0).rendered + "\n"
                    self._widgets = self._widgets[1:]
                if remaining_width < 10 * flex_count:
                    self._file_mode = True
                    remaining_width = 10 * flex_count

        remaining_width = max(remaining_width, 10 * flex_count)  # Ensure at least 10 chars per flexible widget

        for i, widget in enumerate(self._widgets):
            if isinstance(widget, ProgressBarFlexibleWidget):
                target_length = int(remaining_width / flex_count)
                result = widget.render(1, target_length)
                wl[i] = result.length
                remaining_width -= result.length
                flex_count -= 1

        return wl

def _compute_widget_sizes ( self ) : wl = [ 0 ] * len ( self . _widgets ) flex_count = 0 for i , widget in enumerate ( self . _widgets ) : if isinstance ( widget , ProgressBarFlexibleWidget ) : flex_count += 1 else : wl [ i ] = widget . render ( 1 ) . length remaining_width = self . _width - sum ( wl ) remaining_width -= len ( self . _widgets ) - 1 if remaining_width < 10 * flex_count : if self . _file_mode : remaining_width = 10 * flex_count else : widget0 = self . _widgets [ 0 ] if isinstance ( widget0 , PBWString ) and remaining_width + widget0 . render ( 0 ) . length >= 10 * flex_count : remaining_width += widget0 . render ( 0 ) . length + 1 self . _to_render = widget0 . render ( 0 ) . rendered + "\n" self . _widgets = self . _widgets [ 1 : ] if remaining_width < 10 * flex_count : self . _file_mode = True remaining_width = 10 * flex_count remaining_width = max ( remaining_width , 10 * flex_count ) for i , widget in enumerate ( self . _widgets ) : if isinstance ( widget , ProgressBarFlexibleWidget ) : target_length = int ( remaining_width / flex_count ) result = widget . render ( 1 , target_length ) wl [ i ] = result . length remaining_width -= result . length flex_count -= 1 return wl

Initial rendering stage done in order to compute widths of all widgets .


def _get_terminal_size():
        """Find current STDOUT's width, in characters."""
        # If output is not terminal but a regular file, assume 100 chars width
        if not sys.stdout.isatty():
            return 80

        # Otherwise, first try getting the dimensions from shell command `stty`:
        try:
            import subprocess
            ret = subprocess.check_output(["stty", "size"]).strip().split(" ")
            if len(ret) == 2:
                return int(ret[1])
        except:
            pass

        # Otherwise try using ioctl
        try:
            from termios import TIOCGWINSZ
            from fcntl import ioctl
            from struct import unpack
            res = unpack("hh", ioctl(sys.stdout, TIOCGWINSZ, b"1234"))
            return int(res[1])
        except:
            pass

        # Finally check the COLUMNS environment variable
        return int(os.environ.get("COLUMNS", 80))

def
_get_terminal_size
(
)
:
if
not
sys
.
stdout
.
isatty
(
)
:
return
80
try
:
import
subprocess
ret
=
subprocess
.
check_output
(
[
"stty"
,
"size"
]
)
.
strip
(
)
.
split
(
" "
)
if
len
(
ret
)
==
2
:
return
int
(
ret
[
1
]
)
except
:
pass
try
:
from
termios
import
TIOCGWINSZ
from
fcntl
import
ioctl
from
struct
import
unpack
res
=
unpack
(
"hh"
,
ioctl
(
sys
.
stdout
,
TIOCGWINSZ
,
b"1234"
)
)
return
int
(
res
[
1
]
)
except
:
pass
return
int
(
os
.
environ
.
get
(
"COLUMNS"
,
80
)
)

Find current STDOUT s width in characters .


def set_encoding(self, encoding):
        """Inform the widget about the encoding of the underlying character stream."""
        self._bar_ends = "[]"
        self._bar_symbols = "#"
        if not encoding: return
        s1 = "\u258F\u258E\u258D\u258C\u258B\u258A\u2589\u2588"
        s2 = "\u258C\u2588"
        s3 = "\u2588"
        if self._file_mode:
            s1 = s2 = None
        assert len(s3) == 1
        for s in (s1, s2, s3):
            if s is None: continue
            try:
                s.encode(encoding)
                self._bar_ends = "||"
                self._bar_symbols = s
                return
            except UnicodeEncodeError:
                pass
            except LookupError:
                print("Warning: unknown encoding %s" % encoding)

Inform the widget about the encoding of the underlying character stream .


def fit(self, frame = None):
        """
        Returns encoding map as an object that maps 'column_name' -> 'frame_with_encoding_map_for_this_column_name'

        :param frame frame: An H2OFrame object with which to create the target encoding map
        """
        self._teColumns = list(map(lambda i: frame.names[i], self._teColumns)) if all(isinstance(n, int) for n in self._teColumns) else self._teColumns
        self._responseColumnName = frame.names[self._responseColumnName] if isinstance(self._responseColumnName, int) else self._responseColumnName
        self._foldColumnName = frame.names[self._foldColumnName] if isinstance(self._foldColumnName, int) else self._foldColumnName

        self._encodingMap = ExprNode("target.encoder.fit", frame, self._teColumns, self._responseColumnName,
                                     self._foldColumnName)._eager_map_frame()

        return self._encodingMap

def fit ( self , frame = None ) : self . _teColumns = list ( map ( lambda i : frame . names [ i ] , self . _teColumns ) ) if all ( isinstance ( n , int ) for n in self . _teColumns ) else self . _teColumns self . _responseColumnName = frame . names [ self . _responseColumnName ] if isinstance ( self . _responseColumnName , int ) else self . _responseColumnName self . _foldColumnName = frame . names [ self . _foldColumnName ] if isinstance ( self . _foldColumnName , int ) else self . _foldColumnName self . _encodingMap = ExprNode ( "target.encoder.fit" , frame , self . _teColumns , self . _responseColumnName , self . _foldColumnName ) . _eager_map_frame ( ) return self . _encodingMap


def get_frame(frame_id, rows=10, rows_offset=0, cols=-1, full_cols=-1, cols_offset=0, light=False):
        """
        Retrieve an existing H2OFrame from the H2O cluster using the frame's id.

        :param str frame_id: id of the frame to retrieve
        :param int rows: number of rows to fetch for preview (10 by default)
        :param int rows_offset: offset to fetch rows from (0 by default)
        :param int cols: number of columns to fetch (all by default)
        :param full_cols: number of columns to fetch together with backed data
        :param int cols_offset: offset to fetch rows from (0 by default)
        :param bool light: wether to use light frame endpoint or not
        :returns: an existing H2OFrame with the id provided; or None if such frame doesn't exist.
        """
        fr = H2OFrame()
        fr._ex._cache._id = frame_id
        try:
            fr._ex._cache.fill(rows=rows, rows_offset=rows_offset, cols=cols, full_cols=full_cols, cols_offset=cols_offset, light=light)
        except EnvironmentError:
            return None
        return fr

def get_frame ( frame_id , rows = 10 , rows_offset = 0 , cols = - 1 , full_cols = - 1 , cols_offset = 0 , light = False ) : fr = H2OFrame ( ) fr . _ex . _cache . _id = frame_id try : fr . _ex . _cache . fill ( rows = rows , rows_offset = rows_offset , cols = cols , full_cols = full_cols , cols_offset = cols_offset , light = light ) except EnvironmentError : return None return fr

Retrieve an existing H2OFrame from the H2O cluster using the frame s id .


def refresh(self):
        """Reload frame information from the backend H2O server."""
        self._ex._cache.flush()
        self._frame(fill_cache=True)

def refresh ( self ) : self . _ex . _cache . flush ( ) self . _frame ( fill_cache = True )

Reload frame information from the backend H2O server .


def type(self, col):
        """
        The type for the given column.

        :param col: either a name, or an index of the column to look up
        :returns: type of the column, one of: ``str``, ``int``, ``real``, ``enum``, ``time``, ``bool``.
        :raises H2OValueError: if such column does not exist in the frame.
        """
        assert_is_type(col, int, str)
        if not self._ex._cache.types_valid() or not self._ex._cache.names_valid():
            self._ex._cache.flush()
            self._frame(fill_cache=True)
        types = self._ex._cache.types
        if is_type(col, str):
            if col in types:
                return types[col]
        else:
            names = self._ex._cache.names
            if -len(names) <= col < len(names):
                return types[names[col]]
        raise H2OValueError("Column '%r' does not exist in the frame" % col)


def columns_by_type(self, coltype="numeric"):
        """
        Extract columns of the specified type from the frame.

        :param str coltype: A character string indicating which column type to filter by. This must be
            one of the following:

            - ``"numeric"``      - Numeric, but not categorical or time
            - ``"categorical"``  - Integer, with a categorical/factor String mapping
            - ``"string"``       - String column
            - ``"time"``         - Long msec since the Unix Epoch - with a variety of display/parse options
            - ``"uuid"``         - UUID
            - ``"bad"``          - No none-NA rows (triple negative! all NAs or zero rows)

        :returns: list of indices of columns that have the requested type
        """
        assert_is_type(coltype, "numeric", "categorical", "string", "time", "uuid", "bad")
        assert_is_type(self, H2OFrame)
        return ExprNode("columnsByType", self, coltype)._eager_scalar()

def columns_by_type ( self , coltype = "numeric" ) : assert_is_type ( coltype , "numeric" , "categorical" , "string" , "time" , "uuid" , "bad" ) assert_is_type ( self , H2OFrame ) return ExprNode ( "columnsByType" , self , coltype ) . _eager_scalar ( )

Extract columns of the specified type from the frame .


def summary(self, return_data=False):
        """
        Display summary information about the frame.

        Summary includes min/mean/max/sigma and other rollup data.

        :param bool return_data: Return a dictionary of the summary output
        """
        if not self._has_content():
            print("This H2OFrame is empty and not initialized.")
            return self._ex._cache._data;
        if not self._ex._cache.is_valid(): self._frame()._ex._cache.fill()
        if not return_data:
            if self.nrows == 0:
                print("This H2OFrame is empty.")
            elif H2ODisplay._in_ipy():
                import IPython.display
                IPython.display.display_html(self._ex._cache._tabulate("html", True), raw=True)
            else:
                print(self._ex._cache._tabulate("simple", True))
        else:
            return self._ex._cache._data

Display summary information about the frame .


def describe(self, chunk_summary=False):
        """
        Generate an in-depth description of this H2OFrame.

        This will print to the console the dimensions of the frame; names/types/summary statistics for each column;
        and finally first ten rows of the frame.

        :param bool chunk_summary: Retrieve the chunk summary along with the distribution summary
        """
        if self._has_content():
            res = h2o.api("GET /3/Frames/%s" % self.frame_id, data={"row_count": 10})["frames"][0]
            self._ex._cache._fill_data(res)

            print("Rows:{}".format(self.nrow))
            print("Cols:{}".format(self.ncol))

            #The chunk & distribution summaries are not cached, so must be pulled if chunk_summary=True.
            if chunk_summary:
                res["chunk_summary"].show()
                res["distribution_summary"].show()
            print("\n")
        self.summary()

def
describe
(
self
,
chunk_summary
=
False
)
:
if
self
.
_has_content
(
)
:
res
=
h2o
.
api
(
"GET /3/Frames/%s"
%
self
.
frame_id
,
data
=
{
"row_count"
:
10
}
)
[
"frames"
]
[
0
]
self
.
_ex
.
_cache
.
_fill_data
(
res
)
print
(
"Rows:{}"
.
format
(
self
.
nrow
)
)
print
(
"Cols:{}"
.
format
(
self
.
ncol
)
)
if
chunk_summary
:
res
[
"chunk_summary"
]
.
show
(
)
res
[
"distribution_summary"
]
.
show
(
)
print
(
"\n"
)
self
.
summary
(
)

Generate an in - depth description of this H2OFrame .


def head(self, rows=10, cols=200):
        """
        Return the first ``rows`` and ``cols`` of the frame as a new H2OFrame.

        :param int rows: maximum number of rows to return
        :param int cols: maximum number of columns to return
        :returns: a new H2OFrame cut from the top left corner of the current frame, and having dimensions at
            most ``rows`` x ``cols``.
        """
        assert_is_type(rows, int)
        assert_is_type(cols, int)
        nrows = min(self.nrows, rows)
        ncols = min(self.ncols, cols)
        newdt = self[:nrows, :ncols]
        return newdt._frame(rows=nrows, cols=cols, fill_cache=True)

def head ( self , rows = 10 , cols = 200 ) : assert_is_type ( rows , int ) assert_is_type ( cols , int ) nrows = min ( self . nrows , rows ) ncols = min ( self . ncols , cols ) newdt = self [ : nrows , : ncols ] return newdt . _frame ( rows = nrows , cols = cols , fill_cache = True )

Return the first rows and cols of the frame as a new H2OFrame .


def mult(self, matrix):
        """
        Multiply this frame, viewed as a matrix, by another matrix.

        :param matrix: another frame that you want to multiply the current frame by; must be compatible with the
            current frame (i.e. its number of rows must be the same as number of columns in the current frame).
        :returns: new H2OFrame, which is the result of multiplying the current frame by ``matrix``.
        """
        if self.ncols != matrix.nrows:
            raise H2OValueError("Matrix is not compatible for multiplication with the current frame")
        return H2OFrame._expr(expr=ExprNode("x", self, matrix))

Multiply this frame viewed as a matrix by another matrix .


def levels(self):
        """
        Get the factor levels.

        :returns: A list of lists, one list per column, of levels.
        """
        lol = H2OFrame._expr(expr=ExprNode("levels", self)).as_data_frame(False)
        lol.pop(0)  # Remove column headers
        lol = list(zip(*lol))
        return [[ll for ll in l if ll != ''] for l in lol]

def levels ( self ) : lol = H2OFrame . _expr ( expr = ExprNode ( "levels" , self ) ) . as_data_frame ( False ) lol . pop ( 0 ) lol = list ( zip ( * lol ) ) return [ [ ll for ll in l if ll != '' ] for l in lol ]


def nlevels(self):
        """
        Get the number of factor levels for each categorical column.

        :returns: A list of the number of levels per column.
        """
        levels = self.levels()
        return [len(l) for l in levels] if levels else 0

def nlevels ( self ) : levels = self . levels ( ) return [ len ( l ) for l in levels ] if levels else 0

Get the number of factor levels for each categorical column .


def set_level(self, level):
        """
        A method to set all column values to one of the levels.

        :param str level: The level at which the column will be set (a string)

        :returns: H2OFrame with entries set to the desired level.
        """
        return H2OFrame._expr(expr=ExprNode("setLevel", self, level), cache=self._ex._cache)

def set_level ( self , level ) : return H2OFrame . _expr ( expr = ExprNode ( "setLevel" , self , level ) , cache = self . _ex . _cache )

A method to set all column values to one of the levels .


def set_levels(self, levels):
        """
        Replace the levels of a categorical column.

        New levels must be aligned with the old domain. This call has copy-on-write semantics.

        :param List[str] levels: A list of strings specifying the new levels. The number of new
            levels must match the number of old levels.
        :returns: A single-column H2OFrame with the desired levels.
        """
        assert_is_type(levels, [str])
        return H2OFrame._expr(expr=ExprNode("setDomain", self, False, levels), cache=self._ex._cache)

def set_levels ( self , levels ) : assert_is_type ( levels , [ str ] ) return H2OFrame . _expr ( expr = ExprNode ( "setDomain" , self , False , levels ) , cache = self . _ex . _cache )

Replace the levels of a categorical column .


def rename(self, columns=None):
        """
        Change names of columns in the frame.

        Dict key is an index or name of the column whose name is to be set.
        Dict value is the new name of the column.

        :param columns: dict-like transformations to apply to the column names
        """
        assert_is_type(columns, None, dict)
        new_names = self.names
        ncols = self.ncols

        for col, name in columns.items():
            col_index = None
            if is_type(col, int) and (-ncols <= col < ncols):
                col_index = (col + ncols) % ncols  # handle negative indices
            elif is_type(col, str) and col in self.names:
                col_index = self.names.index(col)  # lookup the name

            if col_index is not None:
                new_names[col_index] = name

        return self.set_names(new_names)

def rename ( self , columns = None ) : assert_is_type ( columns , None , dict ) new_names = self . names ncols = self . ncols for col , name in columns . items ( ) : col_index = None if is_type ( col , int ) and ( - ncols <= col < ncols ) : col_index = ( col + ncols ) % ncols elif is_type ( col , str ) and col in self . names : col_index = self . names . index ( col ) if col_index is not None : new_names [ col_index ] = name return self . set_names ( new_names )


def set_names(self, names):
        """
        Change names of all columns in the frame.

        :param List[str] names: The list of new names for every column in the frame.
        """
        assert_is_type(names, [str])
        assert_satisfies(names, len(names) == self.ncol)
        self._ex = ExprNode("colnames=", self, range(self.ncol), names)  # Update-in-place, but still lazy
        return self

def set_names ( self , names ) : assert_is_type ( names , [ str ] ) assert_satisfies ( names , len ( names ) == self . ncol ) self . _ex = ExprNode ( "colnames=" , self , range ( self . ncol ) , names ) return self

Change names of all columns in the frame .


def set_name(self, col=None, name=None):
        """
        Set a new name for a column.

        :param col: index or name of the column whose name is to be set; may be skipped for 1-column frames
        :param name: the new name of the column
        """
        assert_is_type(col, None, int, str)
        assert_is_type(name, str)
        ncols = self.ncols

        col_index = None
        if is_type(col, int):
            if not(-ncols <= col < ncols):
                raise H2OValueError("Index %d is out of bounds for a frame with %d columns" % (col, ncols))
            col_index = (col + ncols) % ncols  # handle negative indices
        elif is_type(col, str):
            if col not in self.names:
                raise H2OValueError("Column %s doesn't exist in the frame." % col)
            col_index = self.names.index(col)  # lookup the name
        else:
            assert col is None
            if ncols != 1:
                raise H2OValueError("The frame has %d columns; please specify which one to rename" % ncols)
            col_index = 0
        if name != self.names[col_index] and name in self.types:
            raise H2OValueError("Column '%s' already exists in the frame" % name)

        oldname = self.names[col_index]
        old_cache = self._ex._cache
        self._ex = ExprNode("colnames=", self, col_index, name)  # Update-in-place, but still lazy
        self._ex._cache.fill_from(old_cache)
        if self.names is None:
            self._frame()._ex._cache.fill()
        else:
            self._ex._cache._names = self.names[:col_index] + [name] + self.names[col_index + 1:]
            self._ex._cache._types[name] = self._ex._cache._types.pop(oldname)
        return


def isin(self, item):
        """
        Test whether elements of an H2OFrame are contained in the ``item``.

        :param items: An item or a list of items to compare the H2OFrame against.

        :returns: An H2OFrame of 0s and 1s showing whether each element in the original H2OFrame is contained in item.
        """
        if is_type(item, list, tuple, set):
            if self.ncols == 1 and (self.type(0) == 'str' or self.type(0) == 'enum'):
                return self.match(item)
            else:
                return functools.reduce(H2OFrame.__or__, (self == i for i in item))
        else:
            return self == item

def isin ( self , item ) : if is_type ( item , list , tuple , set ) : if self . ncols == 1 and ( self . type ( 0 ) == 'str' or self . type ( 0 ) == 'enum' ) : return self . match ( item ) else : return functools . reduce ( H2OFrame . __or__ , ( self == i for i in item ) ) else : return self == item

Test whether elements of an H2OFrame are contained in the item .


def modulo_kfold_column(self, n_folds=3):
        """
        Build a fold assignments column for cross-validation.

        Rows are assigned a fold according to the current row number modulo ``n_folds``.

        :param int n_folds: An integer specifying the number of validation sets to split the training data into.
        :returns: A single-column H2OFrame with the fold assignments.
        """
        return H2OFrame._expr(expr=ExprNode("modulo_kfold_column", self, n_folds))._frame()

def modulo_kfold_column ( self , n_folds = 3 ) : return H2OFrame . _expr ( expr = ExprNode ( "modulo_kfold_column" , self , n_folds ) ) . _frame ( )

Build a fold assignments column for cross - validation .


def stratified_kfold_column(self, n_folds=3, seed=-1):
        """
        Build a fold assignment column with the constraint that each fold has the same class
        distribution as the fold column.

        :param int n_folds: The number of folds to build.
        :param int seed: A seed for the random number generator.

        :returns: A single column H2OFrame with the fold assignments.
        """
        return H2OFrame._expr(
            expr=ExprNode("stratified_kfold_column", self, n_folds, seed))._frame()

def stratified_kfold_column ( self , n_folds = 3 , seed = - 1 ) : return H2OFrame . _expr ( expr = ExprNode ( "stratified_kfold_column" , self , n_folds , seed ) ) . _frame ( )

Build a fold assignment column with the constraint that each fold has the same class distribution as the fold column .


def structure(self):
        """Compactly display the internal structure of an H2OFrame."""
        df = self.as_data_frame(use_pandas=False)
        cn = df.pop(0)
        nr = self.nrow
        nc = self.ncol
        width = max([len(c) for c in cn])
        isfactor = self.isfactor()
        numlevels = self.nlevels()
        lvls = self.levels()
        print("H2OFrame: '{}' \nDimensions: {} obs. of {} variables".format(self.frame_id, nr, nc))
        for i in range(nc):
            print("$ {} {}: ".format(cn[i], ' ' * (width - max(0, len(cn[i])))), end=' ')
            if isfactor[i]:
                nl = numlevels[i]
                print("Factor w/ {} level(s) {} ".format(nl, '"' + '","'.join(lvls[i]) + '"'), end='\n')
            else:
                print("num {}".format(" ".join(it[0] if it else "nan" for it in h2o.as_list(self[:10, i], False)[1:])))

Compactly display the internal structure of an H2OFrame .


def as_data_frame(self, use_pandas=True, header=True):
        """
        Obtain the dataset as a python-local object.

        :param bool use_pandas: If True (default) then return the H2OFrame as a pandas DataFrame (requires that the
            ``pandas`` library was installed). If False, then return the contents of the H2OFrame as plain nested
            list, in a row-wise order.
        :param bool header: If True (default), then column names will be appended as the first row in list

        :returns: A python object (a list of lists of strings, each list is a row, if use_pandas=False, otherwise
            a pandas DataFrame) containing this H2OFrame instance's data.
        """
        if can_use_pandas() and use_pandas:
            import pandas
            return pandas.read_csv(StringIO(self.get_frame_data()), low_memory=False, skip_blank_lines=False)
        from h2o.utils.csv.readers import reader
        frame = [row for row in reader(StringIO(self.get_frame_data()))]
        if not header:
            frame.pop(0)
        return frame

def as_data_frame ( self , use_pandas = True , header = True ) : if can_use_pandas ( ) and use_pandas : import pandas return pandas . read_csv ( StringIO ( self . get_frame_data ( ) ) , low_memory = False , skip_blank_lines = False ) from h2o . utils . csv . readers import reader frame = [ row for row in reader ( StringIO ( self . get_frame_data ( ) ) ) ] if not header : frame . pop ( 0 ) return frame

Obtain the dataset as a python - local object .


def pop(self, i):
        """
        Pop a column from the H2OFrame at index i.

        :param i: The index (int) or name (str) of the column to pop.
        :returns: an H2OFrame containing the column dropped from the current frame; the current frame is modified
            in-place and loses the column.
        """
        if is_type(i, str): i = self.names.index(i)
        col = H2OFrame._expr(expr=ExprNode("cols", self, i))
        old_cache = self._ex._cache
        self._ex = ExprNode("cols", self, -(i + 1))
        self._ex._cache.ncols -= 1
        self._ex._cache.names = old_cache.names[:i] + old_cache.names[i + 1:]
        self._ex._cache.types = {name: old_cache.types[name] for name in self._ex._cache.names}
        self._ex._cache._data = None
        col._ex._cache.ncols = 1
        col._ex._cache.names = [old_cache.names[i]]
        return col

def pop ( self , i ) : if is_type ( i , str ) : i = self . names . index ( i ) col = H2OFrame . _expr ( expr = ExprNode ( "cols" , self , i ) ) old_cache = self . _ex . _cache self . _ex = ExprNode ( "cols" , self , - ( i + 1 ) ) self . _ex . _cache . ncols -= 1 self . _ex . _cache . names = old_cache . names [ : i ] + old_cache . names [ i + 1 : ] self . _ex . _cache . types = { name : old_cache . types [ name ] for name in self . _ex . _cache . names } self . _ex . _cache . _data = None col . _ex . _cache . ncols = 1 col . _ex . _cache . names = [ old_cache . names [ i ] ] return col

Pop a column from the H2OFrame at index i .


def quantile(self, prob=None, combine_method="interpolate", weights_column=None):
        """
        Compute quantiles.

        :param List[float] prob: list of probabilities for which quantiles should be computed.
        :param str combine_method: for even samples this setting determines how to combine quantiles. This can be
            one of ``"interpolate"``, ``"average"``, ``"low"``, ``"high"``.
        :param weights_column: optional weights for each row. If not given, all rows are assumed to have equal
            importance. This parameter can be either the name of column containing the observation weights in
            this frame, or a single-column separate H2OFrame of observation weights.

        :returns: a new H2OFrame containing the quantiles and probabilities.
        """
        if len(self) == 0: return self
        if prob is None: prob = [0.01, 0.1, 0.25, 0.333, 0.5, 0.667, 0.75, 0.9, 0.99]
        if weights_column is None:
            weights_column = "_"
        else:
            assert_is_type(weights_column, str, I(H2OFrame, lambda wc: wc.ncol == 1 and wc.nrow == self.nrow))
            if isinstance(weights_column, H2OFrame):
                merged = self.cbind(weights_column)
                weights_column = merged.names[-1]
                return H2OFrame._expr(expr=ExprNode("quantile", merged, prob, combine_method, weights_column))
        return H2OFrame._expr(expr=ExprNode("quantile", self, prob, combine_method, weights_column))

def quantile ( self , prob = None , combine_method = "interpolate" , weights_column = None ) : if len ( self ) == 0 : return self if prob is None : prob = [ 0.01 , 0.1 , 0.25 , 0.333 , 0.5 , 0.667 , 0.75 , 0.9 , 0.99 ] if weights_column is None : weights_column = "_" else : assert_is_type ( weights_column , str , I ( H2OFrame , lambda wc : wc . ncol == 1 and wc . nrow == self . nrow ) ) if isinstance ( weights_column , H2OFrame ) : merged = self . cbind ( weights_column ) weights_column = merged . names [ - 1 ] return H2OFrame . _expr ( expr = ExprNode ( "quantile" , merged , prob , combine_method , weights_column ) ) return H2OFrame . _expr ( expr = ExprNode ( "quantile" , self , prob , combine_method , weights_column ) )


def concat(self, frames, axis=1):
        """
        Append multiple H2OFrames to this frame, column-wise or row-wise.

        :param List[H2OFrame] frames: list of frames that should be appended to the current frame.
        :param int axis: if 1 then append column-wise (default), if 0 then append row-wise.

        :returns: an H2OFrame of the combined datasets.
        """
        if len(frames) == 0:
            raise ValueError("Input list of frames is empty! Nothing to concat.")

        if axis == 1:
            df = self.cbind(frames)
        else:
            df = self.rbind(frames)
        return df

Append multiple H2OFrames to this frame column - wise or row - wise .


def cbind(self, data):
        """
        Append data to this frame column-wise.

        :param H2OFrame data: append columns of frame ``data`` to the current frame. You can also cbind a number,
            in which case it will get converted into a constant column.

        :returns: new H2OFrame with all frames in ``data`` appended column-wise.
        """
        assert_is_type(data, H2OFrame, numeric, [H2OFrame, numeric])
        frames = [data] if not isinstance(data, list) else data
        new_cols = list(self.columns)
        new_types = dict(self.types)
        for frame in frames:
            if isinstance(frame, H2OFrame):
                if frame.nrow != self.nrow:
                    raise H2OValueError("Cannot bind a dataframe with %d rows to a data frame with %d rows: "
                                        "the number of rows should match" % (frame.nrow, self.nrow))
                new_cols += frame.columns
                new_types.update(frame.types)
            else:
                new_cols += [None]
        unique_cols = set(new_cols)
        fr = H2OFrame._expr(expr=ExprNode("cbind", self, *frames), cache=self._ex._cache)
        fr._ex._cache.ncols = len(new_cols)
        if len(new_cols) == len(unique_cols) and None not in unique_cols:
            fr._ex._cache.names = new_cols
            fr._ex._cache.types = new_types
        else:
            # Invalidate names and types since they contain duplicate / unknown names, and the server will choose those.
            fr._ex._cache.names = None
            fr._ex._cache.types = None
        return fr

Append data to this frame column - wise .


def rbind(self, data):
        """
        Append data to this frame row-wise.

        :param data: an H2OFrame or a list of H2OFrame's to be combined with current frame row-wise.
        :returns: this H2OFrame with all frames in data appended row-wise.
        """
        assert_is_type(data, H2OFrame, [H2OFrame])
        frames = [data] if not isinstance(data, list) else data
        for frame in frames:
            if frame.ncol != self.ncol:
                raise H2OValueError("Cannot row-bind a dataframe with %d columns to a data frame with %d columns: "
                                    "the columns must match" % (frame.ncol, self.ncol))
            if frame.columns != self.columns or frame.types != self.types:
                raise H2OValueError("Column names and types must match for rbind() to work")
        fr = H2OFrame._expr(expr=ExprNode("rbind", self, *frames), cache=self._ex._cache)
        fr._ex._cache.nrows = self.nrow + sum(frame.nrow for frame in frames)
        return fr


def split_frame(self, ratios=None, destination_frames=None, seed=None):
        """
        Split a frame into distinct subsets of size determined by the given ratios.

        The number of subsets is always 1 more than the number of ratios given. Note that
        this does not give an exact split. H2O is designed to be efficient on big data
        using a probabilistic splitting method rather than an exact split. For example
        when specifying a split of 0.75/0.25, H2O will produce a test/train split with
        an expected value of 0.75/0.25 rather than exactly 0.75/0.25. On small datasets,
        the sizes of the resulting splits will deviate from the expected value more than
        on big data, where they will be very close to exact.

        :param List[float] ratios: The fractions of rows for each split.
        :param List[str] destination_frames: The names of the split frames.
        :param int seed: seed for the random number generator

        :returns: A list of H2OFrames
        """
        assert_is_type(ratios, [numeric], None)
        assert_is_type(destination_frames, [str], None)
        assert_is_type(seed, int, None)

        if ratios is None:
            ratios = [0.75]
        if not ratios:
            raise ValueError("Ratios array may not be empty")

        if destination_frames is not None:
            if len(ratios) + 1 != len(destination_frames):
                raise ValueError("The number of provided destination_frames must be one more "
                                 "than the number of provided ratios")

        num_slices = len(ratios) + 1
        boundaries = []

        last_boundary = 0
        i = 0
        while i < num_slices - 1:
            ratio = ratios[i]
            if ratio < 0:
                raise ValueError("Ratio must be greater than 0")
            boundary = last_boundary + ratio
            if boundary >= 1.0:
                raise ValueError("Ratios must add up to less than 1.0")
            boundaries.append(boundary)
            last_boundary = boundary
            i += 1

        splits = []
        tmp_runif = self.runif(seed)
        tmp_runif.frame_id = "%s_splitter" % _py_tmp_key(h2o.connection().session_id)

        i = 0
        while i < num_slices:
            if i == 0:
                # lower_boundary is 0.0
                upper_boundary = boundaries[i]
                tmp_slice = self[(tmp_runif <= upper_boundary), :]
            elif i == num_slices - 1:
                lower_boundary = boundaries[i - 1]
                # upper_boundary is 1.0
                tmp_slice = self[(tmp_runif > lower_boundary), :]
            else:
                lower_boundary = boundaries[i - 1]
                upper_boundary = boundaries[i]
                tmp_slice = self[((tmp_runif > lower_boundary) & (tmp_runif <= upper_boundary)), :]

            if destination_frames is None:
                splits.append(tmp_slice)
            else:
                destination_frame_id = destination_frames[i]
                tmp_slice.frame_id = destination_frame_id
                splits.append(tmp_slice)

            i += 1

        del tmp_runif
        return splits

Split a frame into distinct subsets of size determined by the given ratios .


def group_by(self, by):
        """
        Return a new ``GroupBy`` object using this frame and the desired grouping columns.

        The returned groups are sorted by the natural group-by column sort.

        :param by: The columns to group on (either a single column name, or a list of column names, or
            a list of column indices).
        """
        assert_is_type(by, str, int, [str, int])
        return GroupBy(self, by)

def group_by ( self , by ) : assert_is_type ( by , str , int , [ str , int ] ) return GroupBy ( self , by )

Return a new GroupBy object using this frame and the desired grouping columns .


def fillna(self,method="forward",axis=0,maxlen=1):
        """
        Return a new Frame that fills NA along a given axis and along a given direction with a maximum fill length

        :param method: ``"forward"`` or ``"backward"``
        :param axis:  0 for columnar-wise or 1 for row-wise fill
        :param maxlen: Max number of consecutive NA's to fill

        :return:
        """
        assert_is_type(axis, 0, 1)
        assert_is_type(method,str)
        assert_is_type(maxlen, int)
        return H2OFrame._expr(expr=ExprNode("h2o.fillna",self,method,axis,maxlen))

def fillna ( self , method = "forward" , axis = 0 , maxlen = 1 ) : assert_is_type ( axis , 0 , 1 ) assert_is_type ( method , str ) assert_is_type ( maxlen , int ) return H2OFrame . _expr ( expr = ExprNode ( "h2o.fillna" , self , method , axis , maxlen ) )


def impute(self, column=-1, method="mean", combine_method="interpolate", by=None, group_by_frame=None, values=None):
        """
        Impute missing values into the frame, modifying it in-place.

        :param int column: Index of the column to impute, or -1 to impute the entire frame.
        :param str method: The method of imputation: ``"mean"``, ``"median"``, or ``"mode"``.
        :param str combine_method: When the method is ``"median"``, this setting dictates how to combine quantiles
            for even samples. One of ``"interpolate"``, ``"average"``, ``"low"``, ``"high"``.
        :param by: The list of columns to group on.
        :param H2OFrame group_by_frame: Impute the values with this pre-computed grouped frame.
        :param List values: The list of impute values, one per column. None indicates to skip the column.

        :returns: A list of values used in the imputation or the group-by result used in imputation.
        """
        if is_type(column, str): column = self.names.index(column)
        if is_type(by, str):     by = self.names.index(by)

        if values is None:
            values = "_"
        else:
            assert len(values) == len(self.columns), "Length of values does not match length of columns"
            # convert string values to categorical num values
            values2 = []
            for i in range(0,len(values)):
                if self.type(i) == "enum":
                    try:
                        values2.append(self.levels()[i].index(values[i]))
                    except:
                        raise H2OValueError("Impute value of: " + values[i] + " not found in existing levels of"
                                            " column: " + self.col_names[i])
                else:
                    values2.append(values[i])
            values = values2
        if group_by_frame is None: group_by_frame = "_"

        # This code below is needed to ensure the frame (self) exists on the server. Without it, self._ex._cache.fill()
        # fails with an assertion that ._id is None.
        # This code should be removed / reworked once we have a more consistent strategy of dealing with frames.
        self._ex._eager_frame()

        if by is not None or group_by_frame is not "_":
            res = H2OFrame._expr(
                expr=ExprNode("h2o.impute", self, column, method, combine_method, by, group_by_frame, values))._frame()
        else:
            res = ExprNode("h2o.impute", self, column, method, combine_method, by, group_by_frame,
                           values)._eager_scalar()

        self._ex._cache.flush()
        self._ex._cache.fill(10)
        return res

Impute missing values into the frame modifying it in - place .


def merge(self, other, all_x=False, all_y=False, by_x=None, by_y=None, method="auto"):
        """
        Merge two datasets based on common column names.  We do not support all_x=True and all_y=True.
        Only one can be True or none is True.  The default merge method is auto and it will default to the
        radix method.  The radix method will return the correct merge result regardless of duplicated rows
         in the right frame.  In addition, the radix method can perform merge even if you have string columns
         in your frames.  If there are duplicated rows in your rite frame, they will not be included if you use
        the hash method.  The hash method cannot perform merge if you have string columns in your left frame.
        Hence, we consider the radix method superior to the hash method and is the default method to use.

        :param H2OFrame other: The frame to merge to the current one. By default, must have at least one column in common with
            this frame, and all columns in common are used as the merge key.  If you want to use only a subset of the
            columns in common, rename the other columns so the columns are unique in the merged result.
        :param bool all_x: If True, include all rows from the left/self frame
        :param bool all_y: If True, include all rows from the right/other frame
        :param by_x: list of columns in the current frame to use as a merge key.
        :param by_y: list of columns in the ``other`` frame to use as a merge key. Should have the same number of
            columns as in the ``by_x`` list.
        :param method: string representing the merge method, one of auto(default), radix or hash.

        :returns: New H2OFrame with the result of merging the current frame with the ``other`` frame.
        """

        if by_x is None and by_y is None:
            common_names = list(set(self.names) & set(other.names))
            if not common_names:
                raise H2OValueError("No columns in common to merge on!")

        if by_x is None:
            by_x = [self.names.index(c) for c in common_names]
        else:
            by_x = _getValidCols(by_x,self)

        if by_y is None:
            by_y = [other.names.index(c) for c in common_names]
        else:
            by_y = _getValidCols(by_y,other)

        return H2OFrame._expr(expr=ExprNode("merge", self, other, all_x, all_y, by_x, by_y, method))

def
merge
(
self
,
other
,
all_x
=
False
,
all_y
=
False
,
by_x
=
,
by_y
=
,
method
=
"auto"
)
:
if
by_x
is
and
by_y
is
:
common_names
=
list
(
set
(
self
.
names
)
&
set
(
other
.
names
)
)
if
not
common_names
:
raise
H2OValueError
(
"No columns in common to merge on!"
)
if
by_x
is
:
by_x
=
[
self
.
names
.
index
(
c
)
for
c
in
common_names
]
else
:
by_x
=
_getValidCols
(
by_x
,
self
)
if
by_y
is
:
by_y
=
[
other
.
names
.
index
(
c
)
for
c
in
common_names
]
else
:
by_y
=
_getValidCols
(
by_y
,
other
)
return
H2OFrame
.
_expr
(
expr
=
ExprNode
(
"merge"
,
self
,
other
,
all_x
,
all_y
,
by_x
,
by_y
,
method
)
)

Merge two datasets based on common column names . We do not support all_x = True and all_y = True . Only one can be True or none is True . The default merge method is auto and it will default to the radix method . The radix method will return the correct merge result regardless of duplicated rows in the right frame . In addition the radix method can perform merge even if you have string columns in your frames . If there are duplicated rows in your rite frame they will not be included if you use the hash method . The hash method cannot perform merge if you have string columns in your left frame . Hence we consider the radix method superior to the hash method and is the default method to use .


def relevel(self, y):
        """
        Reorder levels of an H2O factor for one single column of a H2O frame

        The levels of a factor are reordered such that the reference level is at level 0, all remaining levels are
        moved down as needed.

        :param str y: The reference level
        :returns: New reordered factor column
        """
        return H2OFrame._expr(expr=ExprNode("relevel", self, quote(y)))

def relevel ( self , y ) : return H2OFrame . _expr ( expr = ExprNode ( "relevel" , self , quote ( y ) ) )


def insert_missing_values(self, fraction=0.1, seed=None):
        """
        Insert missing values into the current frame, modifying it in-place.

        Randomly replaces a user-specified fraction of entries in a H2O dataset with missing
        values.

        :param float fraction: A number between 0 and 1 indicating the fraction of entries to replace with missing.
        :param int seed: The seed for the random number generator used to determine which values to make missing.

        :returns: the original H2OFrame with missing values inserted.
        """
        kwargs = {}
        kwargs['dataset'] = self.frame_id  # Eager; forces eval now for following REST call
        kwargs['fraction'] = fraction
        if seed is not None: kwargs['seed'] = seed
        job = {}
        job['job'] = h2o.api("POST /3/MissingInserter", data=kwargs)
        H2OJob(job, job_type=("Insert Missing Values")).poll()
        self._ex._cache.flush()
        return self

def
insert_missing_values
(
self
,
fraction
=
0.1
,
seed
=
)
:
kwargs
=
{
}
kwargs
[
'dataset'
]
=
self
.
frame_id
kwargs
[
'fraction'
]
=
fraction
if
seed
is
not
:
kwargs
[
'seed'
]
=
seed
job
=
{
}
job
[
'job'
]
=
h2o
.
api
(
"POST /3/MissingInserter"
,
data
=
kwargs
)
H2OJob
(
job
,
job_type
=
(
"Insert Missing Values"
)
)
.
poll
(
)
self
.
_ex
.
_cache
.
flush
(
)
return
self

Insert missing values into the current frame modifying it in - place .


def var(self, y=None, na_rm=False, use=None):
        """
        Compute the variance-covariance matrix of one or two H2OFrames.

        :param H2OFrame y: If this parameter is given, then a covariance  matrix between the columns of the target
            frame and the columns of ``y`` is computed. If this parameter is not provided then the covariance matrix
            of the target frame is returned. If target frame has just a single column, then return the scalar variance
            instead of the matrix. Single rows are treated as single columns.
        :param str use: A string indicating how to handle missing values. This could be one of the following:

            - ``"everything"``: outputs NaNs whenever one of its contributing observations is missing
            - ``"all.obs"``: presence of missing observations will throw an error
            - ``"complete.obs"``: discards missing values along with all observations in their rows so that only
              complete observations are used
        :param bool na_rm: an alternative to ``use``: when this is True then default value for ``use`` is
            ``"everything"``; and if False then default ``use`` is ``"complete.obs"``. This parameter has no effect
            if ``use`` is given explicitly.

        :returns: An H2OFrame of the covariance matrix of the columns of this frame (if ``y`` is not given),
            or with the columns of ``y`` (if ``y`` is given). However when this frame and ``y`` are both single rows
            or single columns, then the variance is returned as a scalar.
        """
        symmetric = False
        if y is None:
            y = self
            symmetric = True
        if use is None: use = "complete.obs" if na_rm else "everything"
        if self.nrow == 1 or (self.ncol == 1 and y.ncol == 1):
            return ExprNode("var", self, y, use, symmetric)._eager_scalar()
        return H2OFrame._expr(expr=ExprNode("var", self, y, use, symmetric))._frame()

def var ( self , y = None , na_rm = False , use = None ) : symmetric = False if y is None : y = self symmetric = True if use is None : use = "complete.obs" if na_rm else "everything" if self . nrow == 1 or ( self . ncol == 1 and y . ncol == 1 ) : return ExprNode ( "var" , self , y , use , symmetric ) . _eager_scalar ( ) return H2OFrame . _expr ( expr = ExprNode ( "var" , self , y , use , symmetric ) ) . _frame ( )

Compute the variance - covariance matrix of one or two H2OFrames .


def cor(self, y=None, na_rm=False, use=None):
        """
        Compute the correlation matrix of one or two H2OFrames.

        :param H2OFrame y: If this parameter is provided, then compute correlation between the columns of ``y``
            and the columns of the current frame. If this parameter is not given, then just compute the correlation
            matrix for the columns of the current frame.
        :param str use: A string indicating how to handle missing values. This could be one of the following:

            - ``"everything"``: outputs NaNs whenever one of its contributing observations is missing
            - ``"all.obs"``: presence of missing observations will throw an error
            - ``"complete.obs"``: discards missing values along with all observations in their rows so that only
              complete observations are used
        :param bool na_rm: an alternative to ``use``: when this is True then default value for ``use`` is
            ``"everything"``; and if False then default ``use`` is ``"complete.obs"``. This parameter has no effect
            if ``use`` is given explicitly.

        :returns: An H2OFrame of the correlation matrix of the columns of this frame (if ``y`` is not given),
            or with the columns of ``y`` (if ``y`` is given). However when this frame and ``y`` are both single rows
            or single columns, then the correlation is returned as a scalar.
        """
        assert_is_type(y, H2OFrame, None)
        assert_is_type(na_rm, bool)
        assert_is_type(use, None, "everything", "all.obs", "complete.obs")
        if y is None:
            y = self
        if use is None: use = "complete.obs" if na_rm else "everything"
        if self.nrow == 1 or (self.ncol == 1 and y.ncol == 1): return ExprNode("cor", self, y, use)._eager_scalar()
        return H2OFrame._expr(expr=ExprNode("cor", self, y, use))._frame()

def cor ( self , y = None , na_rm = False , use = None ) : assert_is_type ( y , H2OFrame , None ) assert_is_type ( na_rm , bool ) assert_is_type ( use , None , "everything" , "all.obs" , "complete.obs" ) if y is None : y = self if use is None : use = "complete.obs" if na_rm else "everything" if self . nrow == 1 or ( self . ncol == 1 and y . ncol == 1 ) : return ExprNode ( "cor" , self , y , use ) . _eager_scalar ( ) return H2OFrame . _expr ( expr = ExprNode ( "cor" , self , y , use ) ) . _frame ( )

Compute the correlation matrix of one or two H2OFrames .