def _build(self, model):
    """Creates `dist_fn`, `dist_fn_wrapped`, `dist_fn_args`, `dist_fn_name`."""
    if not _is_dict_like(model):
      raise TypeError('`model` must be convertible to `dict` (saw: {}).'.format(
          type(model).__name__))
    [
        self._dist_fn,
        self._dist_fn_wrapped,
        self._dist_fn_args,
        self._dist_fn_name,  # JointDistributionSequential doesn't have this.
    ] = _prob_chain_rule_flatten(model)


def variational_loss(self,
                       observations,
                       observation_index_points=None,
                       kl_weight=1.,
                       name='variational_loss'):
    """Variational loss for the VGP.

    Given `observations` and `observation_index_points`, compute the
    negative variational lower bound as specified in [Hensman, 2013][1].

    Args:
      observations: `float` `Tensor` representing collection, or batch of
        collections, of observations corresponding to
        `observation_index_points`. Shape has the form `[b1, ..., bB, e]`, which
        must be brodcastable with the batch and example shapes of
        `observation_index_points`. The batch shape `[b1, ..., bB]` must be
        broadcastable with the shapes of all other batched parameters
        (`kernel.batch_shape`, `observation_index_points`, etc.).
      observation_index_points: `float` `Tensor` representing finite (batch of)
        vector(s) of points where observations are defined. Shape has the
        form `[b1, ..., bB, e1, f1, ..., fF]` where `F` is the number of feature
        dimensions and must equal `kernel.feature_ndims` and `e1` is the number
        (size) of index points in each batch (we denote it `e1` to distinguish
        it from the numer of inducing index points, denoted `e2` below). If
        set to `None` uses `index_points` as the origin for observations.
        Default value: None.
      kl_weight: Amount by which to scale the KL divergence loss between prior
        and posterior.
        Default value: 1.
      name: Python `str` name prefixed to Ops created by this class.
        Default value: "GaussianProcess".
    Returns:
      loss: Scalar tensor representing the negative variational lower bound.
        Can be directly used in a `tf.Optimizer`.
    Raises:
      ValueError: if `mean_fn` is not `None` and is not callable.

    #### References

    [1]: Hensman, J., Lawrence, N. "Gaussian Processes for Big Data", 2013
         https://arxiv.org/abs/1309.6835
    """

    with tf.name_scope(name or 'variational_gp_loss'):
      if observation_index_points is None:
        observation_index_points = self._index_points
      observation_index_points = tf.convert_to_tensor(
          value=observation_index_points, dtype=self._dtype,
          name='observation_index_points')
      observations = tf.convert_to_tensor(
          value=observations, dtype=self._dtype, name='observations')
      kl_weight = tf.convert_to_tensor(
          value=kl_weight, dtype=self._dtype,
          name='kl_weight')

      # The variational loss is a negative ELBO. The ELBO can be broken down
      # into three terms:
      #  1. a likelihood term
      #  2. a trace term arising from the covariance of the posterior predictive

      kzx = self.kernel.matrix(self._inducing_index_points,
                               observation_index_points)

      kzx_linop = tf.linalg.LinearOperatorFullMatrix(kzx)
      loc = (self._mean_fn(observation_index_points) +
             kzx_linop.matvec(self._kzz_inv_varloc, adjoint=True))

      likelihood = independent.Independent(
          normal.Normal(
              loc=loc,
              scale=tf.sqrt(self._observation_noise_variance + self._jitter),
              name='NormalLikelihood'),
          reinterpreted_batch_ndims=1)
      obs_ll = likelihood.log_prob(observations)

      chol_kzz_linop = tf.linalg.LinearOperatorLowerTriangular(self._chol_kzz)
      chol_kzz_inv_kzx = chol_kzz_linop.solve(kzx)
      kzz_inv_kzx = chol_kzz_linop.solve(chol_kzz_inv_kzx, adjoint=True)

      kxx_diag = tf.linalg.diag_part(
          self.kernel.matrix(
              observation_index_points, observation_index_points))
      ktilde_trace_term = (
          tf.reduce_sum(input_tensor=kxx_diag, axis=-1) -
          tf.reduce_sum(input_tensor=chol_kzz_inv_kzx ** 2, axis=[-2, -1]))

      # Tr(SB)
      # where S = A A.T, A = variational_inducing_observations_scale
      # and B = Kzz^-1 Kzx Kzx.T Kzz^-1
      #
      # Now Tr(SB) = Tr(A A.T Kzz^-1 Kzx Kzx.T Kzz^-1)
      #            = Tr(A.T Kzz^-1 Kzx Kzx.T Kzz^-1 A)
      #            = sum_ij (A.T Kzz^-1 Kzx)_{ij}^2
      other_trace_term = tf.reduce_sum(
          input_tensor=(
              self._variational_inducing_observations_posterior.scale.matmul(
                  kzz_inv_kzx) ** 2),
          axis=[-2, -1])

      trace_term = (.5 * (ktilde_trace_term + other_trace_term) /
                    self._observation_noise_variance)

      inducing_prior = gaussian_process.GaussianProcess(
          kernel=self._kernel,
          mean_fn=self._mean_fn,
          index_points=self._inducing_index_points,
          observation_noise_variance=self._observation_noise_variance)

      kl_term = kl_weight * kullback_leibler.kl_divergence(
          self._variational_inducing_observations_posterior,
          inducing_prior)

      lower_bound = (obs_ll - trace_term - kl_term)

      return -tf.reduce_mean(input_tensor=lower_bound)


def optimal_variational_posterior(
      kernel,
      inducing_index_points,
      observation_index_points,
      observations,
      observation_noise_variance,
      mean_fn=None,
      jitter=1e-6,
      name=None):
    """Model selection for optimal variational hyperparameters.

    Given the full training set (parameterized by `observations` and
    `observation_index_points`), compute the optimal variational
    location and scale for the VGP. This is based of the method suggested
    in [Titsias, 2009][1].

    Args:
      kernel: `PositiveSemidefiniteKernel`-like instance representing the
        GP's covariance function.
      inducing_index_points: `float` `Tensor` of locations of inducing points in
        the index set. Shape has the form `[b1, ..., bB, e2, f1, ..., fF]`, just
        like `observation_index_points`. The batch shape components needn't be
        identical to those of `observation_index_points`, but must be broadcast
        compatible with them.
      observation_index_points: `float` `Tensor` representing finite (batch of)
        vector(s) of points where observations are defined. Shape has the
        form `[b1, ..., bB, e1, f1, ..., fF]` where `F` is the number of feature
        dimensions and must equal `kernel.feature_ndims` and `e1` is the number
        (size) of index points in each batch (we denote it `e1` to distinguish
        it from the numer of inducing index points, denoted `e2` below).
      observations: `float` `Tensor` representing collection, or batch of
        collections, of observations corresponding to
        `observation_index_points`. Shape has the form `[b1, ..., bB, e]`, which
        must be brodcastable with the batch and example shapes of
        `observation_index_points`. The batch shape `[b1, ..., bB]` must be
        broadcastable with the shapes of all other batched parameters
        (`kernel.batch_shape`, `observation_index_points`, etc.).
      observation_noise_variance: `float` `Tensor` representing the variance
        of the noise in the Normal likelihood distribution of the model. May be
        batched, in which case the batch shape must be broadcastable with the
        shapes of all other batched parameters (`kernel.batch_shape`,
        `index_points`, etc.).
        Default value: `0.`
      mean_fn: Python `callable` that acts on index points to produce a (batch
        of) vector(s) of mean values at those index points. Takes a `Tensor` of
        shape `[b1, ..., bB, f1, ..., fF]` and returns a `Tensor` whose shape is
        (broadcastable with) `[b1, ..., bB]`. Default value: `None` implies
        constant zero function.
      jitter: `float` scalar `Tensor` added to the diagonal of the covariance
        matrix to ensure positive definiteness of the covariance matrix.
        Default value: `1e-6`.
      name: Python `str` name prefixed to Ops created by this class.
        Default value: "optimal_variational_posterior".
    Returns:
      loc, scale: Tuple representing the variational location and scale.
    Raises:
      ValueError: if `mean_fn` is not `None` and is not callable.

    #### References

    [1]: Titsias, M. "Variational Model Selection for Sparse Gaussian Process
         Regression", 2009.
         http://proceedings.mlr.press/v5/titsias09a/titsias09a.pdf
    """

    with tf.name_scope(name or 'optimal_variational_posterior'):
      dtype = dtype_util.common_dtype(
          [inducing_index_points,
           observation_index_points,
           observations,
           observation_noise_variance,
           jitter], tf.float32)

      inducing_index_points = tf.convert_to_tensor(
          value=inducing_index_points,
          dtype=dtype, name='inducing_index_points')
      observation_index_points = tf.convert_to_tensor(
          value=observation_index_points, dtype=dtype,
          name='observation_index_points')
      observations = tf.convert_to_tensor(
          value=observations, dtype=dtype, name='observations')
      observation_noise_variance = tf.convert_to_tensor(
          value=observation_noise_variance,
          dtype=dtype,
          name='observation_noise_variance')
      jitter = tf.convert_to_tensor(
          value=jitter, dtype=dtype, name='jitter')

      # Default to a constant zero function.
      if mean_fn is None:
        mean_fn = lambda x: tf.zeros([1], dtype=dtype)
      else:
        if not callable(mean_fn):
          raise ValueError('`mean_fn` must be a Python callable')

      # z are the inducing points and x are the observation index points.
      kzz = kernel.matrix(inducing_index_points, inducing_index_points)
      kzx = kernel.matrix(inducing_index_points, observation_index_points)

      noise_var_inv = tf.math.reciprocal(observation_noise_variance)

      sigma_inv = _add_diagonal_shift(
          kzz + noise_var_inv * tf.matmul(kzx, kzx, adjoint_b=True),
          jitter)

      chol_sigma_inv = tf.linalg.cholesky(sigma_inv)

      kzx_lin_op = tf.linalg.LinearOperatorFullMatrix(kzx)
      kzx_obs = kzx_lin_op.matvec(
          observations - mean_fn(observation_index_points))
      kzz_lin_op = tf.linalg.LinearOperatorFullMatrix(kzz)
      loc = (mean_fn(inducing_index_points) +
             noise_var_inv * kzz_lin_op.matvec(
                 _solve_cholesky_factored_system_vec(chol_sigma_inv, kzx_obs)))

      chol_sigma_inv_lin_op = tf.linalg.LinearOperatorLowerTriangular(
          chol_sigma_inv)
      scale = chol_sigma_inv_lin_op.solve(kzz)

      return loc, scale

def
optimal_variational_posterior
(
kernel
,
inducing_index_points
,
observation_index_points
,
observations
,
observation_noise_variance
,
mean_fn
=
,
jitter
=
1e-6
,
name
=
)
:
with
tf
.
name_scope
(
name
or
'optimal_variational_posterior'
)
:
dtype
=
dtype_util
.
common_dtype
(
[
inducing_index_points
,
observation_index_points
,
observations
,
observation_noise_variance
,
jitter
]
,
tf
.
float32
)
inducing_index_points
=
tf
.
convert_to_tensor
(
value
=
inducing_index_points
,
dtype
=
dtype
,
name
=
'inducing_index_points'
)
observation_index_points
=
tf
.
convert_to_tensor
(
value
=
observation_index_points
,
dtype
=
dtype
,
name
=
'observation_index_points'
)
observations
=
tf
.
convert_to_tensor
(
value
=
observations
,
dtype
=
dtype
,
name
=
'observations'
)
observation_noise_variance
=
tf
.
convert_to_tensor
(
value
=
observation_noise_variance
,
dtype
=
dtype
,
name
=
'observation_noise_variance'
)
jitter
=
tf
.
convert_to_tensor
(
value
=
jitter
,
dtype
=
dtype
,
name
=
'jitter'
)
if
mean_fn
is
:
mean_fn
=
lambda
x
:
tf
.
zeros
(
[
1
]
,
dtype
=
dtype
)
else
:
if
not
callable
(
mean_fn
)
:
raise
ValueError
(
'`mean_fn` must be a Python callable'
)
kzz
=
kernel
.
matrix
(
inducing_index_points
,
inducing_index_points
)
kzx
=
kernel
.
matrix
(
inducing_index_points
,
observation_index_points
)
noise_var_inv
=
tf
.
math
.
reciprocal
(
observation_noise_variance
)
sigma_inv
=
_add_diagonal_shift
(
kzz
+
noise_var_inv
*
tf
.
matmul
(
kzx
,
kzx
,
adjoint_b
=
True
)
,
jitter
)
chol_sigma_inv
=
tf
.
linalg
.
cholesky
(
sigma_inv
)
kzx_lin_op
=
tf
.
linalg
.
LinearOperatorFullMatrix
(
kzx
)
kzx_obs
=
kzx_lin_op
.
matvec
(
observations
mean_fn
(
observation_index_points
)
)
kzz_lin_op
=
tf
.
linalg
.
LinearOperatorFullMatrix
(
kzz
)
loc
=
(
mean_fn
(
inducing_index_points
)
+
noise_var_inv
*
kzz_lin_op
.
matvec
(
_solve_cholesky_factored_system_vec
(
chol_sigma_inv
,
kzx_obs
)
)
)
chol_sigma_inv_lin_op
=
tf
.
linalg
.
LinearOperatorLowerTriangular
(
chol_sigma_inv
)
scale
=
chol_sigma_inv_lin_op
.
solve
(
kzz
)
return
loc
,
scale

Model selection for optimal variational hyperparameters .


def build_is_last_day_of_season(num_steps_per_season):
  """Build utility method to compute whether the season is changing."""
  num_steps_per_cycle = np.sum(num_steps_per_season)
  changepoints = np.cumsum(np.ravel(num_steps_per_season)) - 1
  def is_last_day_of_season(t):
    t_ = dist_util.maybe_get_static_value(t)
    if t_ is not None:  # static case
      step_in_cycle = t_ % num_steps_per_cycle
      return any(step_in_cycle == changepoints)
    else:
      step_in_cycle = tf.math.floormod(t, num_steps_per_cycle)
      return tf.reduce_any(
          input_tensor=tf.equal(step_in_cycle, changepoints))
  return is_last_day_of_season

def build_is_last_day_of_season ( num_steps_per_season ) : num_steps_per_cycle = np . sum ( num_steps_per_season ) changepoints = np . cumsum ( np . ravel ( num_steps_per_season ) ) - 1 def is_last_day_of_season ( t ) : t_ = dist_util . maybe_get_static_value ( t ) if t_ is not None : step_in_cycle = t_ % num_steps_per_cycle return any ( step_in_cycle == changepoints ) else : step_in_cycle = tf . math . floormod ( t , num_steps_per_cycle ) return tf . reduce_any ( input_tensor = tf . equal ( step_in_cycle , changepoints ) ) return is_last_day_of_season

Build utility method to compute whether the season is changing .


def build_effects_to_residuals_matrix(num_seasons, dtype):
  """Build change-of-basis matrices for constrained seasonal effects.

  This method builds the matrix that transforms seasonal effects into
  effect residuals (differences from the mean effect), and additionally
  projects these residuals onto the subspace where the mean effect is zero.

  See `ConstrainedSeasonalStateSpaceModel` for mathematical details.

  Args:
    num_seasons: scalar `int` number of seasons.
    dtype: TensorFlow `dtype` for the returned values.
  Returns:
    effects_to_residuals: `Tensor` of shape
      `[num_seasons-1, num_seasons]`, such that `differences_from_mean_effect =
      matmul(effects_to_residuals, seasonal_effects)`.  In the
      notation of `ConstrainedSeasonalStateSpaceModel`, this is
      `effects_to_residuals = P * R`.
    residuals_to_effects: the (pseudo)-inverse of the above; a
      `Tensor` of shape `[num_seasons, num_seasons-1]`. In the
      notation of `ConstrainedSeasonalStateSpaceModel`, this is
      `residuals_to_effects = R^{-1} * P'`.
  """

  # Build the matrix that converts effects `e_i` into differences from the mean
  # effect `(e_i - sum(e_i)) / num_seasons`, with the mean effect in the last
  # row so that the transformation is invertible.
  effects_to_residuals_fullrank = np.eye(num_seasons) - 1./num_seasons
  effects_to_residuals_fullrank[-1, :] = 1./num_seasons  # compute mean effect
  residuals_to_effects_fullrank = np.linalg.inv(effects_to_residuals_fullrank)

  # Drop the final dimension, effectively setting the mean effect to zero.
  effects_to_residuals = effects_to_residuals_fullrank[:-1, :]
  residuals_to_effects = residuals_to_effects_fullrank[:, :-1]

  # Return Tensor values of the specified dtype.
  effects_to_residuals = tf.cast(
      effects_to_residuals, dtype=dtype, name='effects_to_residuals')
  residuals_to_effects = tf.cast(
      residuals_to_effects, dtype=dtype, name='residuals_to_effects')

  return effects_to_residuals, residuals_to_effects

def build_effects_to_residuals_matrix ( num_seasons , dtype ) : effects_to_residuals_fullrank = np . eye ( num_seasons ) - 1. / num_seasons effects_to_residuals_fullrank [ - 1 , : ] = 1. / num_seasons residuals_to_effects_fullrank = np . linalg . inv ( effects_to_residuals_fullrank ) effects_to_residuals = effects_to_residuals_fullrank [ : - 1 , : ] residuals_to_effects = residuals_to_effects_fullrank [ : , : - 1 ] effects_to_residuals = tf . cast ( effects_to_residuals , dtype = dtype , name = 'effects_to_residuals' ) residuals_to_effects = tf . cast ( residuals_to_effects , dtype = dtype , name = 'residuals_to_effects' ) return effects_to_residuals , residuals_to_effects

Build change - of - basis matrices for constrained seasonal effects .


def build_seasonal_transition_matrix(
    num_seasons, is_last_day_of_season, dtype,
    basis_change_matrix=None, basis_change_matrix_inv=None):
  """Build a function computing transitions for a seasonal effect model."""

  with tf.compat.v1.name_scope('build_seasonal_transition_matrix'):
    # If the season is changing, the transition matrix permutes the latent
    # state to shift all seasons up by a dimension, and sends the current
    # season's effect to the bottom.
    seasonal_permutation = np.concatenate(
        [np.arange(1, num_seasons), [0]], axis=0)
    seasonal_permutation_matrix = tf.constant(
        np.eye(num_seasons)[seasonal_permutation], dtype=dtype)

    # Optionally transform the transition matrix into a reparameterized space,
    # enforcing the zero-sum constraint for ConstrainedSeasonalStateSpaceModel.
    if basis_change_matrix is not None:
      seasonal_permutation_matrix = tf.matmul(
          basis_change_matrix,
          tf.matmul(seasonal_permutation_matrix, basis_change_matrix_inv))

    identity_matrix = tf.eye(
        tf.shape(input=seasonal_permutation_matrix)[-1], dtype=dtype)

    def seasonal_transition_matrix(t):
      return tf.linalg.LinearOperatorFullMatrix(
          matrix=dist_util.pick_scalar_condition(
              is_last_day_of_season(t),
              seasonal_permutation_matrix,
              identity_matrix))

  return seasonal_transition_matrix

def build_seasonal_transition_matrix ( num_seasons , is_last_day_of_season , dtype , basis_change_matrix = None , basis_change_matrix_inv = None ) : with tf . compat . v1 . name_scope ( 'build_seasonal_transition_matrix' ) : seasonal_permutation = np . concatenate ( [ np . arange ( 1 , num_seasons ) , [ 0 ] ] , axis = 0 ) seasonal_permutation_matrix = tf . constant ( np . eye ( num_seasons ) [ seasonal_permutation ] , dtype = dtype ) if basis_change_matrix is not None : seasonal_permutation_matrix = tf . matmul ( basis_change_matrix , tf . matmul ( seasonal_permutation_matrix , basis_change_matrix_inv ) ) identity_matrix = tf . eye ( tf . shape ( input = seasonal_permutation_matrix ) [ - 1 ] , dtype = dtype ) def seasonal_transition_matrix ( t ) : return tf . linalg . LinearOperatorFullMatrix ( matrix = dist_util . pick_scalar_condition ( is_last_day_of_season ( t ) , seasonal_permutation_matrix , identity_matrix ) ) return seasonal_transition_matrix

Build a function computing transitions for a seasonal effect model .


def build_seasonal_transition_noise(
    drift_scale, num_seasons, is_last_day_of_season):
  """Build the transition noise model for a SeasonalStateSpaceModel."""

  # If the current season has just ended, increase the variance of its effect
  # following drift_scale. (the just-ended seasonal effect will always be the
  # bottom element of the vector). Otherwise, do nothing.
  drift_scale_diag = tf.stack(
      [tf.zeros_like(drift_scale)] * (num_seasons - 1) + [drift_scale],
      axis=-1)
  def seasonal_transition_noise(t):
    noise_scale_diag = dist_util.pick_scalar_condition(
        is_last_day_of_season(t),
        drift_scale_diag,
        tf.zeros_like(drift_scale_diag))
    return tfd.MultivariateNormalDiag(
        loc=tf.zeros(num_seasons, dtype=drift_scale.dtype),
        scale_diag=noise_scale_diag)
  return seasonal_transition_noise

def build_seasonal_transition_noise ( drift_scale , num_seasons , is_last_day_of_season ) : drift_scale_diag = tf . stack ( [ tf . zeros_like ( drift_scale ) ] * ( num_seasons - 1 ) + [ drift_scale ] , axis = - 1 ) def seasonal_transition_noise ( t ) : noise_scale_diag = dist_util . pick_scalar_condition ( is_last_day_of_season ( t ) , drift_scale_diag , tf . zeros_like ( drift_scale_diag ) ) return tfd . MultivariateNormalDiag ( loc = tf . zeros ( num_seasons , dtype = drift_scale . dtype ) , scale_diag = noise_scale_diag ) return seasonal_transition_noise

Build the transition noise model for a SeasonalStateSpaceModel .


def build_constrained_seasonal_transition_noise(
    drift_scale, num_seasons, is_last_day_of_season):
  """Build transition noise distribution for a ConstrainedSeasonalSSM."""

  # Conceptually, this method takes the noise covariance on effects L @ L'
  # computed by `build_seasonal_transition_noise`, with scale factor
  #       L = [ 0, 0, ..., 0
  #             ...
  #             0, 0, ..., drift_scale],
  # and transforms it to act on the constrained-residual representation.
  #
  # The resulting noise covariance M @ M' is equivalent to
  #    M @ M' = effects_to_residuals @ LL' @ residuals_to_effects
  # where `@` is matrix multiplication. However because this matrix is
  # rank-deficient, we can't take its Cholesky decomposition directly, so we'll
  # construct its lower-triangular scale factor `M` by hand instead.
  #
  # Concretely, let `M = P @ R @ L` be the scale factor in the
  # transformed space, with matrices `R`, `P` applying the reparameterization
  # and zero-mean constraint respectively as defined in the
  # "Mathematical Details" section of `ConstrainedSeasonalStateSpaceModel`. It's
  # easy to see (*) that the implied covariance
  # `M @ M' = P @ R @ L @ L' @ R' @ P'` is just the constant matrix
  #  `M @ M' = [ 1, 1, ..., 1, 0
  #              1, 1, ..., 1, 0
  #              ...
  #              1, 1, ..., 1, 0
  #              0, 0, ..., 0, 0] * (drift_scale / num_seasons)**2`
  # with zeros in the final row and column. So we can directly construct
  # the lower-triangular factor
  #  `Q = [ 1, 0, ...  0
  #         1, 0, ..., 0
  #         ...
  #         1, 0, ..., 0
  #         0, 0, ..., 0 ] * drift_scale/num_seasons`
  # such that Q @ Q' = M @ M'. In practice, we don't reify the final row and
  # column full of zeroes, i.e., we construct
  # `Q[:num_seasons-1, :num_seasons-1]` as the scale-TriL covariance factor.
  #
  # (*) Argument: `L` is zero everywhere but the last column, so `R @ L` will be
  # too. Since the last column of `R` is the constant `-1/num_seasons`, `R @ L`
  # is simply the matrix with constant `-drift_scale/num_seasons` in the final
  # column (except the final row, which is negated) and zero in all other
  # columns, and `M = P @ R @ L` additionally zeroes out the final row. Then
  # M @ M' is just the outer product of that final column with itself (since all
  # other columns are zero), which gives the matrix shown above.

  drift_scale_tril_nonzeros = tf.concat([
      tf.ones([num_seasons - 1, 1], dtype=drift_scale.dtype),
      tf.zeros([num_seasons - 1, num_seasons - 2], dtype=drift_scale.dtype)],
                                        axis=-1)
  drift_scale_tril = (drift_scale_tril_nonzeros *
                      drift_scale[..., tf.newaxis, tf.newaxis] / num_seasons)

  # Inject transition noise iff it is the last day of the season.
  def seasonal_transition_noise(t):
    noise_scale_tril = dist_util.pick_scalar_condition(
        is_last_day_of_season(t),
        drift_scale_tril,
        tf.zeros_like(drift_scale_tril))
    return tfd.MultivariateNormalTriL(
        loc=tf.zeros(num_seasons-1, dtype=drift_scale.dtype),
        scale_tril=noise_scale_tril)
  return seasonal_transition_noise

Build transition noise distribution for a ConstrainedSeasonalSSM .


def _is_empty_observation_data(
    feature_ndims, observation_index_points, observations):
  """Returns `True` if given observation data is empty.

  Emptiness means either
    1. Both `observation_index_points` and `observations` are `None`, or
    2. the "number of observations" shape is 0. The shape of
    `observation_index_points` is `[..., N, f1, ..., fF]`, where `N` is the
    number of observations and the `f`s are feature dims. Thus, we look at the
    shape element just to the left of the leftmost feature dim. If that shape is
    zero, we consider the data empty.

  We don't check the shape of observations; validations are checked elsewhere in
  the calling code, to ensure these shapes are consistent.

  Args:
    feature_ndims: the number of feature dims, as reported by the GP kernel.
    observation_index_points: the observation data locations in the index set.
    observations: the observation data.

  Returns:
    is_empty: True if the data were deemed to be empty.
  """
  # If both input locations and observations are `None`, we consider this
  # "empty" observation data.
  if observation_index_points is None and observations is None:
    return True
  num_obs = tf.compat.dimension_value(
      observation_index_points.shape[-(feature_ndims + 1)])
  if num_obs is not None and num_obs == 0:
    return True
  return False

def _is_empty_observation_data ( feature_ndims , observation_index_points , observations ) : if observation_index_points is None and observations is None : return True num_obs = tf . compat . dimension_value ( observation_index_points . shape [ - ( feature_ndims + 1 ) ] ) if num_obs is not None and num_obs == 0 : return True return False

Returns True if given observation data is empty .


def _validate_observation_data(
    kernel, observation_index_points, observations):
  """Ensure that observation data and locations have consistent shapes.

  This basically means that the batch shapes are broadcastable. We can only
  ensure this when those shapes are fully statically defined.

  Args:
    kernel: The GP kernel.
    observation_index_points: the observation data locations in the index set.
    observations: the observation data.

  Raises:
    ValueError: if the observations' batch shapes are not broadcastable.
  """
  # Check that observation index points and observation counts broadcast.
  ndims = kernel.feature_ndims
  if (tensorshape_util.is_fully_defined(
      observation_index_points.shape[:-ndims]) and
      tensorshape_util.is_fully_defined(observations.shape)):
    index_point_count = observation_index_points.shape[:-ndims]
    observation_count = observations.shape
    try:
      tf.broadcast_static_shape(index_point_count, observation_count)
    except ValueError:
      # Re-raise with our own more contextual error message.
      raise ValueError(
          'Observation index point and observation counts are not '
          'broadcastable: {} and {}, respectively.'.format(
              index_point_count, observation_count))

Ensure that observation data and locations have consistent shapes .


def add(self, scheduler, max_iteration, bigdl_type="float"):
        """
        Add a learning rate scheduler to the contained `schedules`

        :param scheduler: learning rate scheduler to be add
        :param max_iteration: iteration numbers this scheduler will run
        """
        return callBigDlFunc(bigdl_type, "addScheduler", self.value, scheduler, max_iteration)

def add ( self , scheduler , max_iteration , bigdl_type = "float" ) : return callBigDlFunc ( bigdl_type , "addScheduler" , self . value , scheduler , max_iteration )

Add a learning rate scheduler to the contained schedules


def set_checkpoint(self, checkpoint_trigger,
                       checkpoint_path, isOverWrite=True):
        """
        Configure checkpoint settings.

        :param checkpoint_trigger: the interval to write snapshots
        :param checkpoint_path: the path to write snapshots into
        :param isOverWrite: whether to overwrite existing snapshots in path.default is True
        """
        if not os.path.exists(checkpoint_path):
            mkpath(checkpoint_path)
        callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value,
                      checkpoint_trigger, checkpoint_path, isOverWrite)

def set_checkpoint ( self , checkpoint_trigger , checkpoint_path , isOverWrite = True ) : if not os . path . exists ( checkpoint_path ) : mkpath ( checkpoint_path ) callBigDlFunc ( self . bigdl_type , "setCheckPoint" , self . value , checkpoint_trigger , checkpoint_path , isOverWrite )


def set_gradclip_const(self, min_value, max_value):
        """
        Configure constant clipping settings.

        :param min_value: the minimum value to clip by
        :param max_value: the maxmimum value to clip by
        """
        callBigDlFunc(self.bigdl_type, "setConstantClip", self.value, min_value, max_value)

def set_gradclip_const ( self , min_value , max_value ) : callBigDlFunc ( self . bigdl_type , "setConstantClip" , self . value , min_value , max_value )


def optimize(self):
        """
        Do an optimization.
        """
        jmodel = callJavaFunc(self.value.optimize)
        from bigdl.nn.layer import Layer
        return Layer.of(jmodel)

def optimize ( self ) : jmodel = callJavaFunc ( self . value . optimize ) from bigdl . nn . layer import Layer return Layer . of ( jmodel )


def set_train_summary(self, summary):
        """
        Set train summary. A TrainSummary object contains information
        necessary for the optimizer to know how often the logs are recorded,
        where to store the logs and how to retrieve them, etc. For details,
        refer to the docs of TrainSummary.

        :param summary: a TrainSummary object
        """
        callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value,
                      summary)
        return self

def set_train_summary ( self , summary ) : callBigDlFunc ( self . bigdl_type , "setTrainSummary" , self . value , summary ) return self

Set train summary . A TrainSummary object contains information necessary for the optimizer to know how often the logs are recorded where to store the logs and how to retrieve them etc . For details refer to the docs of TrainSummary .


def set_val_summary(self, summary):
        """
        Set validation summary. A ValidationSummary object contains information
        necessary for the optimizer to know how often the logs are recorded,
        where to store the logs and how to retrieve them, etc. For details,
        refer to the docs of ValidationSummary.

        :param summary: a ValidationSummary object

        """
        callBigDlFunc(self.bigdl_type, "setValSummary", self.value,
                      summary)
        return self

def set_val_summary ( self , summary ) : callBigDlFunc ( self . bigdl_type , "setValSummary" , self . value , summary ) return self

Set validation summary . A ValidationSummary object contains information necessary for the optimizer to know how often the logs are recorded where to store the logs and how to retrieve them etc . For details refer to the docs of ValidationSummary .


def create(model,
               training_set,
               criterion,
               end_trigger=None,
               batch_size=32,
               optim_method=None,
               cores=None,
               bigdl_type="float"):
        """
        Create an optimizer.
        Depend on the input type, the returning optimizer can be a local optimizer \
        or a distributed optimizer.

        :param model: the neural net model
        :param training_set: (features, label) for local mode. RDD[Sample] for distributed mode.
        :param criterion: the loss function
        :param optim_method: the algorithm to use for optimization,
           e.g. SGD, Adagrad, etc. If optim_method is None, the default algorithm is SGD.
        :param end_trigger: when to end the optimization. default value is MapEpoch(1)
        :param batch_size: training batch size
        :param cores: This is for local optimizer only and use total physical cores as the default value
        """
        if not end_trigger:
            end_trigger = MaxEpoch(1)
        if not optim_method:
            optim_method = SGD()
        if isinstance(training_set, RDD) or isinstance(training_set, DataSet):
            return DistriOptimizer(model=model,
                                   training_rdd=training_set,
                                   criterion=criterion,
                                   end_trigger=end_trigger,
                                   batch_size=batch_size,
                                   optim_method=optim_method,
                                   bigdl_type=bigdl_type)
        elif isinstance(training_set, tuple) and len(training_set) == 2:
            x, y = training_set
            return LocalOptimizer(X=x,
                                  Y=y,
                                  model=model,
                                  criterion=criterion,
                                  end_trigger=end_trigger,
                                  batch_size=batch_size,
                                  optim_method=optim_method,
                                  cores=cores,
                                  bigdl_type="float")
        else:
            raise Exception("Not supported training set: %s" % type(training_set))

def
create
(
model
,
training_set
,
criterion
,
end_trigger
=
,
batch_size
=
32
,
optim_method
=
,
cores
=
,
bigdl_type
=
"float"
)
:
if
not
end_trigger
:
end_trigger
=
MaxEpoch
(
1
)
if
not
optim_method
:
optim_method
=
SGD
(
)
if
isinstance
(
training_set
,
RDD
)
or
isinstance
(
training_set
,
DataSet
)
:
return
DistriOptimizer
(
model
=
model
,
training_rdd
=
training_set
,
criterion
=
criterion
,
end_trigger
=
end_trigger
,
batch_size
=
batch_size
,
optim_method
=
optim_method
,
bigdl_type
=
bigdl_type
)
elif
isinstance
(
training_set
,
tuple
)
and
len
(
training_set
)
==
2
:
x
,
y
=
training_set
return
LocalOptimizer
(
X
=
x
,
Y
=
y
,
model
=
model
,
criterion
=
criterion
,
end_trigger
=
end_trigger
,
batch_size
=
batch_size
,
optim_method
=
optim_method
,
cores
=
cores
,
bigdl_type
=
"float"
)
else
:
raise
Exception
(
"Not supported training set: %s"
%
type
(
training_set
)
)

Create an optimizer . Depend on the input type the returning optimizer can be a local optimizer \ or a distributed optimizer .


def set_traindata(self, training_rdd, batch_size):
        """
        Set new training dataset, for optimizer reuse

        :param training_rdd: the training dataset
        :param batch_size: training batch size
        :return:
        """
        callBigDlFunc(self.bigdl_type, "setTrainData", self.value,
                     training_rdd, batch_size)

def set_traindata ( self , training_rdd , batch_size ) : callBigDlFunc ( self . bigdl_type , "setTrainData" , self . value , training_rdd , batch_size )

Set new training dataset for optimizer reuse


def set_summary_trigger(self, name, trigger):
        """
        Set the interval of recording for each indicator.

        :param tag: tag name. Supported tag names are "LearningRate", "Loss","Throughput", "Parameters". "Parameters" is an umbrella tag thatincludes weight, bias, gradWeight, gradBias, and some running status(eg. runningMean and runningVar in BatchNormalization). If youdidn't set any triggers, we will by default record Loss and Throughputin each iteration, while *NOT* recording LearningRate and Parameters,as recording parameters may introduce substantial overhead when themodel is very big, LearningRate is not a public attribute for allOptimMethod.
        :param trigger: trigger
        """
        return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value,
                             name, trigger)

def set_summary_trigger ( self , name , trigger ) : return callBigDlFunc ( self . bigdl_type , "summarySetTrigger" , self . value , name , trigger )

Set the interval of recording for each indicator .


def read_data_sets(train_dir, data_type="train"):
    """
    Parse or download mnist data if train_dir is empty.

    :param: train_dir: The directory storing the mnist data

    :param: data_type: Reading training set or testing set.It can be either "train" or "test"

    :return:

    ```
    (ndarray, ndarray) representing (features, labels)
    features is a 4D unit8 numpy array [index, y, x, depth] representing each pixel valued from 0 to 255.
    labels is 1D unit8 nunpy array representing the label valued from 0 to 9.
    ```

    """
    TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
    TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'
    TEST_IMAGES = 't10k-images-idx3-ubyte.gz'
    TEST_LABELS = 't10k-labels-idx1-ubyte.gz'

    if data_type == "train":
        local_file = base.maybe_download(TRAIN_IMAGES, train_dir,
                                         SOURCE_URL + TRAIN_IMAGES)
        with open(local_file, 'rb') as f:
            train_images = extract_images(f)

        local_file = base.maybe_download(TRAIN_LABELS, train_dir,
                                         SOURCE_URL + TRAIN_LABELS)
        with open(local_file, 'rb') as f:
            train_labels = extract_labels(f)
        return train_images, train_labels

    else:
        local_file = base.maybe_download(TEST_IMAGES, train_dir,
                                         SOURCE_URL + TEST_IMAGES)
        with open(local_file, 'rb') as f:
            test_images = extract_images(f)

        local_file = base.maybe_download(TEST_LABELS, train_dir,
                                         SOURCE_URL + TEST_LABELS)
        with open(local_file, 'rb') as f:
            test_labels = extract_labels(f)
        return test_images, test_labels

def read_data_sets ( train_dir , data_type = "train" ) : TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' TEST_IMAGES = 't10k-images-idx3-ubyte.gz' TEST_LABELS = 't10k-labels-idx1-ubyte.gz' if data_type == "train" : local_file = base . maybe_download ( TRAIN_IMAGES , train_dir , SOURCE_URL + TRAIN_IMAGES ) with open ( local_file , 'rb' ) as f : train_images = extract_images ( f ) local_file = base . maybe_download ( TRAIN_LABELS , train_dir , SOURCE_URL + TRAIN_LABELS ) with open ( local_file , 'rb' ) as f : train_labels = extract_labels ( f ) return train_images , train_labels else : local_file = base . maybe_download ( TEST_IMAGES , train_dir , SOURCE_URL + TEST_IMAGES ) with open ( local_file , 'rb' ) as f : test_images = extract_images ( f ) local_file = base . maybe_download ( TEST_LABELS , train_dir , SOURCE_URL + TEST_LABELS ) with open ( local_file , 'rb' ) as f : test_labels = extract_labels ( f ) return test_images , test_labels

Parse or download mnist data if train_dir is empty .


def get_news20(source_dir="./data/news20/"):
    """
    Parse or download news20 if source_dir is empty.

    :param source_dir: The directory storing news data.
    :return: A list of (tokens, label)
    """
    news_dir = download_news20(source_dir)
    texts = []  # list of text samples
    label_id = 0
    for name in sorted(os.listdir(news_dir)):
        path = os.path.join(news_dir, name)
        label_id += 1
        if os.path.isdir(path):
            for fname in sorted(os.listdir(path)):
                if fname.isdigit():
                    fpath = os.path.join(path, fname)
                    if sys.version_info < (3,):
                        f = open(fpath)
                    else:
                        f = open(fpath, encoding='latin-1')
                    content = f.read()
                    texts.append((content, label_id))
                    f.close()

    print('Found %s texts.' % len(texts))
    return texts

def
get_news20
(
source_dir
=
"./data/news20/"
)
:
news_dir
=
download_news20
(
source_dir
)
texts
=
[
]
label_id
=
0
for
name
in
sorted
(
os
.
listdir
(
news_dir
)
)
:
path
=
os
.
path
.
join
(
news_dir
,
name
)
label_id
+=
1
if
os
.
path
.
isdir
(
path
)
:
for
fname
in
sorted
(
os
.
listdir
(
path
)
)
:
if
fname
.
isdigit
(
)
:
fpath
=
os
.
path
.
join
(
path
,
fname
)
if
sys
.
version_info
<
(
3
,
)
:
f
=
open
(
fpath
)
else
:
f
=
open
(
fpath
,
encoding
=
'latin-1'
)
content
=
f
.
read
(
)
texts
.
append
(
(
content
,
label_id
)
)
f
.
close
(
)
print
(
'Found %s texts.'
%
len
(
texts
)
)
return
texts

Parse or download news20 if source_dir is empty .


def get_glove_w2v(source_dir="./data/news20/", dim=100):
    """
    Parse or download the pre-trained glove word2vec if source_dir is empty.

    :param source_dir: The directory storing the pre-trained word2vec
    :param dim: The dimension of a vector
    :return: A dict mapping from word to vector
    """
    w2v_dir = download_glove_w2v(source_dir)
    w2v_path = os.path.join(w2v_dir, "glove.6B.%sd.txt" % dim)
    if sys.version_info < (3,):
        w2v_f = open(w2v_path)
    else:
        w2v_f = open(w2v_path, encoding='latin-1')
    pre_w2v = {}
    for line in w2v_f.readlines():
        items = line.split(" ")
        pre_w2v[items[0]] = [float(i) for i in items[1:]]
    w2v_f.close()
    return pre_w2v

def
get_glove_w2v
(
source_dir
=
"./data/news20/"
,
dim
=
100
)
:
w2v_dir
=
download_glove_w2v
(
source_dir
)
w2v_path
=
os
.
path
.
join
(
w2v_dir
,
"glove.6B.%sd.txt"
%
dim
)
if
sys
.
version_info
<
(
3
,
)
:
w2v_f
=
open
(
w2v_path
)
else
:
w2v_f
=
open
(
w2v_path
,
encoding
=
'latin-1'
)
pre_w2v
=
{
}
for
line
in
w2v_f
.
readlines
(
)
:
items
=
line
.
split
(
" "
)
pre_w2v
[
items
[
0
]
]
=
[
float
(
i
)
for
i
in
items
[
1
:
]
]
w2v_f
.
close
(
)
return
pre_w2v

Parse or download the pre - trained glove word2vec if source_dir is empty .


def compile(self, optimizer, loss, metrics=None):
        """
        Configures the learning process. Must be called before fit or evaluate.

        # Arguments
        optimizer: Optimization method to be used. One can alternatively pass in the corresponding
                   string representation, such as 'sgd'.
        loss: Criterion to be used. One can alternatively pass in the corresponding string
              representation, such as 'mse'.
        metrics: List of validation methods to be used. Default is None. One can alternatively use ['accuracy'].
        """
        if isinstance(optimizer, six.string_types):
            optimizer = self.__convert_optim_method(optimizer)
        if isinstance(loss, six.string_types):
            loss = self.__convert_criterion(loss)
        if all(isinstance(metric, six.string_types) for metric in metrics):
            metrics = self.__convert_metrics(metrics)
        callBigDlFunc(self.bigdl_type, "compile",
                      self.value,
                      optimizer,
                      loss,
                      metrics)

def compile ( self , optimizer , loss , metrics = None ) : if isinstance ( optimizer , six . string_types ) : optimizer = self . __convert_optim_method ( optimizer ) if isinstance ( loss , six . string_types ) : loss = self . __convert_criterion ( loss ) if all ( isinstance ( metric , six . string_types ) for metric in metrics ) : metrics = self . __convert_metrics ( metrics ) callBigDlFunc ( self . bigdl_type , "compile" , self . value , optimizer , loss , metrics )

Configures the learning process . Must be called before fit or evaluate .


def fit(self, x, y=None, batch_size=32, nb_epoch=10, validation_data=None, distributed=True):
        """
        Train a model for a fixed number of epochs on a dataset.

        # Arguments
        x: Input data. A Numpy array or RDD of Sample or Image DataSet.
        y: Labels. A Numpy array. Default is None if x is already RDD of Sample or Image DataSet.
        batch_size: Number of samples per gradient update.
        nb_epoch: Number of iterations to train.
        validation_data: Tuple (x_val, y_val) where x_val and y_val are both Numpy arrays.
                         Or RDD of Sample. Default is None if no validation is involved.
        distributed: Boolean. Whether to train the model in distributed mode or local mode.
                     Default is True. In local mode, x and y must both be Numpy arrays.
        """
        if distributed:
            if isinstance(x, np.ndarray) and isinstance(y, np.ndarray):
                training_data = to_sample_rdd(x, y)
                if validation_data:
                    validation_data = to_sample_rdd(*validation_data)
            elif (isinstance(x, RDD) or isinstance(x, DataSet)) and not y:
                training_data = x
            else:
                raise TypeError("Unsupported training data type: %s" % type(x))
            callBigDlFunc(self.bigdl_type, "fit",
                          self.value,
                          training_data,
                          batch_size,
                          nb_epoch,
                          validation_data)
        else:
            if validation_data:
                val_x = [JTensor.from_ndarray(x) for x in to_list(validation_data[0])]
                val_y = JTensor.from_ndarray(validation_data[1])
            else:
                val_x, val_y = None, None
            callBigDlFunc(self.bigdl_type, "fit",
                          self.value,
                          [JTensor.from_ndarray(x) for x in to_list(x)],
                          JTensor.from_ndarray(y),
                          batch_size,
                          nb_epoch,
                          val_x,
                          val_y,
                          multiprocessing.cpu_count())

def
fit
(
self
,
x
,
y
=
,
batch_size
=
32
,
nb_epoch
=
10
,
validation_data
=
,
distributed
=
True
)
:
if
distributed
:
if
isinstance
(
x
,
np
.
ndarray
)
and
isinstance
(
y
,
np
.
ndarray
)
:
training_data
=
to_sample_rdd
(
x
,
y
)
if
validation_data
:
validation_data
=
to_sample_rdd
(
*
validation_data
)
elif
(
isinstance
(
x
,
RDD
)
or
isinstance
(
x
,
DataSet
)
)
and
not
y
:
training_data
=
x
else
:
raise
TypeError
(
"Unsupported training data type: %s"
%
type
(
x
)
)
callBigDlFunc
(
self
.
bigdl_type
,
"fit"
,
self
.
value
,
training_data
,
batch_size
,
nb_epoch
,
validation_data
)
else
:
if
validation_data
:
val_x
=
[
JTensor
.
from_ndarray
(
x
)
for
x
in
to_list
(
validation_data
[
0
]
)
]
val_y
=
JTensor
.
from_ndarray
(
validation_data
[
1
]
)
else
:
val_x
,
val_y
=
,
callBigDlFunc
(
self
.
bigdl_type
,
"fit"
,
self
.
value
,
[
JTensor
.
from_ndarray
(
x
)
for
x
in
to_list
(
x
)
]
,
JTensor
.
from_ndarray
(
y
)
,
batch_size
,
nb_epoch
,
val_x
,
val_y
,
multiprocessing
.
cpu_count
(
)
)

Train a model for a fixed number of epochs on a dataset .


def evaluate(self, x, y=None, batch_size=32):
        """
        Evaluate a model on a given dataset in distributed mode.

        # Arguments
        x: Input data. A Numpy array or RDD of Sample.
        y: Labels. A Numpy array. Default is None if x is already RDD of Sample.
        batch_size: Number of samples per gradient update.
        """
        if isinstance(x, np.ndarray) and isinstance(y, np.ndarray):
            evaluation_data = to_sample_rdd(x, y)
        elif isinstance(x, RDD) and not y:
            evaluation_data = x
        else:
            raise TypeError("Unsupported evaluation data type: %s" % type(x))
        return callBigDlFunc(self.bigdl_type, "evaluate",
                             self.value,
                             evaluation_data,
                             batch_size)

def
evaluate
(
self
,
x
,
y
=
,
batch_size
=
32
)
:
if
isinstance
(
x
,
np
.
ndarray
)
and
isinstance
(
y
,
np
.
ndarray
)
:
evaluation_data
=
to_sample_rdd
(
x
,
y
)
elif
isinstance
(
x
,
RDD
)
and
not
y
:
evaluation_data
=
x
else
:
raise
TypeError
(
"Unsupported evaluation data type: %s"
%
type
(
x
)
)
return
callBigDlFunc
(
self
.
bigdl_type
,
"evaluate"
,
self
.
value
,
evaluation_data
,
batch_size
)

Evaluate a model on a given dataset in distributed mode .


def predict(self, x, distributed=True):
        """
        Use a model to do prediction.

        # Arguments
        x: Input data. A Numpy array or RDD of Sample.
        distributed: Boolean. Whether to do prediction in distributed mode or local mode.
                     Default is True. In local mode, x must be a Numpy array.
        """
        if is_distributed:
            if isinstance(x, np.ndarray):
                features = to_sample_rdd(x, np.zeros([x.shape[0]]))
            elif isinstance(x, RDD):
                features = x
            else:
                raise TypeError("Unsupported prediction data type: %s" % type(x))
            return self.predict_distributed(features)
        else:
            if isinstance(x, np.ndarray):
                return self.predict_local(x)
            else:
                raise TypeError("Unsupported prediction data type: %s" % type(x))

def
predict
(
self
,
x
,
distributed
=
True
)
:
if
is_distributed
:
if
isinstance
(
x
,
np
.
ndarray
)
:
features
=
to_sample_rdd
(
x
,
np
.
zeros
(
[
x
.
shape
[
0
]
]
)
)
elif
isinstance
(
x
,
RDD
)
:
features
=
x
else
:
raise
TypeError
(
"Unsupported prediction data type: %s"
%
type
(
x
)
)
return
self
.
predict_distributed
(
features
)
else
:
if
isinstance
(
x
,
np
.
ndarray
)
:
return
self
.
predict_local
(
x
)
else
:
raise
TypeError
(
"Unsupported prediction data type: %s"
%
type
(
x
)
)


def get_mnist(sc, data_type="train", location="/tmp/mnist"):
    """
    Get mnist dataset and parallelize into RDDs.
    Data would be downloaded automatically if it doesn't present at the specific location.

    :param sc: SparkContext.
    :param data_type: "train" for training data and "test" for testing data.
    :param location: Location to store mnist dataset.
    :return: RDD of (features: ndarray, label: ndarray).
    """
    (images, labels) = mnist.read_data_sets(location, data_type)
    images = sc.parallelize(images)
    labels = sc.parallelize(labels + 1)  # Target start from 1 in BigDL
    record = images.zip(labels)
    return record

def get_mnist ( sc , data_type = "train" , location = "/tmp/mnist" ) : ( images , labels ) = mnist . read_data_sets ( location , data_type ) images = sc . parallelize ( images ) labels = sc . parallelize ( labels + 1 ) record = images . zip ( labels ) return record

Get mnist dataset and parallelize into RDDs . Data would be downloaded automatically if it doesn t present at the specific location .


def preprocess_mnist(sc, options):
    """
    Preprocess mnist dataset.
    Normalize and transform into Sample of RDDs.
    """
    train_data = get_mnist(sc, "train", options.dataPath)\
        .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
                                rec_tuple[1]))\
        .map(lambda t: Sample.from_ndarray(t[0], t[1]))
    test_data = get_mnist(sc, "test", options.dataPath)\
        .map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD),
                                rec_tuple[1]))\
        .map(lambda t: Sample.from_ndarray(t[0], t[1]))
    return train_data, test_data

def preprocess_mnist ( sc , options ) : train_data = get_mnist ( sc , "train" , options . dataPath ) . map ( lambda rec_tuple : ( normalizer ( rec_tuple [ 0 ] , mnist . TRAIN_MEAN , mnist . TRAIN_STD ) , rec_tuple [ 1 ] ) ) . map ( lambda t : Sample . from_ndarray ( t [ 0 ] , t [ 1 ] ) ) test_data = get_mnist ( sc , "test" , options . dataPath ) . map ( lambda rec_tuple : ( normalizer ( rec_tuple [ 0 ] , mnist . TEST_MEAN , mnist . TEST_STD ) , rec_tuple [ 1 ] ) ) . map ( lambda t : Sample . from_ndarray ( t [ 0 ] , t [ 1 ] ) ) return train_data , test_data

Preprocess mnist dataset . Normalize and transform into Sample of RDDs .


def get_end_trigger(options):
    """
    When to end the optimization based on input option.
    """
    if options.endTriggerType.lower() == "epoch":
        return MaxEpoch(options.endTriggerNum)
    else:
        return MaxIteration(options.endTriggerNum)

def get_end_trigger ( options ) : if options . endTriggerType . lower ( ) == "epoch" : return MaxEpoch ( options . endTriggerNum ) else : return MaxIteration ( options . endTriggerNum )

When to end the optimization based on input option .


def validate_optimizer(optimizer, test_data, options):
    """
    Set validation and checkpoint for distributed optimizer.
    """
    optimizer.set_validation(
        batch_size=options.batchSize,
        val_rdd=test_data,
        trigger=EveryEpoch(),
        val_method=[Top1Accuracy()]
    )
    optimizer.set_checkpoint(EveryEpoch(), options.checkpointPath)

def validate_optimizer ( optimizer , test_data , options ) : optimizer . set_validation ( batch_size = options . batchSize , val_rdd = test_data , trigger = EveryEpoch ( ) , val_method = [ Top1Accuracy ( ) ] ) optimizer . set_checkpoint ( EveryEpoch ( ) , options . checkpointPath )

Set validation and checkpoint for distributed optimizer .


def value(self):
        """ Return the broadcasted value
        """
        if not hasattr(self, "_value") and self._path is not None:
            self._value = self._load(self._path)
        return self._value

def value ( self ) : if not hasattr ( self , "_value" ) and self . _path is not None : self . _value = self . _load ( self . _path ) return self . _value


def callBigDlFunc(bigdl_type, name, *args):
    """ Call API in PythonBigDL """
    gateway = _get_gateway()
    error = Exception("Cannot find function: %s" % name)
    for jinvoker in JavaCreator.instance(bigdl_type, gateway).value:
        # hasattr(jinvoker, name) always return true here,
        # so you need to invoke the method to check if it exist or not
        try:
            api = getattr(jinvoker, name)
            result = callJavaFunc(api, *args)
        except Exception as e:
            error = e
            if "does not exist" not in str(e):
                raise e
        else:
            return result
    raise error

def
callBigDlFunc
(
bigdl_type
,
name
,
*
args
)
:
gateway
=
_get_gateway
(
)
error
=
Exception
(
"Cannot find function: %s"
%
name
)
for
jinvoker
in
JavaCreator
.
instance
(
bigdl_type
,
gateway
)
.
value
:
try
:
api
=
getattr
(
jinvoker
,
name
)
result
=
callJavaFunc
(
api
,
*
args
)
except
Exception
as
e
:
error
=
e
if
"does not exist"
not
in
str
(
e
)
:
raise
e
else
:
return
result
raise
error


def callJavaFunc(func, *args):
    """ Call Java Function """
    gateway = _get_gateway()
    args = [_py2java(gateway, a) for a in args]
    result = func(*args)
    return _java2py(gateway, result)

def callJavaFunc ( func , * args ) : gateway = _get_gateway ( ) args = [ _py2java ( gateway , a ) for a in args ] result = func ( * args ) return _java2py ( gateway , result )


def _to_java_object_rdd(rdd):
    """ Return a JavaRDD of Object by unpickling

    It will convert each Python object into Java object by Pyrolite, whenever
    the RDD is serialized in batch or not.
    """
    rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))
    return \
        rdd.ctx._jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.pythonToJava(
            rdd._jrdd, True)

def _to_java_object_rdd ( rdd ) : rdd = rdd . _reserialize ( AutoBatchedSerializer ( PickleSerializer ( ) ) ) return rdd . ctx . _jvm . org . apache . spark . bigdl . api . python . BigDLSerDe . pythonToJava ( rdd . _jrdd , True )


def _py2java(gateway, obj):
    """ Convert Python object into Java """
    if isinstance(obj, RDD):
        obj = _to_java_object_rdd(obj)
    elif isinstance(obj, DataFrame):
        obj = obj._jdf
    elif isinstance(obj, SparkContext):
        obj = obj._jsc
    elif isinstance(obj, (list, tuple)):
        obj = ListConverter().convert([_py2java(gateway, x) for x in obj],
                                      gateway._gateway_client)
    elif isinstance(obj, dict):
        result = {}
        for (key, value) in obj.items():
            result[key] = _py2java(gateway, value)
        obj = MapConverter().convert(result, gateway._gateway_client)
    elif isinstance(obj, JavaValue):
        obj = obj.value
    elif isinstance(obj, JavaObject):
        pass
    elif isinstance(obj, (int, long, float, bool, bytes, unicode)):
        pass
    else:
        data = bytearray(PickleSerializer().dumps(obj))
        obj = gateway.jvm.org.apache.spark.bigdl.api.python.BigDLSerDe.loads(data)
    return obj

def _py2java ( gateway , obj ) : if isinstance ( obj , RDD ) : obj = _to_java_object_rdd ( obj ) elif isinstance ( obj , DataFrame ) : obj = obj . _jdf elif isinstance ( obj , SparkContext ) : obj = obj . _jsc elif isinstance ( obj , ( list , tuple ) ) : obj = ListConverter ( ) . convert ( [ _py2java ( gateway , x ) for x in obj ] , gateway . _gateway_client ) elif isinstance ( obj , dict ) : result = { } for ( key , value ) in obj . items ( ) : result [ key ] = _py2java ( gateway , value ) obj = MapConverter ( ) . convert ( result , gateway . _gateway_client ) elif isinstance ( obj , JavaValue ) : obj = obj . value elif isinstance ( obj , JavaObject ) : pass elif isinstance ( obj , ( int , long , float , bool , bytes , unicode ) ) : pass else : data = bytearray ( PickleSerializer ( ) . dumps ( obj ) ) obj = gateway . jvm . org . apache . spark . bigdl . api . python . BigDLSerDe . loads ( data ) return obj


def get_activation_by_name(activation_name, activation_id=None):
    """ Convert to a bigdl activation layer
        given the name of the activation as a string  """
    import bigdl.nn.layer as BLayer
    activation = None
    activation_name = activation_name.lower()
    if activation_name == "tanh":
        activation = BLayer.Tanh()
    elif activation_name == "sigmoid":
        activation = BLayer.Sigmoid()
    elif activation_name == "hard_sigmoid":
        activation = BLayer.HardSigmoid()
    elif activation_name == "relu":
        activation = BLayer.ReLU()
    elif activation_name == "softmax":
        activation = BLayer.SoftMax()
    elif activation_name == "softplus":
        activation = BLayer.SoftPlus(beta=1.0)
    elif activation_name == "softsign":
        activation = BLayer.SoftSign()
    elif activation_name == "linear":
        activation = BLayer.Identity()
    else:
        raise Exception("Unsupported activation type: %s" % activation_name)
    if not activation_id:
        activation.set_name(activation_id)
    return activation

def
get_activation_by_name
(
activation_name
,
activation_id
=
)
:
import
bigdl
.
nn
.
layer
as
BLayer
activation
=
activation_name
=
activation_name
.
lower
(
)
if
activation_name
==
"tanh"
:
activation
=
BLayer
.
Tanh
(
)
elif
activation_name
==
"sigmoid"
:
activation
=
BLayer
.
Sigmoid
(
)
elif
activation_name
==
"hard_sigmoid"
:
activation
=
BLayer
.
HardSigmoid
(
)
elif
activation_name
==
"relu"
:
activation
=
BLayer
.
ReLU
(
)
elif
activation_name
==
"softmax"
:
activation
=
BLayer
.
SoftMax
(
)
elif
activation_name
==
"softplus"
:
activation
=
BLayer
.
SoftPlus
(
beta
=
1.0
)
elif
activation_name
==
"softsign"
:
activation
=
BLayer
.
SoftSign
(
)
elif
activation_name
==
"linear"
:
activation
=
BLayer
.
Identity
(
)
else
:
raise
Exception
(
"Unsupported activation type: %s"
%
activation_name
)
if
not
activation_id
:
activation
.
set_name
(
activation_id
)
return
activation


def from_ndarray(cls, a_ndarray, bigdl_type="float"):
        """
        Convert a ndarray to a DenseTensor which would be used in Java side.

        >>> import numpy as np
        >>> from bigdl.util.common import JTensor
        >>> from bigdl.util.common import callBigDlFunc
        >>> np.random.seed(123)
        >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32")
        >>> result = JTensor.from_ndarray(data)
        >>> expected_storage = np.array([[0.69646919, 0.28613934, 0.22685145], [0.55131477, 0.71946895, 0.42310646]])
        >>> expected_shape = np.array([2, 3])
        >>> np.testing.assert_allclose(result.storage, expected_storage, rtol=1e-6, atol=1e-6)
        >>> np.testing.assert_allclose(result.shape, expected_shape)
        >>> data_back = result.to_ndarray()
        >>> (data == data_back).all()
        True
        >>> tensor1 = callBigDlFunc("float", "testTensor", JTensor.from_ndarray(data))  # noqa
        >>> array_from_tensor = tensor1.to_ndarray()
        >>> (array_from_tensor == data).all()
        True
        """
        if a_ndarray is None:
            return None
        assert isinstance(a_ndarray, np.ndarray), \
            "input should be a np.ndarray, not %s" % type(a_ndarray)
        return cls(a_ndarray,
                   a_ndarray.shape if a_ndarray.shape else (a_ndarray.size),
                   bigdl_type)

def
from_ndarray
(
cls
,
a_ndarray
,
bigdl_type
=
"float"
)
:
if
a_ndarray
is
:
return
assert
isinstance
(
a_ndarray
,
np
.
ndarray
)
,
"input should be a np.ndarray, not %s"
%
type
(
a_ndarray
)
return
cls
(
a_ndarray
,
a_ndarray
.
shape
if
a_ndarray
.
shape
else
(
a_ndarray
.
size
)
,
bigdl_type
)

Convert a ndarray to a DenseTensor which would be used in Java side .


def get_label(self):
        """
        get label as ndarray from ImageFeature
        """
        label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor", self.value)
        return label.to_ndarray()

def get_label ( self ) : label = callBigDlFunc ( self . bigdl_type , "imageFeatureToLabelTensor" , self . value ) return label . to_ndarray ( )


def read_parquet(cls, path, sc, bigdl_type="float"):
        """
        Read parquet file as DistributedImageFrame
        """
        return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sc))

def read_parquet ( cls , path , sc , bigdl_type = "float" ) : return DistributedImageFrame ( jvalue = callBigDlFunc ( bigdl_type , "readParquet" , path , sc ) )


def write_parquet(cls, path, output, sc, partition_num = 1, bigdl_type="float"):
        """
        write ImageFrame as parquet file
        """
        return callBigDlFunc(bigdl_type, "writeParquet", path, output, sc, partition_num)

def write_parquet ( cls , path , output , sc , partition_num = 1 , bigdl_type = "float" ) : return callBigDlFunc ( bigdl_type , "writeParquet" , path , output , sc , partition_num )


def get_image(self, float_key="floats", to_chw=True):
        """
        get image from ImageFrame
        """
        return self.image_frame.get_image(float_key, to_chw)

def get_image ( self , float_key = "floats" , to_chw = True ) : return self . image_frame . get_image ( float_key , to_chw )


def get_image(self, float_key="floats", to_chw=True):
        """
        get image list from ImageFrame
        """
        tensors = callBigDlFunc(self.bigdl_type,
                                   "localImageFrameToImageTensor", self.value, float_key, to_chw)
        return map(lambda tensor: tensor.to_ndarray(), tensors)

def get_image ( self , float_key = "floats" , to_chw = True ) : tensors = callBigDlFunc ( self . bigdl_type , "localImageFrameToImageTensor" , self . value , float_key , to_chw ) return map ( lambda tensor : tensor . to_ndarray ( ) , tensors )


def get_label(self):
        """
        get label rdd from ImageFrame
        """
        tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value)
        return tensor_rdd.map(lambda tensor: tensor.to_ndarray())

def get_label ( self ) : tensor_rdd = callBigDlFunc ( self . bigdl_type , "distributedImageFrameToLabelTensorRdd" , self . value ) return tensor_rdd . map ( lambda tensor : tensor . to_ndarray ( ) )


def get_predict(self, key="predict"):
        """
        get prediction rdd from ImageFrame
        """
        predicts = callBigDlFunc(self.bigdl_type, "distributedImageFrameToPredict", self.value, key)
        return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray()) if predict[1] else (predict[0], None))

def get_predict ( self , key = "predict" ) : predicts = callBigDlFunc ( self . bigdl_type , "distributedImageFrameToPredict" , self . value , key ) return predicts . map ( lambda predict : ( predict [ 0 ] , predict [ 1 ] . to_ndarray ( ) ) if predict [ 1 ] else ( predict [ 0 ] , None ) )


def predict(self, x, batch_size=None, verbose=None, is_distributed=False):
        """Generates output predictions for the input samples,
        processing the samples in a batched way.

        # Arguments
            x: the input data, as a Numpy array or list of Numpy array for local mode.
               as RDD[Sample] for distributed mode
            is_distributed: used to control run in local or cluster. the default value is False
        # Returns
            A Numpy array or RDD[Sample] of predictions.
        """
        if batch_size or verbose:
            raise Exception("we don't support batch_size or verbose for now")
        if is_distributed:
            if isinstance(x, np.ndarray):
                input = to_sample_rdd(x, np.zeros([x.shape[0]]))
            #  np.asarray(self.bmodel.predict(x_rdd).collect())
            elif isinstance(x, RDD):
                input = x
            return self.bmodel.predict(input)
        else:
            if isinstance(x, np.ndarray):
                return self.bmodel.predict_local(x)
        raise Exception("not supported type: %s" % x)

Generates output predictions for the input samples processing the samples in a batched way .


def fit(self, x, y=None, batch_size=32, nb_epoch=10, verbose=1, callbacks=None,
            validation_split=0., validation_data=None, shuffle=True,
            class_weight=None, sample_weight=None, initial_epoch=0, is_distributed=False):
        """Optimize the model by the given options

        :param x: ndarray or list of ndarray for local mode.
                  RDD[Sample] for distributed mode
        :param y: ndarray or list of ndarray for local mode and would be None for cluster mode.
            is_distributed: used to control run in local or cluster. the default value is False.
            NB: if is_distributed=true, x should be RDD[Sample] and y should be None
        :param is_distributed: Whether to train in local mode or distributed mode
        :return:
            A Numpy array or RDD[Sample] of predictions.
        """
        if callbacks:
            raise Exception("We don't support callbacks in fit for now")
        if class_weight:
            unsupport_exp("class_weight")
        if sample_weight:
            unsupport_exp("sample_weight")
        if initial_epoch != 0:
            unsupport_exp("initial_epoch")
        if shuffle != True:
            unsupport_exp("shuffle")
        if validation_split != 0.:
            unsupport_exp("validation_split")
        bopt = self.__create_optimizer(x=x,
                                       y=y,
                                       batch_size=batch_size,
                                       nb_epoch=nb_epoch,
                                       validation_data=validation_data,
                                       is_distributed=is_distributed)
        bopt.optimize()


def transform(self, dataset):
        """
        Apply the transformer to the images in "inputCol" and store the transformed result
        into "outputCols"
        """
        self._transfer_params_to_java()
        return callBigDlFunc(self.bigdl_type, "dlImageTransform", self.value, dataset)

def transform ( self , dataset ) : self . _transfer_params_to_java ( ) return callBigDlFunc ( self . bigdl_type , "dlImageTransform" , self . value , dataset )

Apply the transformer to the images in inputCol and store the transformed result into outputCols


def save_keras_definition(keras_model, path):
    """
    Save a Keras model definition to JSON with given path
    """
    model_json = keras_model.to_json()
    with open(path, "w") as json_file:
        json_file.write(model_json)

def save_keras_definition ( keras_model , path ) : model_json = keras_model . to_json ( ) with open ( path , "w" ) as json_file : json_file . write ( model_json )


def build_keras_model():
    """
    Define a convnet model in Keras 1.2.2
    """
    from keras.models import Sequential
    from keras.layers import Dense, Dropout, Activation, Flatten
    from keras.layers import Convolution2D, MaxPooling2D

    keras_model = Sequential()
    keras_model.add(Convolution2D(32, 3, 3, border_mode='valid',
                                  input_shape=input_shape))
    keras_model.add(Activation('relu'))
    keras_model.add(Convolution2D(32, 3, 3))
    keras_model.add(Activation('relu'))
    keras_model.add(MaxPooling2D(pool_size=(2, 2)))
    keras_model.add(Dropout(0.25))
    keras_model.add(Flatten())
    keras_model.add(Dense(128))
    keras_model.add(Activation('relu'))
    keras_model.add(Dropout(0.5))
    keras_model.add(Dense(10))
    keras_model.add(Activation('softmax'))
    return keras_model

def build_keras_model ( ) : from keras . models import Sequential from keras . layers import Dense , Dropout , Activation , Flatten from keras . layers import Convolution2D , MaxPooling2D keras_model = Sequential ( ) keras_model . add ( Convolution2D ( 32 , 3 , 3 , border_mode = 'valid' , input_shape = input_shape ) ) keras_model . add ( Activation ( 'relu' ) ) keras_model . add ( Convolution2D ( 32 , 3 , 3 ) ) keras_model . add ( Activation ( 'relu' ) ) keras_model . add ( MaxPooling2D ( pool_size = ( 2 , 2 ) ) ) keras_model . add ( Dropout ( 0.25 ) ) keras_model . add ( Flatten ( ) ) keras_model . add ( Dense ( 128 ) ) keras_model . add ( Activation ( 'relu' ) ) keras_model . add ( Dropout ( 0.5 ) ) keras_model . add ( Dense ( 10 ) ) keras_model . add ( Activation ( 'softmax' ) ) return keras_model

Define a convnet model in Keras 1 . 2 . 2


def predict_class_distributed(self, data_rdd):
        """
        module predict, return the predict label

        :param data_rdd: the data to be predict.
        :return: An RDD represent the predict label.
        """
        result = callBigDlFunc(self.bigdl_type,
                               "modelPredictClass", self.value, data_rdd)
        return result

def predict_class_distributed ( self , data_rdd ) : result = callBigDlFunc ( self . bigdl_type , "modelPredictClass" , self . value , data_rdd ) return result


def set_weights(self, weights):
        """
        Set weights for this layer

        :param weights: a list of numpy arrays which represent weight and bias
        :return:

        >>> linear = Linear(3,2)
        creating: createLinear
        >>> linear.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])])
        >>> weights = linear.get_weights()
        >>> weights[0].shape == (2,3)
        True
        >>> np.testing.assert_allclose(weights[0][0], np.array([1., 2., 3.]))
        >>> np.testing.assert_allclose(weights[1], np.array([7., 8.]))
        >>> relu = ReLU()
        creating: createReLU
        >>> from py4j.protocol import Py4JJavaError
        >>> try:
        ...     relu.set_weights([np.array([[1,2,3],[4,5,6]]), np.array([7,8])])
        ... except Py4JJavaError as err:
        ...     print(err.java_exception)
        ...
        java.lang.IllegalArgumentException: requirement failed: this layer does not have weight/bias
        >>> relu.get_weights()
        The layer does not have weight/bias
        >>> add = Add(2)
        creating: createAdd
        >>> try:
        ...     add.set_weights([np.array([7,8]), np.array([1,2])])
        ... except Py4JJavaError as err:
        ...     print(err.java_exception)
        ...
        java.lang.IllegalArgumentException: requirement failed: the number of input weight/bias is not consistant with number of weight/bias of this layer, number of input 1, number of output 2
        >>> cAdd = CAdd([4, 1])
        creating: createCAdd
        >>> cAdd.set_weights(np.ones([4, 1]))
        >>> (cAdd.get_weights()[0] == np.ones([4, 1])).all()
        True
        """
        tensors = [JTensor.from_ndarray(param, self.bigdl_type) for param in to_list(weights)]
        callBigDlFunc(self.bigdl_type, "setWeights", self.value, tensors)

def set_weights ( self , weights ) : tensors = [ JTensor . from_ndarray ( param , self . bigdl_type ) for param in to_list ( weights ) ] callBigDlFunc ( self . bigdl_type , "setWeights" , self . value , tensors )


def get_weights(self):
        """
        Get weights for this layer

        :return: list of numpy arrays which represent weight and bias
        """
        tensorWeights = callBigDlFunc(self.bigdl_type,
                              "getWeights", self.value)
        if tensorWeights is not None:
            return [tensor.to_ndarray() for tensor in tensorWeights]
        else:
            print("The layer does not have weight/bias")
            return None

def
get_weights
(
self
)
:
tensorWeights
=
callBigDlFunc
(
self
.
bigdl_type
,
"getWeights"
,
self
.
value
)
if
tensorWeights
is
not
:
return
[
tensor
.
to_ndarray
(
)
for
tensor
in
tensorWeights
]
else
:
print
(
"The layer does not have weight/bias"
)
return


def save_tensorflow(self, inputs, path, byte_order="little_endian", data_format="nhwc"):
        """
        Save a model to protobuf files so that it can be used in tensorflow inference.

        When saving the model, placeholders will be added to the tf model as input nodes. So
        you need to pass in the names and shapes of the placeholders. BigDL model doesn't have
        such information. The order of the placeholder information should be same as the inputs
        of the graph model.
        :param inputs: placeholder information, should be an array of tuples (input_name, shape)
                       where 'input_name' is a string and shape is an array of integer
        :param path: the path to be saved to
        :param byte_order: model byte order
        :param data_format: model data format, should be "nhwc" or "nchw"
        """
        callBigDlFunc(self.bigdl_type, "saveTF", self.value, inputs, path, byte_order, data_format)

def save_tensorflow ( self , inputs , path , byte_order = "little_endian" , data_format = "nhwc" ) : callBigDlFunc ( self . bigdl_type , "saveTF" , self . value , inputs , path , byte_order , data_format )

Save a model to protobuf files so that it can be used in tensorflow inference .


def training(self, is_training=True):
        '''
        Set this layer in the training mode or in predition mode if is_training=False
        '''
        if is_training:
            callJavaFunc(self.value.training)
        else:
            callJavaFunc(self.value.evaluate)
        return self

def training ( self , is_training = True ) : if is_training : callJavaFunc ( self . value . training ) else : callJavaFunc ( self . value . evaluate ) return self

Set this layer in the training mode or in predition mode if is_training = False


def load_torch(path, bigdl_type="float"):
        """
        Load a pre-trained Torch model.

        :param path: The path containing the pre-trained model.
        :return: A pre-trained model.
        """
        jmodel = callBigDlFunc(bigdl_type, "loadTorch", path)
        return Layer.of(jmodel)

def load_torch ( path , bigdl_type = "float" ) : jmodel = callBigDlFunc ( bigdl_type , "loadTorch" , path ) return Layer . of ( jmodel )


def load_keras(json_path=None, hdf5_path=None, by_name=False):
        """
        Load a pre-trained Keras model.

        :param json_path: The json path containing the keras model definition.
        :param hdf5_path: The HDF5 path containing the pre-trained keras model weights with or without the model architecture.
        :return: A bigdl model.
        """
        import os
        try:
            import tensorflow
        except ImportError:
            os.environ['KERAS_BACKEND'] = "theano"
            try:
                # Make theano backend compatible with Python3
                from theano import ifelse
            except ImportError:
                raise Exception("No backend is found for Keras. "
                                "Please install either tensorflow or theano.")
        from bigdl.keras.converter import DefinitionLoader, WeightLoader
        if json_path and not hdf5_path:
            return DefinitionLoader.from_json_path(json_path)
        elif json_path and hdf5_path:
            return WeightLoader.load_weights_from_json_hdf5(json_path, hdf5_path, by_name=by_name)
        elif hdf5_path and not json_path:
            kmodel, bmodel = DefinitionLoader.from_hdf5_path(hdf5_path)
            WeightLoader.load_weights_from_kmodel(bmodel, kmodel)
            return bmodel


def of(cls, jcriterion, bigdl_type="float"):
        """
        Create a python Criterion by a java criterion object

        :param jcriterion: A java criterion object which created by Py4j
        :return: a criterion.
        """
        criterion = Criterion(bigdl_type, jcriterion)
        criterion.value = jcriterion
        criterion.bigdl_type = bigdl_type
        return criterion

def of ( cls , jcriterion , bigdl_type = "float" ) : criterion = Criterion ( bigdl_type , jcriterion ) criterion . value = jcriterion criterion . bigdl_type = bigdl_type return criterion


def load_weights_from_json_hdf5(def_json, weights_hdf5, by_name=False):
        """
        The file path can be stored in a local file system, HDFS, S3,
        or any Hadoop-supported file system.
        """
        bmodel = DefinitionLoader.from_json_path(def_json)
        def_value = BCommon.text_from_path(def_json)
        kmodel = model_from_json(def_value)
        WeightLoader.load_weights_from_hdf5(bmodel, kmodel, weights_hdf5, by_name)
        return bmodel

def load_weights_from_json_hdf5 ( def_json , weights_hdf5 , by_name = False ) : bmodel = DefinitionLoader . from_json_path ( def_json ) def_value = BCommon . text_from_path ( def_json ) kmodel = model_from_json ( def_value ) WeightLoader . load_weights_from_hdf5 ( bmodel , kmodel , weights_hdf5 , by_name ) return bmodel

The file path can be stored in a local file system HDFS S3 or any Hadoop - supported file system .


def load_imdb():
    """
    Load IMDB dataset
    Transform input data into an RDD of Sample
    """
    from keras.preprocessing import sequence
    from keras.datasets import imdb
    (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=20000)
    X_train = sequence.pad_sequences(X_train, maxlen=100)
    X_test = sequence.pad_sequences(X_test, maxlen=100)
    return X_train, y_train, X_test, y_test

def load_imdb ( ) : from keras . preprocessing import sequence from keras . datasets import imdb ( X_train , y_train ) , ( X_test , y_test ) = imdb . load_data ( nb_words = 20000 ) X_train = sequence . pad_sequences ( X_train , maxlen = 100 ) X_test = sequence . pad_sequences ( X_test , maxlen = 100 ) return X_train , y_train , X_test , y_test


def build_keras_model():
    """
    Define a recurrent convolutional model in Keras 1.2.2
    """
    from keras.models import Sequential
    from keras.layers import Dense, Dropout, Activation
    from keras.layers import Embedding
    from keras.layers import LSTM
    from keras.layers import Convolution1D, MaxPooling1D
    keras_model = Sequential()
    keras_model.add(Embedding(20000, 128, input_length=100))
    keras_model.add(Dropout(0.25))
    keras_model.add(Convolution1D(nb_filter=64,
                                  filter_length=5,
                                  border_mode='valid',
                                  activation='relu',
                                  subsample_length=1))
    keras_model.add(MaxPooling1D(pool_length=4))
    keras_model.add(LSTM(70))
    keras_model.add(Dense(1))
    keras_model.add(Activation('sigmoid'))
    return keras_model

def build_keras_model ( ) : from keras . models import Sequential from keras . layers import Dense , Dropout , Activation from keras . layers import Embedding from keras . layers import LSTM from keras . layers import Convolution1D , MaxPooling1D keras_model = Sequential ( ) keras_model . add ( Embedding ( 20000 , 128 , input_length = 100 ) ) keras_model . add ( Dropout ( 0.25 ) ) keras_model . add ( Convolution1D ( nb_filter = 64 , filter_length = 5 , border_mode = 'valid' , activation = 'relu' , subsample_length = 1 ) ) keras_model . add ( MaxPooling1D ( pool_length = 4 ) ) keras_model . add ( LSTM ( 70 ) ) keras_model . add ( Dense ( 1 ) ) keras_model . add ( Activation ( 'sigmoid' ) ) return keras_model

Define a recurrent convolutional model in Keras 1 . 2 . 2


def get_input_shape(self):
        """
        Return a list of shape tuples if there are multiple inputs.
        Return one shape tuple otherwise.
        """
        input = callBigDlFunc(self.bigdl_type, "getInputShape",
                              self.value)
        return self.__process_shape(input)

def get_input_shape ( self ) : input = callBigDlFunc ( self . bigdl_type , "getInputShape" , self . value ) return self . __process_shape ( input )

Return a list of shape tuples if there are multiple inputs . Return one shape tuple otherwise .


def get_output_shape(self):
        """
        Return a list of shape tuples if there are multiple outputs.
        Return one shape tuple otherwise.
        """
        output = callBigDlFunc(self.bigdl_type, "getOutputShape",
                               self.value)
        return self.__process_shape(output)

def get_output_shape ( self ) : output = callBigDlFunc ( self . bigdl_type , "getOutputShape" , self . value ) return self . __process_shape ( output )

Return a list of shape tuples if there are multiple outputs . Return one shape tuple otherwise .


def get_mnist(data_type="train", location="/tmp/mnist"):
    """
    Get mnist dataset with features and label as ndarray.
    Data would be downloaded automatically if it doesn't present at the specific location.

    :param data_type: "train" for training data and "test" for testing data.
    :param location: Location to store mnist dataset.
    :return: (features: ndarray, label: ndarray)
    """
    X, Y = mnist.read_data_sets(location, data_type)
    return X, Y + 1

def get_mnist ( data_type = "train" , location = "/tmp/mnist" ) : X , Y = mnist . read_data_sets ( location , data_type ) return X , Y + 1

Get mnist dataset with features and label as ndarray . Data would be downloaded automatically if it doesn t present at the specific location .


def read_data_sets(data_dir):
    """
    Parse or download movielens 1m  data if train_dir is empty.

    :param data_dir: The directory storing the movielens data
    :return: a 2D numpy array with user index and item index in each row
    """
    WHOLE_DATA = 'ml-1m.zip'
    local_file = base.maybe_download(WHOLE_DATA, data_dir, SOURCE_URL + WHOLE_DATA)
    zip_ref = zipfile.ZipFile(local_file, 'r')
    extracted_to = os.path.join(data_dir, "ml-1m")
    if not os.path.exists(extracted_to):
        print("Extracting %s to %s" % (local_file, data_dir))
        zip_ref.extractall(data_dir)
        zip_ref.close()
    rating_files = os.path.join(extracted_to,"ratings.dat")

    rating_list = [i.strip().split("::") for i in open(rating_files,"r").readlines()]
    movielens_data = np.array(rating_list).astype(int)
    return movielens_data

def
read_data_sets
(
data_dir
)
:
WHOLE_DATA
=
'ml-1m.zip'
local_file
=
base
.
maybe_download
(
WHOLE_DATA
,
data_dir
,
SOURCE_URL
+
WHOLE_DATA
)
zip_ref
=
zipfile
.
ZipFile
(
local_file
,
'r'
)
extracted_to
=
os
.
path
.
join
(
data_dir
,
"ml-1m"
)
if
not
os
.
path
.
exists
(
extracted_to
)
:
print
(
"Extracting %s to %s"
%
(
local_file
,
data_dir
)
)
zip_ref
.
extractall
(
data_dir
)
zip_ref
.
close
(
)
rating_files
=
os
.
path
.
join
(
extracted_to
,
"ratings.dat"
)
rating_list
=
[
i
.
strip
(
)
.
split
(
"::"
)
for
i
in
open
(
rating_files
,
"r"
)
.
readlines
(
)
]
movielens_data
=
np
.
array
(
rating_list
)
.
astype
(
int
)
return
movielens_data

Parse or download movielens 1m data if train_dir is empty .


def get_bigdl_classpath():
    """
    Get and return the jar path for bigdl if exists.
    """
    if os.getenv("BIGDL_CLASSPATH"):
        return os.environ["BIGDL_CLASSPATH"]
    jar_dir = os.path.abspath(__file__ + "/../../")
    jar_paths = glob.glob(os.path.join(jar_dir, "share/lib/*.jar"))
    if jar_paths:
        assert len(jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths)
        return jar_paths[0]
    return ""

def
get_bigdl_classpath
(
)
:
if
os
.
getenv
(
"BIGDL_CLASSPATH"
)
:
return
os
.
environ
[
"BIGDL_CLASSPATH"
]
jar_dir
=
os
.
path
.
abspath
(
__file__
+
"/../../"
)
jar_paths
=
glob
.
glob
(
os
.
path
.
join
(
jar_dir
,
"share/lib/*.jar"
)
)
if
jar_paths
:
assert
len
(
jar_paths
)
==
1
,
"Expecting one jar: %s"
%
len
(
jar_paths
)
return
jar_paths
[
0
]
return
""

Get and return the jar path for bigdl if exists .


def is_spark_below_2_2():
    """
    Check if spark version is below 2.2
    """
    import pyspark
    if(hasattr(pyspark,"version")):
        full_version = pyspark.version.__version__
        # We only need the general spark version (eg, 1.6, 2.2).
        parts = full_version.split(".")
        spark_version = parts[0] + "." + parts[1]
        if(compare_version(spark_version, "2.2")>=0):
            return False
    return True

def is_spark_below_2_2 ( ) : import pyspark if ( hasattr ( pyspark , "version" ) ) : full_version = pyspark . version . __version__ parts = full_version . split ( "." ) spark_version = parts [ 0 ] + "." + parts [ 1 ] if ( compare_version ( spark_version , "2.2" ) >= 0 ) : return False return True


def export_checkpoint(checkpoint_path):
    """
    Export variable tensors from the checkpoint files.

    :param checkpoint_path: tensorflow checkpoint path
    :return: dictionary of tensor. The key is the variable name and the value is the numpy
    """
    reader = tf.train.NewCheckpointReader(checkpoint_path)

    # Get tensor name list
    tensor_names = filter(lambda n: n!='global_step',
                          reader.get_variable_to_shape_map().keys())
    # Prepare key-value dictionary
    tensors = {}
    for tn in tensor_names:
        tensors[tn] = reader.get_tensor(tn)

    return tensors

def export_checkpoint ( checkpoint_path ) : reader = tf . train . NewCheckpointReader ( checkpoint_path ) tensor_names = filter ( lambda n : n != 'global_step' , reader . get_variable_to_shape_map ( ) . keys ( ) ) tensors = { } for tn in tensor_names : tensors [ tn ] = reader . get_tensor ( tn ) return tensors

Export variable tensors from the checkpoint files .


def save_variable_bigdl(tensors, target_path, bigdl_type="float"):
    """
    Save a variable dictionary to a Java object file, so it can be read by BigDL

    :param tensors: tensor dictionary
    :param target_path: where is the Java object file store
    :param bigdl_type: model variable numeric type
    :return: nothing
    """
    import numpy as np
    jtensors = {}
    for tn in tensors.keys():
        if not isinstance(tensors[tn], np.ndarray):
            value = np.array(tensors[tn])
        else:
            value = tensors[tn]
        jtensors[tn] = JTensor.from_ndarray(value)

    callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path)

def save_variable_bigdl ( tensors , target_path , bigdl_type = "float" ) : import numpy as np jtensors = { } for tn in tensors . keys ( ) : if not isinstance ( tensors [ tn ] , np . ndarray ) : value = np . array ( tensors [ tn ] ) else : value = tensors [ tn ] jtensors [ tn ] = JTensor . from_ndarray ( value ) callBigDlFunc ( bigdl_type , "saveTensorDictionary" , jtensors , target_path )

Save a variable dictionary to a Java object file so it can be read by BigDL


def expand_tile(units, axis):
    """
    Expand and tile tensor along given axis

    Args:
        units: tf tensor with dimensions [batch_size, time_steps, n_input_features]
        axis: axis along which expand and tile. Must be 1 or 2

    """
    assert axis in (1, 2)
    n_time_steps = K.int_shape(units)[1]
    repetitions = [1, 1, 1, 1]
    repetitions[axis] = n_time_steps
    if axis == 1:
        expanded = Reshape(target_shape=( (1,) + K.int_shape(units)[1:] ))(units)
    else:
        expanded = Reshape(target_shape=(K.int_shape(units)[1:2] + (1,) + K.int_shape(units)[2:]))(units)
    return K.tile(expanded, repetitions)

def expand_tile ( units , axis ) : assert axis in ( 1 , 2 ) n_time_steps = K . int_shape ( units ) [ 1 ] repetitions = [ 1 , 1 , 1 , 1 ] repetitions [ axis ] = n_time_steps if axis == 1 : expanded = Reshape ( target_shape = ( ( 1 , ) + K . int_shape ( units ) [ 1 : ] ) ) ( units ) else : expanded = Reshape ( target_shape = ( K . int_shape ( units ) [ 1 : 2 ] + ( 1 , ) + K . int_shape ( units ) [ 2 : ] ) ) ( units ) return K . tile ( expanded , repetitions )


def precompute_future_symbols(trie, n, allow_spaces=False):
    """
    Collecting possible continuations of length <= n for every node
    """
    if n == 0:
        return
    if trie.is_terminated and trie.precompute_symbols:
        # символы уже предпосчитаны
        return
    for index, final in enumerate(trie.final):
        trie.data[index] = [set() for i in range(n)]
    for index, (node_data, final) in enumerate(zip(trie.data, trie.final)):
        node_data[0] = set(trie._get_letters(index))
        if allow_spaces and final:
            node_data[0].add(" ")
    for d in range(1, n):
        for index, (node_data, final) in enumerate(zip(trie.data, trie.final)):
            children = set(trie._get_children(index))
            for child in children:
                node_data[d] |= trie.data[child][d - 1]
            # в случае, если разрешён возврат по пробелу в стартовое состояние
            if allow_spaces and final:
                node_data[d] |= trie.data[trie.root][d - 1]
    trie.terminated = True

def
precompute_future_symbols
(
trie
,
n
,
allow_spaces
=
False
)
:
if
n
==
0
:
return
if
trie
.
is_terminated
and
trie
.
precompute_symbols
:
return
for
index
,
final
in
enumerate
(
trie
.
final
)
:
trie
.
data
[
index
]
=
[
set
(
)
for
i
in
range
(
n
)
]
for
index
,
(
node_data
,
final
)
in
enumerate
(
zip
(
trie
.
data
,
trie
.
final
)
)
:
node_data
[
0
]
=
set
(
trie
.
_get_letters
(
index
)
)
if
allow_spaces
and
final
:
node_data
[
0
]
.
add
(
" "
)
for
d
in
range
(
1
,
n
)
:
for
index
,
(
node_data
,
final
)
in
enumerate
(
zip
(
trie
.
data
,
trie
.
final
)
)
:
children
=
set
(
trie
.
_get_children
(
index
)
)
for
child
in
children
:
node_data
[
d
]
|=
trie
.
data
[
child
]
[
d
1
]
if
allow_spaces
and
final
:
node_data
[
d
]
|=
trie
.
data
[
trie
.
root
]
[
d
1
]
trie
.
terminated
=
True

Collecting possible continuations of length < = n for every node


def simple_attention(memory, att_size, mask, keep_prob=1.0, scope="simple_attention"):
    """Simple attention without any conditions.

       Computes weighted sum of memory elements.
    """
    with tf.variable_scope(scope):
        BS, ML, MH = tf.unstack(tf.shape(memory))
        memory_do = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, 1, MH])
        logits = tf.layers.dense(tf.layers.dense(memory_do, att_size, activation=tf.nn.tanh), 1, use_bias=False)
        logits = softmax_mask(tf.squeeze(logits, [2]), mask)
        att_weights = tf.expand_dims(tf.nn.softmax(logits), axis=2)
        res = tf.reduce_sum(att_weights * memory, axis=1)
        return res

def simple_attention ( memory , att_size , mask , keep_prob = 1.0 , scope = "simple_attention" ) : with tf . variable_scope ( scope ) : BS , ML , MH = tf . unstack ( tf . shape ( memory ) ) memory_do = tf . nn . dropout ( memory , keep_prob = keep_prob , noise_shape = [ BS , 1 , MH ] ) logits = tf . layers . dense ( tf . layers . dense ( memory_do , att_size , activation = tf . nn . tanh ) , 1 , use_bias = False ) logits = softmax_mask ( tf . squeeze ( logits , [ 2 ] ) , mask ) att_weights = tf . expand_dims ( tf . nn . softmax ( logits ) , axis = 2 ) res = tf . reduce_sum ( att_weights * memory , axis = 1 ) return res

Simple attention without any conditions .


def attention(inputs, state, att_size, mask, scope="attention"):
    """Computes weighted sum of inputs conditioned on state"""
    with tf.variable_scope(scope):
        u = tf.concat([tf.tile(tf.expand_dims(state, axis=1), [1, tf.shape(inputs)[1], 1]), inputs], axis=2)
        logits = tf.layers.dense(tf.layers.dense(u, att_size, activation=tf.nn.tanh), 1, use_bias=False)
        logits = softmax_mask(tf.squeeze(logits, [2]), mask)
        att_weights = tf.expand_dims(tf.nn.softmax(logits), axis=2)
        res = tf.reduce_sum(att_weights * inputs, axis=1)
        return res, logits

def attention ( inputs , state , att_size , mask , scope = "attention" ) : with tf . variable_scope ( scope ) : u = tf . concat ( [ tf . tile ( tf . expand_dims ( state , axis = 1 ) , [ 1 , tf . shape ( inputs ) [ 1 ] , 1 ] ) , inputs ] , axis = 2 ) logits = tf . layers . dense ( tf . layers . dense ( u , att_size , activation = tf . nn . tanh ) , 1 , use_bias = False ) logits = softmax_mask ( tf . squeeze ( logits , [ 2 ] ) , mask ) att_weights = tf . expand_dims ( tf . nn . softmax ( logits ) , axis = 2 ) res = tf . reduce_sum ( att_weights * inputs , axis = 1 ) return res , logits


def compute_bleu(reference_corpus, translation_corpus, max_order=4,
                 smooth=False):
  """Computes BLEU score of translated segments against one or more references.

  Args:
    reference_corpus: list of lists of references for each translation. Each
        reference should be tokenized into a list of tokens.
    translation_corpus: list of translations to score. Each translation
        should be tokenized into a list of tokens.
    max_order: Maximum n-gram order to use when computing BLEU score.
    smooth: Whether or not to apply Lin et al. 2004 smoothing.

  Returns:
    3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram
    precisions and brevity penalty.
  """
  matches_by_order = [0] * max_order
  possible_matches_by_order = [0] * max_order
  reference_length = 0
  translation_length = 0
  for (references, translation) in zip(reference_corpus,
                                       translation_corpus):
    reference_length += min(len(r) for r in references)
    translation_length += len(translation)

    merged_ref_ngram_counts = collections.Counter()
    for reference in references:
      merged_ref_ngram_counts |= _get_ngrams(reference, max_order)
    translation_ngram_counts = _get_ngrams(translation, max_order)
    overlap = translation_ngram_counts & merged_ref_ngram_counts
    for ngram in overlap:
      matches_by_order[len(ngram)-1] += overlap[ngram]
    for order in range(1, max_order+1):
      possible_matches = len(translation) - order + 1
      if possible_matches > 0:
        possible_matches_by_order[order-1] += possible_matches

  precisions = [0] * max_order
  for i in range(0, max_order):
    if smooth:
      precisions[i] = ((matches_by_order[i] + 1.) /
                       (possible_matches_by_order[i] + 1.))
    else:
      if possible_matches_by_order[i] > 0:
        precisions[i] = (float(matches_by_order[i]) /
                         possible_matches_by_order[i])
      else:
        precisions[i] = 0.0

  if min(precisions) > 0:
    p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions)
    geo_mean = math.exp(p_log_sum)
  else:
    geo_mean = 0

  ratio = float(translation_length) / reference_length

  if ratio > 1.0:
    bp = 1.
  else:
    bp = math.exp(1 - 1. / ratio)

  bleu = geo_mean * bp

  return (bleu, precisions, bp, ratio, translation_length, reference_length)

def compute_bleu ( reference_corpus , translation_corpus , max_order = 4 , smooth = False ) : matches_by_order = [ 0 ] * max_order possible_matches_by_order = [ 0 ] * max_order reference_length = 0 translation_length = 0 for ( references , translation ) in zip ( reference_corpus , translation_corpus ) : reference_length += min ( len ( r ) for r in references ) translation_length += len ( translation ) merged_ref_ngram_counts = collections . Counter ( ) for reference in references : merged_ref_ngram_counts |= _get_ngrams ( reference , max_order ) translation_ngram_counts = _get_ngrams ( translation , max_order ) overlap = translation_ngram_counts & merged_ref_ngram_counts for ngram in overlap : matches_by_order [ len ( ngram ) - 1 ] += overlap [ ngram ] for order in range ( 1 , max_order + 1 ) : possible_matches = len ( translation ) - order + 1 if possible_matches > 0 : possible_matches_by_order [ order - 1 ] += possible_matches precisions = [ 0 ] * max_order for i in range ( 0 , max_order ) : if smooth : precisions [ i ] = ( ( matches_by_order [ i ] + 1. ) / ( possible_matches_by_order [ i ] + 1. ) ) else : if possible_matches_by_order [ i ] > 0 : precisions [ i ] = ( float ( matches_by_order [ i ] ) / possible_matches_by_order [ i ] ) else : precisions [ i ] = 0.0 if min ( precisions ) > 0 : p_log_sum = sum ( ( 1. / max_order ) * math . log ( p ) for p in precisions ) geo_mean = math . exp ( p_log_sum ) else : geo_mean = 0 ratio = float ( translation_length ) / reference_length if ratio > 1.0 : bp = 1. else : bp = math . exp ( 1 - 1. / ratio ) bleu = geo_mean * bp return ( bleu , precisions , bp , ratio , translation_length , reference_length )

Computes BLEU score of translated segments against one or more references .


def _get_log_file(self):
        """Returns opened file object for writing dialog logs.

        Returns:
            log_file: opened Python file object.
        """
        log_dir: Path = Path(self.config['log_path']).expanduser().resolve() / self.agent_name
        log_dir.mkdir(parents=True, exist_ok=True)
        log_file_path = Path(log_dir, f'{self._get_timestamp_utc_str()}_{self.agent_name}.log')
        log_file = open(log_file_path, 'a', buffering=1, encoding='utf8')
        return log_file

Returns opened file object for writing dialog logs .