def _learning_updates(self):
        """
        Return updates in the training.
        """
        params = self.training_params()
        gradients = self.get_gradients(params)
        return self.optimization_updates(params, gradients)

def _learning_updates ( self ) : params = self . training_params ( ) gradients = self . get_gradients ( params ) return self . optimization_updates ( params , gradients )


def training_params(self):
        """
        Get parameters to be optimized.
        """
        params = self.network.parameters
        # Freeze parameters
        if self.config.fixed_parameters:
            logging.info("fixed parameters: %s" % ", ".join(map(str, self.config.fixed_parameters)))
            params = [p for p in params if p not in self.config.fixed_parameters]
        return params

def
training_params
(
self
)
:
params
=
self
.
network
.
parameters
if
self
.
config
.
fixed_parameters
:
logging
.
info
(
"fixed parameters: %s"
%
", "
.
join
(
map
(
str
,
self
.
config
.
fixed_parameters
)
)
)
params
=
[
p
for
p
in
params
if
p
not
in
self
.
config
.
fixed_parameters
]
return
params


def optimization_updates(self, params, gradients):
        """
        Return updates from optimization.
        """
        updates, free_parameters = optimize_updates(params, gradients, self.config)
        self.network.free_parameters.extend(free_parameters)
        logging.info("Added %d free parameters for optimization" % len(free_parameters))
        return updates


def _first_glimpse_sensor(self, x_t):
        """
        Compute first glimpse position using down-sampled image.
        """
        downsampled_img = theano.tensor.signal.downsample.max_pool_2d(x_t, (4,4))
        downsampled_img = downsampled_img.flatten()
        first_l = T.dot(downsampled_img, self.W_f)
        if self.disable_reinforce:
            wf_grad = self.W_f
            if self.random_glimpse:
                first_l = self.srng.uniform((2,), low=-1.7, high=1.7)
        else:
            sampled_l_t = self._sample_gaussian(first_l, self.cov)
            sampled_pdf = self._multi_gaussian_pdf(disconnected_grad(sampled_l_t), first_l)
            wf_grad = T.grad(T.log(sampled_pdf), self.W_f)
            first_l = sampled_l_t
        return first_l, wf_grad

def _first_glimpse_sensor ( self , x_t ) : downsampled_img = theano . tensor . signal . downsample . max_pool_2d ( x_t , ( 4 , 4 ) ) downsampled_img = downsampled_img . flatten ( ) first_l = T . dot ( downsampled_img , self . W_f ) if self . disable_reinforce : wf_grad = self . W_f if self . random_glimpse : first_l = self . srng . uniform ( ( 2 , ) , low = - 1.7 , high = 1.7 ) else : sampled_l_t = self . _sample_gaussian ( first_l , self . cov ) sampled_pdf = self . _multi_gaussian_pdf ( disconnected_grad ( sampled_l_t ) , first_l ) wf_grad = T . grad ( T . log ( sampled_pdf ) , self . W_f ) first_l = sampled_l_t return first_l , wf_grad

Compute first glimpse position using down - sampled image .


def prepare(self):
        """
        All codes that create parameters should be put into 'setup' function.
        """
        self.output_dim = 10
        self.encoder = Chain(self.input_dim).stack(Dense(self.internal_layer_size, 'tanh'))
        self.decoder = Chain(self.internal_layer_size).stack(Dense(self.input_dim))
        self.classifier = Chain(self.internal_layer_size).stack(Dense(50, 'tanh'),
                                                      Dense(self.output_dim),
                                                      Softmax())

        self.register_inner_layers(self.encoder, self.decoder, self.classifier)

        self.target_input = T.ivector('target')
        self.register_external_inputs(self.target_input)

def prepare ( self ) : self . output_dim = 10 self . encoder = Chain ( self . input_dim ) . stack ( Dense ( self . internal_layer_size , 'tanh' ) ) self . decoder = Chain ( self . internal_layer_size ) . stack ( Dense ( self . input_dim ) ) self . classifier = Chain ( self . internal_layer_size ) . stack ( Dense ( 50 , 'tanh' ) , Dense ( self . output_dim ) , Softmax ( ) ) self . register_inner_layers ( self . encoder , self . decoder , self . classifier ) self . target_input = T . ivector ( 'target' ) self . register_external_inputs ( self . target_input )

All codes that create parameters should be put into setup function .


def compute_tensor(self, x):
        """
        Build the computation graph here.
        """
        internal_variable = self.encoder.compute_tensor(x)

        decoding_output = self.decoder.compute_tensor(internal_variable)

        classification_output = self.classifier.compute_tensor(internal_variable)

        auto_encoder_cost = AutoEncoderCost(decoding_output, x).get()

        classification_cost = CrossEntropyCost(classification_output, self.target_input).get()

        final_cost = 0.01 * auto_encoder_cost + classification_cost

        error_rate = ErrorRateCost(classification_output, self.target_input).get()

        self.register_monitors(("err", error_rate),
                               ("encoder_cost", auto_encoder_cost),
                               ("classify_cost", classification_cost))

        return final_cost

def compute_tensor ( self , x ) : internal_variable = self . encoder . compute_tensor ( x ) decoding_output = self . decoder . compute_tensor ( internal_variable ) classification_output = self . classifier . compute_tensor ( internal_variable ) auto_encoder_cost = AutoEncoderCost ( decoding_output , x ) . get ( ) classification_cost = CrossEntropyCost ( classification_output , self . target_input ) . get ( ) final_cost = 0.01 * auto_encoder_cost + classification_cost error_rate = ErrorRateCost ( classification_output , self . target_input ) . get ( ) self . register_monitors ( ( "err" , error_rate ) , ( "encoder_cost" , auto_encoder_cost ) , ( "classify_cost" , classification_cost ) ) return final_cost


def map(self, func):
        """
        Process all data with given function.
        The scheme of function should be x,y -> x,y.
        """
        if self._train_set:
            self._train_set = map(func, self._train_set)
        if self._valid_set:
            self._valid_set = map(func, self._valid_set)
        if self._test_set:
            self._test_set = map(func, self._test_set)

def map ( self , func ) : if self . _train_set : self . _train_set = map ( func , self . _train_set ) if self . _valid_set : self . _valid_set = map ( func , self . _valid_set ) if self . _test_set : self . _test_set = map ( func , self . _test_set )

Process all data with given function . The scheme of function should be x y - > x y .


def vectorize_target(self, size):
        """
        Make targets be one-hot vectors.
        """
        if self._train_set:
            self._train_set = self._vectorize_set(self._train_set, size)
        if self._valid_set:
            self._valid_set = self._vectorize_set(self._valid_set, size)
        if self._test_set:
            self._test_set = self._vectorize_set(self._test_set, size)

def vectorize_target ( self , size ) : if self . _train_set : self . _train_set = self . _vectorize_set ( self . _train_set , size ) if self . _valid_set : self . _valid_set = self . _vectorize_set ( self . _valid_set , size ) if self . _test_set : self . _test_set = self . _vectorize_set ( self . _test_set , size )


def report(self):
        """
        Print dataset statistics.
        """
        logging.info("%s train=%d valid=%d test=%d" % (self.__class__.__name__,
                                                       len(list(self._train_set)) if self._train_set else 0,
                                                       len(list(self._valid_set)) if self._valid_set else 0,
                                                       len(list(self._test_set)) if self._test_set else 0))

def
report
(
self
)
:
logging
.
info
(
"%s train=%d valid=%d test=%d"
%
(
self
.
__class__
.
__name__
,
len
(
list
(
self
.
_train_set
)
)
if
self
.
_train_set
else
0
,
len
(
list
(
self
.
_valid_set
)
)
if
self
.
_valid_set
else
0
,
len
(
list
(
self
.
_test_set
)
)
if
self
.
_test_set
else
0
)
)


def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        '''We train over mini-batches and evaluate periodically.'''
        iteration = 0
        while True:
            if not iteration % self.config.test_frequency and test_set:
                try:
                    self.test(iteration, test_set)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break

            if not iteration % self.validation_frequency and valid_set:
                try:
                    if not self.evaluate(iteration, valid_set):
                        logging.info('patience elapsed, bailing out')
                        break
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break

            train_message = ""
            try:
                train_message = self.train_func(train_set)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            if not iteration % self.config.monitor_frequency:
                logging.info('monitor (iter=%i) %s', iteration + 1, train_message)

            iteration += 1
            if hasattr(self.network, "iteration_callback"):
                self.network.iteration_callback()

            yield train_message

        if valid_set:
            self.set_params(self.best_params)
        if test_set:
            self.test(0, test_set)

def
train
(
self
,
train_set
,
valid_set
=
,
test_set
=
,
train_size
=
)
:
iteration
=
0
while
True
:
if
not
iteration
%
self
.
config
.
test_frequency
and
test_set
:
try
:
self
.
test
(
iteration
,
test_set
)
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
if
not
iteration
%
self
.
validation_frequency
and
valid_set
:
try
:
if
not
self
.
evaluate
(
iteration
,
valid_set
)
:
logging
.
info
(
'patience elapsed, bailing out'
)
break
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
train_message
=
""
try
:
train_message
=
self
.
train_func
(
train_set
)
except
KeyboardInterrupt
:
logging
.
info
(
'interrupted!'
)
break
if
not
iteration
%
self
.
config
.
monitor_frequency
:
logging
.
info
(
'monitor (iter=%i) %s'
,
iteration
+
1
,
train_message
)
iteration
+=
1
if
hasattr
(
self
.
network
,
"iteration_callback"
)
:
self
.
network
.
iteration_callback
(
)
yield
train_message
if
valid_set
:
self
.
set_params
(
self
.
best_params
)
if
test_set
:
self
.
test
(
0
,
test_set
)

We train over mini - batches and evaluate periodically .


def sample(self, input, steps):
        """
        Sample outputs from LM.
        """
        inputs = [[onehot(self.input_dim, x) for x in input]]
        for _ in range(steps):
            target = self.compute(inputs)[0,-1].argmax()
            input.append(target)
            inputs[0].append(onehot(self.input_dim, target))
        return input

def sample ( self , input , steps ) : inputs = [ [ onehot ( self . input_dim , x ) for x in input ] ] for _ in range ( steps ) : target = self . compute ( inputs ) [ 0 , - 1 ] . argmax ( ) input . append ( target ) inputs [ 0 ] . append ( onehot ( self . input_dim , target ) ) return input


def compute_alignments(self, prev_state, precomputed_values, mask=None):
        """
        Compute the alignment weights based on the previous state.
        """

        WaSp = T.dot(prev_state, self.Wa)
        UaH = precomputed_values
        # For test time the UaH will be (time, output_dim)
        if UaH.ndim == 2:
            preact = WaSp[:, None, :] + UaH[None, :, :]
        else:
            preact = WaSp[:, None, :] + UaH
        act = T.activate(preact, 'tanh')
        align_scores = T.dot(act, self.Va)  # ~ (batch, time)
        if mask:
            mask = (1 - mask) * -99.00
            if align_scores.ndim == 3:
                align_scores += mask[None, :]
            else:
                align_scores += mask
        align_weights = T.nnet.softmax(align_scores)
        return align_weights

def compute_alignments ( self , prev_state , precomputed_values , mask = None ) : WaSp = T . dot ( prev_state , self . Wa ) UaH = precomputed_values if UaH . ndim == 2 : preact = WaSp [ : , None , : ] + UaH [ None , : , : ] else : preact = WaSp [ : , None , : ] + UaH act = T . activate ( preact , 'tanh' ) align_scores = T . dot ( act , self . Va ) if mask : mask = ( 1 - mask ) * - 99.00 if align_scores . ndim == 3 : align_scores += mask [ None , : ] else : align_scores += mask align_weights = T . nnet . softmax ( align_scores ) return align_weights

Compute the alignment weights based on the previous state .


def compute_context_vector(self, prev_state, inputs, precomputed_values=None, mask=None):
        """
        Compute the context vector with soft attention.
        """
        precomputed_values = precomputed_values if precomputed_values else self.precompute(inputs)
        align_weights = self.compute_alignments(prev_state, precomputed_values, mask)
        context_vector = T.sum(align_weights[:, :, None] * inputs, axis=1)
        return context_vector

def compute_context_vector ( self , prev_state , inputs , precomputed_values = None , mask = None ) : precomputed_values = precomputed_values if precomputed_values else self . precompute ( inputs ) align_weights = self . compute_alignments ( prev_state , precomputed_values , mask ) context_vector = T . sum ( align_weights [ : , : , None ] * inputs , axis = 1 ) return context_vector

Compute the context vector with soft attention .


def concatenate(vars, axis=-1):
    """
    A utility function of concatenate.
    """
    from deepy.core.neural_var import NeuralVariable
    if isinstance(vars[0], NeuralVariable):
        concat_var = Concatenate(axis=axis).compute(*vars)
        if axis == -1 or axis == vars[0].tensor.ndim - 1:
            concat_var.output_dim = sum([x.output_dim for x in vars], 0)
    else:
        concat_var = TT.concatenate(vars, axis)
    return concat_var

def concatenate ( vars , axis = - 1 ) : from deepy . core . neural_var import NeuralVariable if isinstance ( vars [ 0 ] , NeuralVariable ) : concat_var = Concatenate ( axis = axis ) . compute ( * vars ) if axis == - 1 or axis == vars [ 0 ] . tensor . ndim - 1 : concat_var . output_dim = sum ( [ x . output_dim for x in vars ] , 0 ) else : concat_var = TT . concatenate ( vars , axis ) return concat_var


def _pad(self, side, length):
        """
        Pad sequences to given length in the left or right side.
        """
        if self._train_set:
            self._train_set = pad_dataset(self._train_set, side, length)
        if self._valid_set:
            self._valid_set = pad_dataset(self._valid_set, side, length)
        if self._test_set:
            self._test_set = pad_dataset(self._test_set, side, length)

def _pad ( self , side , length ) : if self . _train_set : self . _train_set = pad_dataset ( self . _train_set , side , length ) if self . _valid_set : self . _valid_set = pad_dataset ( self . _valid_set , side , length ) if self . _test_set : self . _test_set = pad_dataset ( self . _test_set , side , length )

Pad sequences to given length in the left or right side .


def rmsprop_core(params, gradients, momentum=0.9, learning_rate=0.01):
    """
    RMSPROP optimization core.
    """
    for param, grad in zip(params, gradients):
            rms_ = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_rms')
            rms = momentum * rms_ + (1 - momentum) * grad * grad
            yield rms_, rms
            yield param, param - learning_rate * grad / T.sqrt(rms + 1e-8)

def rmsprop_core ( params , gradients , momentum = 0.9 , learning_rate = 0.01 ) : for param , grad in zip ( params , gradients ) : rms_ = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_rms' ) rms = momentum * rms_ + ( 1 - momentum ) * grad * grad yield rms_ , rms yield param , param - learning_rate * grad / T . sqrt ( rms + 1e-8 )


def report(self):
        """
        Report elapsed time.
        """
        if not self.end_time:
            self.end()
        print ("Time: {} mins".format((self.end_time - self.start_time )/ 60))

def
report
(
self
)
:
if
not
self
.
end_time
:
self
.
end
(
)
print
(
"Time: {} mins"
.
format
(
(
self
.
end_time
self
.
start_time
)
/
60
)
)


def run(self, data_x):
        """
        Run the model with validation data and return costs.
        """
        output_vars = self.compute(*data_x)
        return self._extract_costs(output_vars)

def run ( self , data_x ) : output_vars = self . compute ( * data_x ) return self . _extract_costs ( output_vars )

Run the model with validation data and return costs .


def invoke(self):
        """
        This function will be called after each iteration.
        """
        self._counter += 1
        if self._counter % self._freq == 0:
            cnt = 0.
            sum_map = defaultdict(float)
            for x in self._trainer.get_data(self._data_split):
                val_map = self.run(x)
                if not isinstance(val_map, dict):
                    raise Exception("Monitor.run must return a dict.")
                for k, val in val_map.items():
                    sum_map[k] += val
                cnt += 1
            for k in sum_map:
                sum_map[k] /= cnt
            new_best = self.compare(sum_map)
            self._trainer.report(sum_map, self._data_split, new_best=new_best)
            if new_best:
                self._trainer.save_checkpoint(self._save_path)

def
invoke
(
self
)
:
self
.
_counter
+=
1
if
self
.
_counter
%
self
.
_freq
==
0
:
cnt
=
0.
sum_map
=
defaultdict
(
float
)
for
x
in
self
.
_trainer
.
get_data
(
self
.
_data_split
)
:
val_map
=
self
.
run
(
x
)
if
not
isinstance
(
val_map
,
dict
)
:
raise
Exception
(
"Monitor.run must return a dict."
)
for
k
,
val
in
val_map
.
items
(
)
:
sum_map
[
k
]
+=
val
cnt
+=
1
for
k
in
sum_map
:
sum_map
[
k
]
/=
cnt
new_best
=
self
.
compare
(
sum_map
)
self
.
_trainer
.
report
(
sum_map
,
self
.
_data_split
,
new_best
=
new_best
)
if
new_best
:
self
.
_trainer
.
save_checkpoint
(
self
.
_save_path
)

This function will be called after each iteration .


def _build_loop_vars(self):
        """
        Create inner loop variables.
        """
        from theano.tensor.var import TensorVariable
        from deepy.core.neural_var import NeuralVariable
        if not self._loop_vars:
            self._ordered_out_keys = self._outputs.keys()
            seq_keys = self._sequences.keys()
            filled_out_keys = [k for k in self._ordered_out_keys if self._outputs[k]]
            nonseq_keys = self._non_sequences.keys()
            dummy_tensors, self._scan_local_vars = get_dummy_args(
                sequences=[self._sequences[k].tensor for k in seq_keys],
                outputs_info=[self._outputs[k].tensor for k in self._ordered_out_keys],
                non_sequences=[self._non_sequences[k].tensor for k in nonseq_keys],
                **self._kwargs
            )
            dummy_map = dict(zip(seq_keys + filled_out_keys + nonseq_keys, dummy_tensors))
            arg_map = self._sequences.copy()
            arg_map.update(self._outputs)
            arg_map.update(self._non_sequences)
            self._loop_vars = LoopVars()
            for k, dummy_tensor in dummy_map.items():
                dummy_var = NeuralVariable(dummy_tensor, dim=arg_map[k].dim())
                self._loop_vars[k] = dummy_var

def _build_loop_vars ( self ) : from theano . tensor . var import TensorVariable from deepy . core . neural_var import NeuralVariable if not self . _loop_vars : self . _ordered_out_keys = self . _outputs . keys ( ) seq_keys = self . _sequences . keys ( ) filled_out_keys = [ k for k in self . _ordered_out_keys if self . _outputs [ k ] ] nonseq_keys = self . _non_sequences . keys ( ) dummy_tensors , self . _scan_local_vars = get_dummy_args ( sequences = [ self . _sequences [ k ] . tensor for k in seq_keys ] , outputs_info = [ self . _outputs [ k ] . tensor for k in self . _ordered_out_keys ] , non_sequences = [ self . _non_sequences [ k ] . tensor for k in nonseq_keys ] , ** self . _kwargs ) dummy_map = dict ( zip ( seq_keys + filled_out_keys + nonseq_keys , dummy_tensors ) ) arg_map = self . _sequences . copy ( ) arg_map . update ( self . _outputs ) arg_map . update ( self . _non_sequences ) self . _loop_vars = LoopVars ( ) for k , dummy_tensor in dummy_map . items ( ) : dummy_var = NeuralVariable ( dummy_tensor , dim = arg_map [ k ] . dim ( ) ) self . _loop_vars [ k ] = dummy_var


def _scan_step(self, vars):
        """
        Internal scan with dummy input variables.
        """
        from neural_var import NeuralVariable
        if not self._loop_vars:
            raise Exception("The loop is not initialized. To initialize the loop, use `with loop as vars`")
        replace_map = {}
        for k, var in vars.items():
            if var is not None:
                replace_map[self._dummy_nodes[k].tensor] = var.tensor
        outputs = {}
        for k in self._outputs:
            if k not in self._loop_vars:
                raise Exception("{} can not be found in loop vars.".format(k))
            output_node = theano.clone(self._loop_vars[k].tensor, replace_map)
            outputs[k] = NeuralVariable(output_node, self._loop_vars[k].dim())
        return outputs

Internal scan with dummy input variables .


def momentum_core(params, gradients, momentum=0.9, learning_rate=0.01):
    """
    Momentum SGD optimization core.
    """
    free_parameters = []
    updates = []
    for param, grad in zip(params, gradients):
            delta = learning_rate * grad
            velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '_vel')
            updates.append((velocity, momentum * velocity - delta))
            updates.append((param, param + velocity))
            free_parameters.append(velocity)
    return updates, free_parameters

def momentum_core ( params , gradients , momentum = 0.9 , learning_rate = 0.01 ) : free_parameters = [ ] updates = [ ] for param , grad in zip ( params , gradients ) : delta = learning_rate * grad velocity = theano . shared ( np . zeros_like ( param . get_value ( ) ) , name = param . name + '_vel' ) updates . append ( ( velocity , momentum * velocity - delta ) ) updates . append ( ( param , param + velocity ) ) free_parameters . append ( velocity ) return updates , free_parameters


def iftrain(self, then_branch, else_branch):
        """
        Execute `then_branch` when training.
        """
        return ifelse(self._training_flag, then_branch, else_branch, name="iftrain")

def iftrain ( self , then_branch , else_branch ) : return ifelse ( self . _training_flag , then_branch , else_branch , name = "iftrain" )


def skip(self, n_batches, n_epochs=0):
        """
        Skip N batches in the training.
        """
        logging.info("skip %d epochs and %d batches" % (n_epochs, n_batches))
        self._skip_batches = n_batches
        self._skip_epochs = n_epochs

def
skip
(
self
,
n_batches
,
n_epochs
=
0
)
:
logging
.
info
(
"skip %d epochs and %d batches"
%
(
n_epochs
,
n_batches
)
)
self
.
_skip_batches
=
n_batches
self
.
_skip_epochs
=
n_epochs


def load_params(self, path, exclude_free_params=False):
        """
        Load parameters for the training.
        This method can load free parameters and resume the training progress.
        """
        self.network.load_params(path, exclude_free_params=exclude_free_params)
        self.best_params = self.copy_params()
        # Resume the progress
        if self.network.train_logger.progress() > 0 or self.network.train_logger.epoch() > 0:
            self.skip(self.network.train_logger.progress(), self.network.train_logger.epoch() - 1)

def load_params ( self , path , exclude_free_params = False ) : self . network . load_params ( path , exclude_free_params = exclude_free_params ) self . best_params = self . copy_params ( ) if self . network . train_logger . progress ( ) > 0 or self . network . train_logger . epoch ( ) > 0 : self . skip ( self . network . train_logger . progress ( ) , self . network . train_logger . epoch ( ) - 1 )

Load parameters for the training . This method can load free parameters and resume the training progress .


def train(self, train_set, valid_set=None, test_set=None, train_size=None):
        """
        Train the model and return costs.
        """
        self._epoch = 0
        while True:
            if self._skip_epochs > 0:
                logging.info("skipping one epoch ...")
                self._skip_epochs -= 1
                self._epoch += 1
                yield None
                continue
            # Test
            if not self._epoch % self.config.test_frequency and test_set:
                try:
                    self._run_test(self._epoch, test_set)
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
            # Validate
            if not self._epoch % self.validation_frequency and valid_set:
                try:

                    if not self._run_valid(self._epoch, valid_set):
                        logging.info('patience elapsed, bailing out')
                        break
                except KeyboardInterrupt:
                    logging.info('interrupted!')
                    break
            # Train one step

            try:
                costs = self._run_train(self._epoch, train_set, train_size)
            except KeyboardInterrupt:
                logging.info('interrupted!')
                break
            # Check costs
            if np.isnan(costs[0][1]):
                logging.info("NaN detected in costs, rollback to last parameters")
                self.set_params(*self.checkpoint)
            else:
                self._epoch += 1
                self.network.epoch_callback()

            yield dict(costs)

        if valid_set and self.config.get("save_best_parameters", True):
            self.set_params(*self.best_params)
        if test_set:
            self._run_test(-1, test_set)


def _run_train(self, epoch, train_set, train_size=None):
        """
        Run one training iteration.
        """
        self.network.train_logger.record_epoch(epoch + 1)
        costs = self.train_step(train_set, train_size)
        if not epoch % self.config.monitor_frequency:
            self.report(dict(costs), "train", epoch)
        self.last_run_costs = costs
        return costs

def _run_train ( self , epoch , train_set , train_size = None ) : self . network . train_logger . record_epoch ( epoch + 1 ) costs = self . train_step ( train_set , train_size ) if not epoch % self . config . monitor_frequency : self . report ( dict ( costs ) , "train" , epoch ) self . last_run_costs = costs return costs


def _run_valid(self, epoch, valid_set, dry_run=False, save_path=None):
        """
        Run one valid iteration, return true if to continue training.
        """
        costs = self.valid_step(valid_set)
        # this is the same as: (J_i - J_f) / J_i > min improvement
        _, J = costs[0]
        new_best = False
        if self.best_cost - J > self.best_cost * self.min_improvement:
            # save the best cost and parameters
            self.best_params = self.copy_params()
            new_best = True
            if not dry_run:
                self.best_cost = J
                self.best_epoch = epoch
            self.save_checkpoint(save_path)

        self.report(dict(costs), type="valid", epoch=0 if dry_run else epoch, new_best=new_best)
        self.last_run_costs = costs
        return epoch - self.best_epoch < self.patience

def _run_valid ( self , epoch , valid_set , dry_run = False , save_path = None ) : costs = self . valid_step ( valid_set ) _ , J = costs [ 0 ] new_best = False if self . best_cost - J > self . best_cost * self . min_improvement : self . best_params = self . copy_params ( ) new_best = True if not dry_run : self . best_cost = J self . best_epoch = epoch self . save_checkpoint ( save_path ) self . report ( dict ( costs ) , type = "valid" , epoch = 0 if dry_run else epoch , new_best = new_best ) self . last_run_costs = costs return epoch - self . best_epoch < self . patience

Run one valid iteration return true if to continue training .


def report(self, score_map, type="valid", epoch=-1, new_best=False):
        """
        Report the scores and record them in the log.
        """
        type_str = type
        if len(type_str) < 5:
            type_str += " " * (5 - len(type_str))
        info = " ".join("%s=%.2f" % el for el in score_map.items())
        current_epoch = epoch if epoch > 0 else self.current_epoch()
        epoch_str = "epoch={}".format(current_epoch + 1)
        if epoch < 0:
            epoch_str = "dryrun"
            sys.stdout.write("\r")
            sys.stdout.flush()
        marker = " *" if new_best else ""
        message = "{} ({}) {}{}".format(type_str, epoch_str, info, marker)
        self.network.train_logger.record(message)
        logging.info(message)

def
report
(
self
,
score_map
,
type
=
"valid"
,
epoch
=
1
,
new_best
=
False
)
:
type_str
=
type
if
len
(
type_str
)
<
5
:
type_str
+=
" "
*
(
5
len
(
type_str
)
)
info
=
" "
.
join
(
"%s=%.2f"
%
el
for
el
in
score_map
.
items
(
)
)
current_epoch
=
epoch
if
epoch
>
0
else
self
.
current_epoch
(
)
epoch_str
=
"epoch={}"
.
format
(
current_epoch
+
1
)
if
epoch
<
0
:
epoch_str
=
"dryrun"
sys
.
stdout
.
write
(
"\r"
)
sys
.
stdout
.
flush
(
)
marker
=
" *"
if
new_best
else
""
message
=
"{} ({}) {}{}"
.
format
(
type_str
,
epoch_str
,
info
,
marker
)
self
.
network
.
train_logger
.
record
(
message
)
logging
.
info
(
message
)

Report the scores and record them in the log .


def get_data(self, data_split="train"):
        """
        Get specified split of data.
        """
        if data_split == 'train':
            return self._current_train_set
        elif data_split == 'valid':
            return self._current_valid_set
        elif data_split == 'test':
            return self._current_test_set
        else:
            return None

def get_data ( self , data_split = "train" ) : if data_split == 'train' : return self . _current_train_set elif data_split == 'valid' : return self . _current_valid_set elif data_split == 'test' : return self . _current_test_set else : return None


def apply(self, func, dim=None):
        """
        Apply a function to tensors.
        """
        output_dim = dim if dim else self.output_dim
        return NeuralVariable(func(self.tensor), output_dim)

def apply ( self , func , dim = None ) : output_dim = dim if dim else self . output_dim return NeuralVariable ( func ( self . tensor ) , output_dim )


def report(self):
        """
        Report usage of training parameters.
        """
        if self.logger:
            self.logger.info("accessed parameters:")
            for key in self.used_parameters:
                self.logger.info(" - %s %s" % (key, "(undefined)" if key in self.undefined_parameters else ""))

def
report
(
self
)
:
if
self
.
logger
:
self
.
logger
.
info
(
"accessed parameters:"
)
for
key
in
self
.
used_parameters
:
self
.
logger
.
info
(
" - %s %s"
%
(
key
,
"(undefined)"
if
key
in
self
.
undefined_parameters
else
""
)
)


def var(self, tensor_type, last_dim=0, test_shape=None):
        """
        An alias of deepy.tensor.var.
        """
        from deepy.tensor import var
        return var(tensor_type, last_dim=last_dim, test_shape=test_shape)

def var ( self , tensor_type , last_dim = 0 , test_shape = None ) : from deepy . tensor import var return var ( tensor_type , last_dim = last_dim , test_shape = test_shape )


def create_vars_from_data(self, dataset, split="train"):
        """
        Create vars given a dataset and set test values.
        Useful when dataset is already defined.
        """
        from deepy.core.neural_var import NeuralVariable
        vars = []
        if split == "valid":
            data_split = dataset.valid_set()
        elif split == "test":
            data_split = dataset.test_set()
        else:
            data_split = dataset.train_set()
        first_data_piece = list(data_split)[0]
        for i, numpy_tensor in enumerate(first_data_piece):
            if numpy_tensor.dtype == "int64":
                numpy_tensor = numpy_tensor.astype("int32")
            if numpy_tensor.dtype == "float64":
                numpy_tensor = numpy_tensor.astype(env.FLOATX)
            type_map = {
                0: "scalar",
                1: "vector",
                2: "matrix",
                3: "tensor3",
                4: "tensor4",
                5: "tensor5",
            }
            tensor_type = type_map[numpy_tensor.ndim] if numpy_tensor.ndim in type_map else type_map[0]
            if numpy_tensor.dtype.kind == "i":
                tensor_type = "i" + tensor_type
            theano_tensor = getattr(TT, tensor_type)("input_{}_{}".format(i + 1, tensor_type))
            last_dim = numpy_tensor.shape[-1]
            var = NeuralVariable(theano_tensor, dim=last_dim)
            var.set_test_value(numpy_tensor)
            vars.append(var)
        return vars

def create_vars_from_data ( self , dataset , split = "train" ) : from deepy . core . neural_var import NeuralVariable vars = [ ] if split == "valid" : data_split = dataset . valid_set ( ) elif split == "test" : data_split = dataset . test_set ( ) else : data_split = dataset . train_set ( ) first_data_piece = list ( data_split ) [ 0 ] for i , numpy_tensor in enumerate ( first_data_piece ) : if numpy_tensor . dtype == "int64" : numpy_tensor = numpy_tensor . astype ( "int32" ) if numpy_tensor . dtype == "float64" : numpy_tensor = numpy_tensor . astype ( env . FLOATX ) type_map = { 0 : "scalar" , 1 : "vector" , 2 : "matrix" , 3 : "tensor3" , 4 : "tensor4" , 5 : "tensor5" , } tensor_type = type_map [ numpy_tensor . ndim ] if numpy_tensor . ndim in type_map else type_map [ 0 ] if numpy_tensor . dtype . kind == "i" : tensor_type = "i" + tensor_type theano_tensor = getattr ( TT , tensor_type ) ( "input_{}_{}" . format ( i + 1 , tensor_type ) ) last_dim = numpy_tensor . shape [ - 1 ] var = NeuralVariable ( theano_tensor , dim = last_dim ) var . set_test_value ( numpy_tensor ) vars . append ( var ) return vars

Create vars given a dataset and set test values . Useful when dataset is already defined .


def shared(self, value, name=None):
        """
        Create a shared theano scalar value.
        """
        if type(value) == int:
            final_value = np.array(value, dtype="int32")
        elif type(value) == float:
            final_value = np.array(value, dtype=env.FLOATX)
        else:
            final_value = value

        return theano.shared(final_value, name=name)

def shared ( self , value , name = None ) : if type ( value ) == int : final_value = np . array ( value , dtype = "int32" ) elif type ( value ) == float : final_value = np . array ( value , dtype = env . FLOATX ) else : final_value = value return theano . shared ( final_value , name = name )


def stack_encoders(self, *layers):
        """
        Stack encoding layers, this must be done before stacking decoding layers.
        """
        self.stack(*layers)
        self.encoding_layes.extend(layers)

def stack_encoders ( self , * layers ) : self . stack ( * layers ) self . encoding_layes . extend ( layers )

Stack encoding layers this must be done before stacking decoding layers .


def stack_decoders(self, *layers):
        """
        Stack decoding layers.
        """
        self.stack(*layers)
        self.decoding_layers.extend(layers)

def stack_decoders ( self , * layers ) : self . stack ( * layers ) self . decoding_layers . extend ( layers )


def encode(self, x):
        """
        Encode given input.
        """
        if not self.encoding_network:
            self.encoding_network = NeuralNetwork(self.input_dim, self.input_tensor)
            self.encoding_network.input_variables = self.input_variables
            for layer in self.encoding_layes:
                self.encoding_network.stack_layer(layer, no_setup=True)
        return self.encoding_network.compute(*x)

def encode ( self , x ) : if not self . encoding_network : self . encoding_network = NeuralNetwork ( self . input_dim , self . input_tensor ) self . encoding_network . input_variables = self . input_variables for layer in self . encoding_layes : self . encoding_network . stack_layer ( layer , no_setup = True ) return self . encoding_network . compute ( * x )


def decode(self, x):
        """
        Decode given representation.
        """
        if not self.rep_dim:
            raise Exception("rep_dim must be set to decode.")
        if not self.decoding_network:
            self.decoding_network = NeuralNetwork(self.rep_dim)
            for layer in self.decoding_layers:
                self.decoding_network.stack_layer(layer, no_setup=True)
        return self.decoding_network.compute(x)

def
decode
(
self
,
x
)
:
if
not
self
.
rep_dim
:
raise
Exception
(
"rep_dim must be set to decode."
)
if
not
self
.
decoding_network
:
self
.
decoding_network
=
NeuralNetwork
(
self
.
rep_dim
)
for
layer
in
self
.
decoding_layers
:
self
.
decoding_network
.
stack_layer
(
layer
,
no_setup
=
True
)
return
self
.
decoding_network
.
compute
(
x
)


def create_2d_gaussian(dim, sigma):
    """
    This function creates a 2d gaussian kernel with the standard deviation
    denoted by sigma

    :param dim: integer denoting a side (1-d) of gaussian kernel
    :param sigma: floating point indicating the standard deviation

    :returns: a numpy 2d array
    """

    # check if the dimension is odd
    if dim % 2 == 0:
        raise ValueError("Kernel dimension should be odd")

    # initialize the kernel
    kernel = np.zeros((dim, dim), dtype=np.float16)

    # calculate the center point
    center = dim/2

    # calculate the variance
    variance = sigma ** 2

    # calculate the normalization coefficeint
    coeff = 1. / (2 * variance)

    # create the kernel
    for x in range(0, dim):
        for y in range(0, dim):
            x_val = abs(x - center)
            y_val = abs(y - center)
            numerator = x_val**2 + y_val**2
            denom = 2*variance

            kernel[x,y] = coeff * np.exp(-1. * numerator/denom)

    return kernel/sum(sum(kernel))

def
create_2d_gaussian
(
dim
,
sigma
)
:
if
dim
%
2
==
0
:
raise
ValueError
(
"Kernel dimension should be odd"
)
kernel
=
np
.
zeros
(
(
dim
,
dim
)
,
dtype
=
np
.
float16
)
center
=
dim
/
2
variance
=
sigma
**
2
coeff
=
1.
/
(
2
*
variance
)
for
x
in
range
(
0
,
dim
)
:
for
y
in
range
(
0
,
dim
)
:
x_val
=
abs
(
x
center
)
y_val
=
abs
(
y
center
)
numerator
=
x_val
**
2
+
y_val
**
2
denom
=
2
*
variance
kernel
[
x
,
y
]
=
coeff
*
np
.
exp
(
1.
*
numerator
/
denom
)
return
kernel
/
sum
(
sum
(
kernel
)
)


def register_layer(self, layer):
        """
        Register the layer so that it's param will be trained.
        But the output of the layer will not be stacked.
        """
        if type(layer) == Block:
            layer.fix()
        self.parameter_count += layer.parameter_count
        self.parameters.extend(layer.parameters)
        self.free_parameters.extend(layer.free_parameters)
        self.training_monitors.extend(layer.training_monitors)
        self.testing_monitors.extend(layer.testing_monitors)
        self.updates.extend(layer.updates)
        self.training_updates.extend(layer.training_updates)
        self.input_variables.extend(layer.external_inputs)
        self.target_variables.extend(layer.external_targets)

        self.training_callbacks.extend(layer.training_callbacks)
        self.testing_callbacks.extend(layer.testing_callbacks)
        self.epoch_callbacks.extend(layer.epoch_callbacks)

def register_layer ( self , layer ) : if type ( layer ) == Block : layer . fix ( ) self . parameter_count += layer . parameter_count self . parameters . extend ( layer . parameters ) self . free_parameters . extend ( layer . free_parameters ) self . training_monitors . extend ( layer . training_monitors ) self . testing_monitors . extend ( layer . testing_monitors ) self . updates . extend ( layer . updates ) self . training_updates . extend ( layer . training_updates ) self . input_variables . extend ( layer . external_inputs ) self . target_variables . extend ( layer . external_targets ) self . training_callbacks . extend ( layer . training_callbacks ) self . testing_callbacks . extend ( layer . testing_callbacks ) self . epoch_callbacks . extend ( layer . epoch_callbacks )

Register the layer so that it s param will be trained . But the output of the layer will not be stacked .


def monitor_layer_outputs(self):
        """
        Monitoring the outputs of each layer.
        Useful for troubleshooting convergence problems.
        """
        for layer, hidden in zip(self.layers, self._hidden_outputs):
            self.training_monitors.append(('mean(%s)' % (layer.name), abs(hidden).mean()))

def monitor_layer_outputs ( self ) : for layer , hidden in zip ( self . layers , self . _hidden_outputs ) : self . training_monitors . append ( ( 'mean(%s)' % ( layer . name ) , abs ( hidden ) . mean ( ) ) )

Monitoring the outputs of each layer . Useful for troubleshooting convergence problems .


def all_parameters(self):
        """
        Return all parameters.
        """
        params = []
        params.extend(self.parameters)
        params.extend(self.free_parameters)

        return params

def all_parameters ( self ) : params = [ ] params . extend ( self . parameters ) params . extend ( self . free_parameters ) return params


def setup_variables(self):
        """
        Set up variables.
        """
        if self.input_tensor:
            if type(self.input_tensor) == int:
                x = dim_to_var(self.input_tensor, name="x")
            else:
                x = self.input_tensor
        else:
            x = T.matrix('x')
        self.input_variables.append(x)
        self._output = x
        self._test_output = x

def setup_variables ( self ) : if self . input_tensor : if type ( self . input_tensor ) == int : x = dim_to_var ( self . input_tensor , name = "x" ) else : x = self . input_tensor else : x = T . matrix ( 'x' ) self . input_variables . append ( x ) self . _output = x self . _test_output = x


def compute(self, *x):
        """
        Return network output.
        """
        self._compile()
        outs = self._compute(*x)
        if self._output_keys:
            return MapDict(dict(zip(self._output_keys, outs)))
        else:
            return outs

def compute ( self , * x ) : self . _compile ( ) outs = self . _compute ( * x ) if self . _output_keys : return MapDict ( dict ( zip ( self . _output_keys , outs ) ) ) else : return outs


def save_params(self, path, new_thread=False):
        """
        Save parameters to file.
        """
        save_logger.info(path)
        param_variables = self.all_parameters
        params = [p.get_value().copy() for p in param_variables]
        if new_thread:
            thread = Thread(target=save_network_params, args=(params, path))
            thread.start()
        else:
            save_network_params(params, path)
        self.train_logger.save(path)

def save_params ( self , path , new_thread = False ) : save_logger . info ( path ) param_variables = self . all_parameters params = [ p . get_value ( ) . copy ( ) for p in param_variables ] if new_thread : thread = Thread ( target = save_network_params , args = ( params , path ) ) thread . start ( ) else : save_network_params ( params , path ) self . train_logger . save ( path )


def load_params(self, path, exclude_free_params=False):
        """
        Load parameters from file.
        """
        if not os.path.exists(path): return;
        logging.info("loading parameters from %s" % path)
        # Decide which parameters to load
        if exclude_free_params:
            params_to_load = self.parameters
        else:
            params_to_load = self.all_parameters
        # Load parameters
        if path.endswith(".gz"):
            opener = gzip.open if path.lower().endswith('.gz') else open
            handle = opener(path, 'rb')
            saved_params = pickle.load(handle)
            handle.close()
            # Write parameters
            for target, source in zip(params_to_load, saved_params):
                logging.info('%s: setting value %s', target.name, source.shape)
                target.set_value(source)
        elif path.endswith(".npz"):
            arrs = np.load(path)
            # Write parameters
            for target, idx in zip(params_to_load, range(len(arrs.keys()))):
                source = arrs['arr_%d' % idx]
                logging.info('%s: setting value %s', target.name, source.shape)
                target.set_value(source)
        else:
            raise Exception("File format of %s is not supported, use '.gz' or '.npz' or '.uncompressed.gz'" % path)

        self.train_logger.load(path)


def report(self):
        """
        Print network statistics.
        """
        logging.info("network inputs: %s", " ".join(map(str, self.input_variables)))
        logging.info("network targets: %s", " ".join(map(str, self.target_variables)))
        logging.info("network parameters: %s", " ".join(map(str, self.all_parameters)))
        logging.info("parameter count: %d", self.parameter_count)

def
report
(
self
)
:
logging
.
info
(
"network inputs: %s"
,
" "
.
join
(
map
(
str
,
self
.
input_variables
)
)
)
logging
.
info
(
"network targets: %s"
,
" "
.
join
(
map
(
str
,
self
.
target_variables
)
)
)
logging
.
info
(
"network parameters: %s"
,
" "
.
join
(
map
(
str
,
self
.
all_parameters
)
)
)
logging
.
info
(
"parameter count: %d"
,
self
.
parameter_count
)


def register_parameters(self, *parameters):
        """
        Register parameters.
        """
        for param in parameters:
            self.parameter_count += np.prod(param.get_value().shape)
        self.parameters.extend(parameters)

def register_parameters ( self , * parameters ) : for param in parameters : self . parameter_count += np . prod ( param . get_value ( ) . shape ) self . parameters . extend ( parameters )


def register_updates(self, *updates):
        """
        Register updates that will be executed in each iteration.
        """
        for key, node in updates:
            if key not in self._registered_updates:
                self.updates.append((key, node))
                self._registered_updates.add(key)

def register_updates ( self , * updates ) : for key , node in updates : if key not in self . _registered_updates : self . updates . append ( ( key , node ) ) self . _registered_updates . add ( key )

Register updates that will be executed in each iteration .


def register_training_updates(self, *updates):
        """
        Register updates that will only be executed in training phase.
        """
        for key, node in updates:
            if key not in self._registered_training_updates:
                self.training_updates.append((key, node))
                self._registered_training_updates.add(key)

def register_training_updates ( self , * updates ) : for key , node in updates : if key not in self . _registered_training_updates : self . training_updates . append ( ( key , node ) ) self . _registered_training_updates . add ( key )

Register updates that will only be executed in training phase .


def register_monitors(self, *monitors):
        """
        Register monitors they should be tuple of name and Theano variable.
        """
        for key, node in monitors:
            if key not in self._registered_monitors:
                node *= 1.0 # Avoid CudaNdarray
                self.training_monitors.append((key, node))
                self.testing_monitors.append((key, node))
                self._registered_monitors.add(key)

def register_monitors ( self , * monitors ) : for key , node in monitors : if key not in self . _registered_monitors : node *= 1.0 self . training_monitors . append ( ( key , node ) ) self . testing_monitors . append ( ( key , node ) ) self . _registered_monitors . add ( key )

Register monitors they should be tuple of name and Theano variable .


def multiple_l2_norm(tensors):
    """
    Get the L2 norm of multiple tensors.
    This function is taken from blocks.
    """
    # Another way for doing this, I don't know which one is fast
    # return T.sqrt(sum(T.sum(t ** 2) for t in tensors))
    flattened = [T.as_tensor_variable(t).flatten() for t in tensors]
    flattened = [(t if t.ndim > 0 else t.dimshuffle('x'))
                 for t in flattened]
    joined = T.join(0, *flattened)
    return T.sqrt(T.sqr(joined).sum())

def multiple_l2_norm ( tensors ) : flattened = [ T . as_tensor_variable ( t ) . flatten ( ) for t in tensors ] flattened = [ ( t if t . ndim > 0 else t . dimshuffle ( 'x' ) ) for t in flattened ] joined = T . join ( 0 , * flattened ) return T . sqrt ( T . sqr ( joined ) . sum ( ) )

Get the L2 norm of multiple tensors . This function is taken from blocks .


def dump_one(elt_to_pickle, file_obj):
        """
        dumps one element to file_obj, a file opened in write mode
        """
        pickled_elt_str = dumps(elt_to_pickle)
        file_obj.write(pickled_elt_str)
        # record separator is a blank line
        # (since pickled_elt_str might contain its own newlines)
        file_obj.write('\n\n')

def dump_one ( elt_to_pickle , file_obj ) : pickled_elt_str = dumps ( elt_to_pickle ) file_obj . write ( pickled_elt_str ) file_obj . write ( '\n\n' )

dumps one element to file_obj a file opened in write mode


def load(file_obj):
        """
        load contents from file_obj, returning a generator that yields one
        element at a time
        """
        cur_elt = []
        for line in file_obj:
          cur_elt.append(line)

          if line == '\n':
            pickled_elt_str = ''.join(cur_elt)
            cur_elt = []
            try:
                elt = loads(pickled_elt_str)
            except ValueError:
                continue

            yield elt

def load ( file_obj ) : cur_elt = [ ] for line in file_obj : cur_elt . append ( line ) if line == '\n' : pickled_elt_str = '' . join ( cur_elt ) cur_elt = [ ] try : elt = loads ( pickled_elt_str ) except ValueError : continue yield elt

load contents from file_obj returning a generator that yields one element at a time


def load_params(self, path, exclude_free_params=False):
        from deepy.core import graph
        """
        Load parameters to the block.
        """
        from deepy.core.comp_graph import ComputationalGraph
        model = graph.compile(blocks=[self])
        model.load_params(path, exclude_free_params=exclude_free_params)

def load_params ( self , path , exclude_free_params = False ) : from deepy . core import graph from deepy . core . comp_graph import ComputationalGraph model = graph . compile ( blocks = [ self ] ) model . load_params ( path , exclude_free_params = exclude_free_params )


def create_request_elements(
            cls, request_type, credentials, url, method='GET', params=None,
            headers=None, body='', secret=None, redirect_uri='', scope='',
            csrf='', user_state=''
    ):
        """
        Creates |oauth2| request elements.
        """

        headers = headers or {}
        params = params or {}

        consumer_key = credentials.consumer_key or ''
        consumer_secret = credentials.consumer_secret or ''
        token = credentials.token or ''
        refresh_token = credentials.refresh_token or credentials.token or ''

        # Separate url base and query parameters.
        url, base_params = cls._split_url(url)

        # Add params extracted from URL.
        params.update(dict(base_params))

        if request_type == cls.USER_AUTHORIZATION_REQUEST_TYPE:
            # User authorization request.
            # TODO: Raise error for specific message for each missing argument.
            if consumer_key and redirect_uri and (
                    csrf or not cls.supports_csrf_protection):
                params['client_id'] = consumer_key
                params['redirect_uri'] = redirect_uri
                params['scope'] = scope
                if cls.supports_user_state:
                    params['state'] = base64.urlsafe_b64encode(
                        json.dumps(
                            {"csrf": csrf, "user_state": user_state}
                        ).encode('utf-8')
                    )
                else:
                    params['state'] = csrf
                params['response_type'] = 'code'

                # Add authorization header
                headers.update(cls._authorization_header(credentials))
            else:
                raise OAuth2Error(
                    'Credentials with valid consumer_key and arguments '
                    'redirect_uri, scope and state are required to create '
                    'OAuth 2.0 user authorization request elements!')

        elif request_type == cls.ACCESS_TOKEN_REQUEST_TYPE:
            # Access token request.
            if consumer_key and consumer_secret:
                params['code'] = token
                params['client_id'] = consumer_key
                params['client_secret'] = consumer_secret
                params['redirect_uri'] = redirect_uri
                params['grant_type'] = 'authorization_code'

                # TODO: Check whether all providers accept it
                headers.update(cls._authorization_header(credentials))
            else:
                raise OAuth2Error(
                    'Credentials with valid token, consumer_key, '
                    'consumer_secret and argument redirect_uri are required '
                    'to create OAuth 2.0 access token request elements!')

        elif request_type == cls.REFRESH_TOKEN_REQUEST_TYPE:
            # Refresh access token request.
            if refresh_token and consumer_key and consumer_secret:
                params['refresh_token'] = refresh_token
                params['client_id'] = consumer_key
                params['client_secret'] = consumer_secret
                params['grant_type'] = 'refresh_token'
            else:
                raise OAuth2Error(
                    'Credentials with valid refresh_token, consumer_key, '
                    'consumer_secret are required to create OAuth 2.0 '
                    'refresh token request elements!')

        elif request_type == cls.PROTECTED_RESOURCE_REQUEST_TYPE:
            # Protected resource request.

            # Add Authorization header. See:
            # http://tools.ietf.org/html/rfc6749#section-7.1
            if credentials.token_type == cls.BEARER:
                # http://tools.ietf.org/html/rfc6750#section-2.1
                headers.update(
                    {'Authorization': 'Bearer {0}'.format(credentials.token)})

            elif token:
                params['access_token'] = token
            else:
                raise OAuth2Error(
                    'Credentials with valid token are required to create '
                    'OAuth 2.0 protected resources request elements!')

        request_elements = core.RequestElements(
            url, method, params, headers, body)

        return cls._x_request_elements_filter(
            request_type, request_elements, credentials)


def decode_state(cls, state, param='user_state'):
        """
        Decode state and return param.

        :param str state:
            state parameter passed through by provider

        :param str param:
            key to query from decoded state variable. Options include 'csrf'
            and 'user_state'.

        :returns:
            string value from decoded state

        """
        if state and cls.supports_user_state:
            # urlsafe_b64 may include = which the browser quotes so must
            # unquote Cast to str to void b64decode translation error. Base64
            # should be str compatible.
            return json.loads(base64.urlsafe_b64decode(
                unquote(str(state))).decode('utf-8'))[param]
        else:
            return state if param == 'csrf' else ''

def decode_state ( cls , state , param = 'user_state' ) : if state and cls . supports_user_state : return json . loads ( base64 . urlsafe_b64decode ( unquote ( str ( state ) ) ) . decode ( 'utf-8' ) ) [ param ] else : return state if param == 'csrf' else ''


def _x_credentials_parser(credentials, data):
        """
        We need to override this method to fix Facebooks naming deviation.
        """

        # Facebook returns "expires" instead of "expires_in".
        credentials.expire_in = data.get('expires')

        if data.get('token_type') == 'bearer':
            # TODO: cls is not available here, hardcode for now.
            credentials.token_type = 'Bearer'

        return credentials

def _x_credentials_parser ( credentials , data ) : credentials . expire_in = data . get ( 'expires' ) if data . get ( 'token_type' ) == 'bearer' : credentials . token_type = 'Bearer' return credentials

We need to override this method to fix Facebooks naming deviation .


def _x_request_elements_filter(cls, request_type, request_elements,
                                   credentials):
        """
        Google doesn't accept client ID and secret to be at the same time in
        request parameters and in the basic authorization header in the access
        token request.
        """
        if request_type is cls.ACCESS_TOKEN_REQUEST_TYPE:
            params = request_elements[2]
            del params['client_id']
            del params['client_secret']
        return request_elements

def _x_request_elements_filter ( cls , request_type , request_elements , credentials ) : if request_type is cls . ACCESS_TOKEN_REQUEST_TYPE : params = request_elements [ 2 ] del params [ 'client_id' ] del params [ 'client_secret' ] return request_elements

Google doesn t accept client ID and secret to be at the same time in request parameters and in the basic authorization header in the access token request .


def login(provider_name):
    """
    Login handler, must accept both GET and POST to be able to use OpenID.
    """

    # We need response object for the WerkzeugAdapter.
    response = make_response()

    # Log the user in, pass it the adapter and the provider name.
    result = authomatic.login(
        WerkzeugAdapter(
            request,
            response),
        provider_name)

    # If there is no LoginResult object, the login procedure is still pending.
    if result:
        if result.user:
            # We need to update the user to get more info.
            result.user.update()

        # The rest happens inside the template.
        return render_template('login.html', result=result)

    # Don't forget to return the response.
    return response

def login ( provider_name ) : response = make_response ( ) result = authomatic . login ( WerkzeugAdapter ( request , response ) , provider_name ) if result : if result . user : result . user . update ( ) return render_template ( 'login.html' , result = result ) return response

Login handler must accept both GET and POST to be able to use OpenID .


def normalize_dict(dict_):
    """
    Replaces all values that are single-item iterables with the value of its
    index 0.

    :param dict dict_:
        Dictionary to normalize.

    :returns:
        Normalized dictionary.

    """

    return dict([(k, v[0] if not isinstance(v, str) and len(v) == 1 else v)
                 for k, v in list(dict_.items())])

def normalize_dict ( dict_ ) : return dict ( [ ( k , v [ 0 ] if not isinstance ( v , str ) and len ( v ) == 1 else v ) for k , v in list ( dict_ . items ( ) ) ] )

Replaces all values that are single - item iterables with the value of its index 0 .


def items_to_dict(items):
    """
    Converts list of tuples to dictionary with duplicate keys converted to
    lists.

    :param list items:
        List of tuples.

    :returns:
        :class:`dict`

    """

    res = collections.defaultdict(list)

    for k, v in items:
        res[k].append(v)

    return normalize_dict(dict(res))

def items_to_dict ( items ) : res = collections . defaultdict ( list ) for k , v in items : res [ k ] . append ( v ) return normalize_dict ( dict ( res ) )

Converts list of tuples to dictionary with duplicate keys converted to lists .


def json_qs_parser(body):
    """
    Parses response body from JSON, XML or query string.

    :param body:
        string

    :returns:
        :class:`dict`, :class:`list` if input is JSON or query string,
        :class:`xml.etree.ElementTree.Element` if XML.

    """
    try:
        # Try JSON first.
        return json.loads(body)
    except (OverflowError, TypeError, ValueError):
        pass

    try:
        # Then XML.
        return ElementTree.fromstring(body)
    except (ElementTree.ParseError, TypeError, ValueError):
        pass

    # Finally query string.
    return dict(parse.parse_qsl(body))

def json_qs_parser ( body ) : try : return json . loads ( body ) except ( OverflowError , TypeError , ValueError ) : pass try : return ElementTree . fromstring ( body ) except ( ElementTree . ParseError , TypeError , ValueError ) : pass return dict ( parse . parse_qsl ( body ) )

Parses response body from JSON XML or query string .


def resolve_provider_class(class_):
    """
    Returns a provider class.

    :param class_name: :class:`string` or
    :class:`authomatic.providers.BaseProvider` subclass.

    """

    if isinstance(class_, str):
        # prepare path for authomatic.providers package
        path = '.'.join([__package__, 'providers', class_])

        # try to import class by string from providers module or by fully
        # qualified path
        return import_string(class_, True) or import_string(path)
    else:
        return class_

def resolve_provider_class ( class_ ) : if isinstance ( class_ , str ) : path = '.' . join ( [ __package__ , 'providers' , class_ ] ) return import_string ( class_ , True ) or import_string ( path ) else : return class_


def create_cookie(self, delete=None):
        """
        Creates the value for ``Set-Cookie`` HTTP header.

        :param bool delete:
            If ``True`` the cookie value will be ``deleted`` and the
            Expires value will be ``Thu, 01-Jan-1970 00:00:01 GMT``.

        """
        value = 'deleted' if delete else self._serialize(self.data)
        split_url = parse.urlsplit(self.adapter.url)
        domain = split_url.netloc.split(':')[0]

        # Work-around for issue #11, failure of WebKit-based browsers to accept
        # cookies set as part of a redirect response in some circumstances.
        if '.' not in domain:
            template = '{name}={value}; Path={path}; HttpOnly{secure}{expires}'
        else:
            template = ('{name}={value}; Domain={domain}; Path={path}; '
                        'HttpOnly{secure}{expires}')

        return template.format(
            name=self.name,
            value=value,
            domain=domain,
            path=split_url.path,
            secure='; Secure' if self.secure else '',
            expires='; Expires=Thu, 01-Jan-1970 00:00:01 GMT' if delete else ''
        )

Creates the value for Set - Cookie HTTP header .


def save(self):
        """
        Adds the session cookie to headers.
        """
        if self.data:
            cookie = self.create_cookie()
            cookie_len = len(cookie)

            if cookie_len > 4093:
                raise SessionError('Cookie too long! The cookie size {0} '
                                   'is more than 4093 bytes.'
                                   .format(cookie_len))

            self.adapter.set_header('Set-Cookie', cookie)

            # Reset data
            self._data = {}

def
save
(
self
)
:
if
self
.
data
:
cookie
=
self
.
create_cookie
(
)
cookie_len
=
len
(
cookie
)
if
cookie_len
>
4093
:
raise
SessionError
(
'Cookie too long! The cookie size {0} '
'is more than 4093 bytes.'
.
format
(
cookie_len
)
)
self
.
adapter
.
set_header
(
'Set-Cookie'
,
cookie
)
self
.
_data
=
{
}


def _get_data(self):
        """
        Extracts the session data from cookie.
        """
        cookie = self.adapter.cookies.get(self.name)
        return self._deserialize(cookie) if cookie else {}

def _get_data ( self ) : cookie = self . adapter . cookies . get ( self . name ) return self . _deserialize ( cookie ) if cookie else { }


def data(self):
        """
        Gets session data lazily.
        """
        if not self._data:
            self._data = self._get_data()
        # Always return a dict, even if deserialization returned nothing
        if self._data is None:
            self._data = {}
        return self._data

def data ( self ) : if not self . _data : self . _data = self . _get_data ( ) if self . _data is None : self . _data = { } return self . _data


def _signature(self, *parts):
        """
        Creates signature for the session.
        """
        signature = hmac.new(six.b(self.secret), digestmod=hashlib.sha1)
        signature.update(six.b('|'.join(parts)))
        return signature.hexdigest()

def _signature ( self , * parts ) : signature = hmac . new ( six . b ( self . secret ) , digestmod = hashlib . sha1 ) signature . update ( six . b ( '|' . join ( parts ) ) ) return signature . hexdigest ( )


def _serialize(self, value):
        """
        Converts the value to a signed string with timestamp.

        :param value:
            Object to be serialized.

        :returns:
            Serialized value.

        """

        # data = copy.deepcopy(value)
        data = value

        # 1. Serialize
        serialized = pickle.dumps(data).decode('latin-1')

        # 2. Encode
        # Percent encoding produces smaller result then urlsafe base64.
        encoded = parse.quote(serialized, '')

        # 3. Concatenate
        timestamp = str(int(time.time()))
        signature = self._signature(self.name, encoded, timestamp)
        concatenated = '|'.join([encoded, timestamp, signature])

        return concatenated

def _serialize ( self , value ) : data = value serialized = pickle . dumps ( data ) . decode ( 'latin-1' ) encoded = parse . quote ( serialized , '' ) timestamp = str ( int ( time . time ( ) ) ) signature = self . _signature ( self . name , encoded , timestamp ) concatenated = '|' . join ( [ encoded , timestamp , signature ] ) return concatenated

Converts the value to a signed string with timestamp .


def valid(self):
        """
        ``True`` if credentials are valid, ``False`` if expired.
        """

        if self.expiration_time:
            return self.expiration_time > int(time.time())
        else:
            return True

def valid ( self ) : if self . expiration_time : return self . expiration_time > int ( time . time ( ) ) else : return True

True if credentials are valid False if expired .


def expire_soon(self, seconds):
        """
        Returns ``True`` if credentials expire sooner than specified.

        :param int seconds:
            Number of seconds.

        :returns:
            ``True`` if credentials expire sooner than specified,
            else ``False``.

        """

        if self.expiration_time:
            return self.expiration_time < int(time.time()) + int(seconds)
        else:
            return False

def expire_soon ( self , seconds ) : if self . expiration_time : return self . expiration_time < int ( time . time ( ) ) + int ( seconds ) else : return False

Returns True if credentials expire sooner than specified .


def serialize(self):
        """
        Converts the credentials to a percent encoded string to be stored for
        later use.

        :returns:
            :class:`string`

        """

        if self.provider_id is None:
            raise ConfigError(
                'To serialize credentials you need to specify a '
                'unique integer under the "id" key in the config '
                'for each provider!')

        # Get the provider type specific items.
        rest = self.provider_type_class().to_tuple(self)

        # Provider ID and provider type ID are always the first two items.
        result = (self.provider_id, self.provider_type_id) + rest

        # Make sure that all items are strings.
        stringified = [str(i) for i in result]

        # Concatenate by newline.
        concatenated = '\n'.join(stringified)

        # Percent encode.
        return parse.quote(concatenated, '')

Converts the credentials to a percent encoded string to be stored for later use .


def is_binary_string(content):
        """
        Return true if string is binary data.
        """

        textchars = (bytearray([7, 8, 9, 10, 12, 13, 27]) +
                     bytearray(range(0x20, 0x100)))
        return bool(content.translate(None, textchars))

def is_binary_string ( content ) : textchars = ( bytearray ( [ 7 , 8 , 9 , 10 , 12 , 13 , 27 ] ) + bytearray ( range ( 0x20 , 0x100 ) ) ) return bool ( content . translate ( None , textchars ) )


def content(self):
        """
        The whole response content.
        """

        if not self._content:
            content = self.httplib_response.read()
            if self.is_binary_string(content):
                self._content = content
            else:
                self._content = content.decode('utf-8')
        return self._content

def content ( self ) : if not self . _content : content = self . httplib_response . read ( ) if self . is_binary_string ( content ) : self . _content = content else : self . _content = content . decode ( 'utf-8' ) return self . _content


def create_request_elements(
            cls, request_type, credentials, url, params=None, headers=None,
            body='', method='GET', verifier='', callback=''
    ):
        """
        Creates |oauth1| request elements.
        """

        params = params or {}
        headers = headers or {}

        consumer_key = credentials.consumer_key or ''
        consumer_secret = credentials.consumer_secret or ''
        token = credentials.token or ''
        token_secret = credentials.token_secret or ''

        # separate url base and query parameters
        url, base_params = cls._split_url(url)

        # add extracted params to future params
        params.update(dict(base_params))

        if request_type == cls.USER_AUTHORIZATION_REQUEST_TYPE:
            # no need for signature
            if token:
                params['oauth_token'] = token
            else:
                raise OAuth1Error(
                    'Credentials with valid token are required to create '
                    'User Authorization URL!')
        else:
            # signature needed
            if request_type == cls.REQUEST_TOKEN_REQUEST_TYPE:
                # Request Token URL
                if consumer_key and consumer_secret and callback:
                    params['oauth_consumer_key'] = consumer_key
                    params['oauth_callback'] = callback
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, consumer_secret '
                        'and callback are required to create Request Token '
                        'URL!')

            elif request_type == cls.ACCESS_TOKEN_REQUEST_TYPE:
                # Access Token URL
                if consumer_key and consumer_secret and token and verifier:
                    params['oauth_token'] = token
                    params['oauth_consumer_key'] = consumer_key
                    params['oauth_verifier'] = verifier
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, '
                        'consumer_secret, token and argument verifier'
                        ' are required to create Access Token URL!')

            elif request_type == cls.PROTECTED_RESOURCE_REQUEST_TYPE:
                # Protected Resources URL
                if consumer_key and consumer_secret and token and token_secret:
                    params['oauth_token'] = token
                    params['oauth_consumer_key'] = consumer_key
                else:
                    raise OAuth1Error(
                        'Credentials with valid consumer_key, ' +
                        'consumer_secret, token and token_secret are required '
                        'to create Protected Resources URL!')

            # Sign request.
            # http://oauth.net/core/1.0a/#anchor13

            # Prepare parameters for signature base string
            # http://oauth.net/core/1.0a/#rfc.section.9.1
            params['oauth_signature_method'] = cls._signature_generator.method
            params['oauth_timestamp'] = str(int(time.time()))
            params['oauth_nonce'] = cls.csrf_generator(str(uuid.uuid4()))
            params['oauth_version'] = '1.0'

            # add signature to params
            params['oauth_signature'] = cls._signature_generator.create_signature(  # noqa
                method, url, params, consumer_secret, token_secret)

        request_elements = core.RequestElements(
            url, method, params, headers, body)

        return cls._x_request_elements_filter(
            request_type, request_elements, credentials)


def _access_user_info(self):
        """
        Email is available in separate method so second request is needed.
        """
        response = super(Bitbucket, self)._access_user_info()

        response.data.setdefault("email", None)

        email_response = self.access(self.user_email_url)
        if email_response.data:
            for item in email_response.data:
                if item.get("primary", False):
                    response.data.update(email=item.get("email", None))

        return response

def _access_user_info ( self ) : response = super ( Bitbucket , self ) . _access_user_info ( ) response . data . setdefault ( "email" , None ) email_response = self . access ( self . user_email_url ) if email_response . data : for item in email_response . data : if item . get ( "primary" , False ) : response . data . update ( email = item . get ( "email" , None ) ) return response

Email is available in separate method so second request is needed .


def login(self, *login_args, **login_kwargs):
        """
        Decorator for Flask view functions.
        """

        def decorator(f):
            @wraps(f)
            def decorated(*args, **kwargs):
                self.response = make_response()
                adapter = WerkzeugAdapter(request, self.response)
                login_kwargs.setdefault('session', session)
                login_kwargs.setdefault('session_saver', self.session_saver)
                self.result = super(FlaskAuthomatic, self).login(
                    adapter,
                    *login_args,
                    **login_kwargs)
                return f(*args, **kwargs)
            return decorated
        return decorator

def login ( self , * login_args , ** login_kwargs ) : def decorator ( f ) : @ wraps ( f ) def decorated ( * args , ** kwargs ) : self . response = make_response ( ) adapter = WerkzeugAdapter ( request , self . response ) login_kwargs . setdefault ( 'session' , session ) login_kwargs . setdefault ( 'session_saver' , self . session_saver ) self . result = super ( FlaskAuthomatic , self ) . login ( adapter , * login_args , ** login_kwargs ) return f ( * args , ** kwargs ) return decorated return decorator


def login(self):
        """
        Launches the OpenID authentication procedure.
        """

        if self.params.get(self.identifier_param):
            # =================================================================
            # Phase 1 before redirect.
            # =================================================================
            self._log(
                logging.INFO,
                u'Starting OpenID authentication procedure.')

            url = users.create_login_url(
                dest_url=self.url, federated_identity=self.identifier)

            self._log(logging.INFO, u'Redirecting user to {0}.'.format(url))

            self.redirect(url)
        else:
            # =================================================================
            # Phase 2 after redirect.
            # =================================================================

            self._log(
                logging.INFO,
                u'Continuing OpenID authentication procedure after redirect.')

            user = users.get_current_user()

            if user:
                self._log(logging.INFO, u'Authentication successful.')
                self._log(logging.INFO, u'Creating user.')
                self.user = core.User(self,
                                      id=user.federated_identity(),
                                      email=user.email(),
                                      gae_user=user)

                # =============================================================
                # We're done
                # =============================================================
            else:
                raise FailureError(
                    'Unable to authenticate identifier "{0}"!'.format(
                        self.identifier))

Launches the OpenID authentication procedure .


def _session_key(self, key):
        """
        Generates session key string.

        :param str key:
            e.g. ``"authomatic:facebook:key"``

        """

        return '{0}:{1}:{2}'.format(self.settings.prefix, self.name, key)

def _session_key ( self , key ) : return '{0}:{1}:{2}' . format ( self . settings . prefix , self . name , key )


def _session_set(self, key, value):
        """
        Saves a value to session.
        """

        self.session[self._session_key(key)] = value

def _session_set ( self , key , value ) : self . session [ self . _session_key ( key ) ] = value


def csrf_generator(secret):
        """
        Generates CSRF token.

        Inspired by this article:
        http://blog.ptsecurity.com/2012/10/random-number-security-in-python.html

        :returns:
            :class:`str` Random unguessable string.

        """

        # Create hash from random string plus salt.
        hashed = hashlib.md5(uuid.uuid4().bytes + six.b(secret)).hexdigest()

        # Each time return random portion of the hash.
        span = 5
        shift = random.randint(0, span)
        return hashed[shift:shift - span - 1]

def csrf_generator ( secret ) : hashed = hashlib . md5 ( uuid . uuid4 ( ) . bytes + six . b ( secret ) ) . hexdigest ( ) span = 5 shift = random . randint ( 0 , span ) return hashed [ shift : shift - span - 1 ]


def _log(cls, level, msg, **kwargs):
        """
        Logs a message with pre-formatted prefix.

        :param int level:
            Logging level as specified in the
            `login module <http://docs.python.org/2/library/logging.html>`_ of
            Python standard library.

        :param str msg:
            The actual message.

        """

        logger = getattr(cls, '_logger', None) or authomatic.core._logger
        logger.log(
            level, ': '.join(
                ('authomatic', cls.__name__, msg)), **kwargs)

def
_log
(
cls
,
level
,
msg
,
**
kwargs
)
:
logger
=
getattr
(
cls
,
'_logger'
,
)
or
authomatic
.
core
.
_logger
logger
.
log
(
level
,
': '
.
join
(
(
'authomatic'
,
cls
.
__name__
,
msg
)
)
,
**
kwargs
)

Logs a message with pre - formatted prefix .


def _http_status_in_category(status, category):
        """
        Checks whether a HTTP status code is in the category denoted by the
        hundreds digit.
        """

        assert category < 10, 'HTTP status category must be a one-digit int!'
        cat = category * 100
        return status >= cat and status < cat + 100

Checks whether a HTTP status code is in the category denoted by the hundreds digit .


def _split_url(url):
        """
        Splits given url to url base and params converted to list of tuples.
        """

        split = parse.urlsplit(url)
        base = parse.urlunsplit((split.scheme, split.netloc, split.path, 0, 0))
        params = parse.parse_qsl(split.query, True)

        return base, params

def _split_url ( url ) : split = parse . urlsplit ( url ) base = parse . urlunsplit ( ( split . scheme , split . netloc , split . path , 0 , 0 ) ) params = parse . parse_qsl ( split . query , True ) return base , params

Splits given url to url base and params converted to list of tuples .


def cross_origin(app, *args, **kwargs):
    """
    This function is the decorator which is used to wrap a Sanic route with.
    In the simplest case, simply use the default parameters to allow all
    origins in what is the most permissive configuration. If this method
    modifies state or performs authentication which may be brute-forced, you
    should add some degree of protection, such as Cross Site Forgery
    Request protection.

    :param origins:
        The origin, or list of origins to allow requests from.
        The origin(s) may be regular expressions, case-sensitive strings,
        or else an asterisk

        Default : '*'
    :type origins: list, string or regex

    :param methods:
        The method or list of methods which the allowed origins are allowed to
        access for non-simple requests.

        Default : [GET, HEAD, POST, OPTIONS, PUT, PATCH, DELETE]
    :type methods: list or string

    :param expose_headers:
        The header or list which are safe to expose to the API of a CORS API
        specification.

        Default : None
    :type expose_headers: list or string

    :param allow_headers:
        The header or list of header field names which can be used when this
        resource is accessed by allowed origins. The header(s) may be regular
        expressions, case-sensitive strings, or else an asterisk.

        Default : '*', allow all headers
    :type allow_headers: list, string or regex

    :param supports_credentials:
        Allows users to make authenticated requests. If true, injects the
        `Access-Control-Allow-Credentials` header in responses. This allows
        cookies and credentials to be submitted across domains.

        :note: This option cannot be used in conjuction with a '*' origin

        Default : False
    :type supports_credentials: bool

    :param max_age:
        The maximum time for which this CORS request maybe cached. This value
        is set as the `Access-Control-Max-Age` header.

        Default : None
    :type max_age: timedelta, integer, string or None

    :param send_wildcard: If True, and the origins parameter is `*`, a wildcard
        `Access-Control-Allow-Origin` header is sent, rather than the
        request's `Origin` header.

        Default : False
    :type send_wildcard: bool

    :param vary_header:
        If True, the header Vary: Origin will be returned as per the W3
        implementation guidelines.

        Setting this header when the `Access-Control-Allow-Origin` is
        dynamically generated (e.g. when there is more than one allowed
        origin, and an Origin than '*' is returned) informs CDNs and other
        caches that the CORS headers are dynamic, and cannot be cached.

        If False, the Vary header will never be injected or altered.

        Default : True
    :type vary_header: bool

    :param automatic_options:
        Only applies to the `cross_origin` decorator. If True, Sanic-CORS will
        override Sanic's default OPTIONS handling to return CORS headers for
        OPTIONS requests.

        Default : True
    :type automatic_options: bool

    """
    _options = kwargs
    _real_decorator = cors.decorate(app, *args, run_middleware=False, with_context=False, **kwargs)

    def wrapped_decorator(f):
        spf = SanicPluginsFramework(app)  # get the singleton from the app
        try:
            plugin = spf.register_plugin(cors, skip_reg=True)
        except ValueError as e:
            # this is normal, if this plugin has been registered previously
            assert e.args and len(e.args) > 1
            plugin = e.args[1]
        context = cors.get_context_from_spf(spf)
        log = context.log
        log(logging.DEBUG, "Enabled {:s} for cross_origin using options: {}".format(str(f), str(_options)))
        return _real_decorator(f)

    return wrapped_decorator

This function is the decorator which is used to wrap a Sanic route with . In the simplest case simply use the default parameters to allow all origins in what is the most permissive configuration . If this method modifies state or performs authentication which may be brute - forced you should add some degree of protection such as Cross Site Forgery Request protection .


def set_cors_headers(req, resp, context, options):
    """
    Performs the actual evaluation of Sanic-CORS options and actually
    modifies the response object.

    This function is used both in the decorator and the after_request
    callback
    :param sanic.request.Request req:

    """
    try:
        request_context = context.request[id(req)]
    except AttributeError:
        LOG.debug("Cannot find the request context. Is request already finished?")
        return resp
    # If CORS has already been evaluated via the decorator, skip
    evaluated = request_context.get(SANIC_CORS_EVALUATED, False)
    if evaluated:
        LOG.debug('CORS have been already evaluated, skipping')
        return resp

    # `resp` can be None in the case of using Websockets
    # however this case should have been handled in the `extension` and `decorator` methods
    # before getting here. This is a final failsafe check to prevent crashing
    if resp is None:
        return None

    if resp.headers is None:
        resp.headers = CIMultiDict()

    headers_to_set = get_cors_headers(options, req.headers, req.method)

    LOG.debug('Settings CORS headers: %s', str(headers_to_set))

    # dict .extend() does not work on CIDict so
    # iterate over them and add them individually.
    try:
        resp.headers.extend(headers_to_set)
    except Exception as e1:
        for k, v in headers_to_set.items():
            try:
                resp.headers.add(k, v)
            except Exception as e2:
                resp.headers[k] = v
        return resp

Performs the actual evaluation of Sanic - CORS options and actually modifies the response object .


def get_app_kwarg_dict(appInstance):
    """Returns the dictionary of CORS specific app configurations."""
    # In order to support blueprints which do not have a config attribute
    app_config = getattr(appInstance, 'config', {})
    return dict(
        (k.lower().replace('cors_', ''), app_config.get(k))
        for k in CONFIG_OPTIONS
        if app_config.get(k) is not None
    )

def get_app_kwarg_dict ( appInstance ) : app_config = getattr ( appInstance , 'config' , { } ) return dict ( ( k . lower ( ) . replace ( 'cors_' , '' ) , app_config . get ( k ) ) for k in CONFIG_OPTIONS if app_config . get ( k ) is not None )

Returns the dictionary of CORS specific app configurations .


def flexible_str(obj):
    """
    A more flexible str function which intelligently handles stringifying
    strings, lists and other iterables. The results are lexographically sorted
    to ensure generated responses are consistent when iterables such as Set
    are used.
    """
    if obj is None:
        return None
    elif(not isinstance(obj, str)
            and isinstance(obj, collections.abc.Iterable)):
        return ', '.join(str(item) for item in sorted(obj))
    else:
        return str(obj)

def
flexible_str
(
obj
)
:
if
obj
is
:
return
elif
(
not
isinstance
(
obj
,
str
)
and
isinstance
(
obj
,
collections
.
abc
.
Iterable
)
)
:
return
', '
.
join
(
str
(
item
)
for
item
in
sorted
(
obj
)
)
else
:
return
str
(
obj
)

A more flexible str function which intelligently handles stringifying strings lists and other iterables . The results are lexographically sorted to ensure generated responses are consistent when iterables such as Set are used .


def ensure_iterable(inst):
    """
    Wraps scalars or string types as a list, or returns the iterable instance.
    """
    if isinstance(inst, str):
        return [inst]
    elif not isinstance(inst, collections.abc.Iterable):
        return [inst]
    else:
        return inst

def ensure_iterable ( inst ) : if isinstance ( inst , str ) : return [ inst ] elif not isinstance ( inst , collections . abc . Iterable ) : return [ inst ] else : return inst

Wraps scalars or string types as a list or returns the iterable instance .


def isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0):
    """
    Python 3.4 does not have math.isclose, so we need to steal it and add it here.
    """
    try:
        return math.isclose(a, b, rel_tol=rel_tol, abs_tol=abs_tol)
    except AttributeError:
        # Running on older version of python, fall back to hand-rolled implementation
        if (rel_tol < 0.0) or (abs_tol < 0.0):
            raise ValueError("Tolerances must be non-negative, but are rel_tol: {} and abs_tol: {}".format(rel_tol, abs_tol))
        if math.isnan(a) or math.isnan(b):
            return False  # NaNs are never close to anything, even other NaNs
        if (a == b):
            return True
        if math.isinf(a) or math.isinf(b):
            return False  # Infinity is only close to itself, and we already handled that case
        diff = abs(a - b)
        return (diff <= rel_tol * abs(b)) or (diff <= rel_tol * abs(a)) or (diff <= abs_tol)

Python 3 . 4 does not have math . isclose so we need to steal it and add it here .


def deprecated(func):
    """
    Deprecator decorator.
    """
    @functools.wraps(func)
    def new_func(*args, **kwargs):
        warnings.warn("Call to deprecated function {}.".format(func.__name__), category=DeprecationWarning, stacklevel=2)
        return func(*args, **kwargs)

    return new_func

def
deprecated
(
func
)
:
@
functools
.
wraps
(
func
)
def
new_func
(
*
args
,
**
kwargs
)
:
warnings
.
warn
(
"Call to deprecated function {}."
.
format
(
func
.
__name__
)
,
category
=
DeprecationWarning
,
stacklevel
=
2
)
return
func
(
*
args
,
**
kwargs
)
return
new_func


def deserialize(bstr):
    """
    Attempts to deserialize a bytestring into an audiosegment.

    :param bstr: The bytestring serialized via an audiosegment's serialize() method.
    :returns: An AudioSegment object deserialized from `bstr`.
    """
    d = pickle.loads(bstr)
    seg = pickle.loads(d['seg'])
    return AudioSegment(seg, d['name'])

def deserialize ( bstr ) : d = pickle . loads ( bstr ) seg = pickle . loads ( d [ 'seg' ] ) return AudioSegment ( seg , d [ 'name' ] )

Attempts to deserialize a bytestring into an audiosegment .


def from_file(path):
    """
    Returns an AudioSegment object from the given file based on its file extension.
    If the extension is wrong, this will throw some sort of error.

    :param path: The path to the file, including the file extension.
    :returns: An AudioSegment instance from the file.
    """
    _name, ext = os.path.splitext(path)
    ext = ext.lower()[1:]
    seg = pydub.AudioSegment.from_file(path, ext)
    return AudioSegment(seg, path)

def from_file ( path ) : _name , ext = os . path . splitext ( path ) ext = ext . lower ( ) [ 1 : ] seg = pydub . AudioSegment . from_file ( path , ext ) return AudioSegment ( seg , path )

Returns an AudioSegment object from the given file based on its file extension . If the extension is wrong this will throw some sort of error .


def from_numpy_array(nparr, framerate):
    """
    Returns an AudioSegment created from the given numpy array.

    The numpy array must have shape = (num_samples, num_channels).

    :param nparr: The numpy array to create an AudioSegment from.
    :returns: An AudioSegment created from the given array.
    """
    # interleave the audio across all channels and collapse
    if nparr.dtype.itemsize not in (1, 2, 4):
        raise ValueError("Numpy Array must contain 8, 16, or 32 bit values.")
    if len(nparr.shape) == 1:
        arrays = [nparr]
    elif len(nparr.shape) == 2:
        arrays = [nparr[i,:] for i in range(nparr.shape[0])]
    else:
        raise ValueError("Numpy Array must be one or two dimensional. Shape must be: (num_samples, num_channels).")
    interleaved = np.vstack(arrays).reshape((-1,), order='F')
    dubseg = pydub.AudioSegment(interleaved.tobytes(),
                                frame_rate=framerate,
                                sample_width=interleaved.dtype.itemsize,
                                channels=len(interleaved.shape)
                               )
    return AudioSegment(dubseg, "")

Returns an AudioSegment created from the given numpy array .


def _execute_sox_cmd(self, cmd, console_output=False):
        """
        Executes a Sox command in a platform-independent manner.

        `cmd` must be a format string that includes {inputfile} and {outputfile}.
        """
        on_windows = platform.system().lower() == "windows"

        # On Windows, a temporary file cannot be shared outside the process that creates it
        # so we need to create a "permanent" file that we will use and delete afterwards
        def _get_random_tmp_file():
            if on_windows:
                rand_string = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8))
                tmp = self.name + "_" + rand_string
                WinTempFile = collections.namedtuple("WinTempFile", "name")
                tmp = WinTempFile(tmp)
            else:
                tmp = tempfile.NamedTemporaryFile()
            return tmp

        # Get a temp file to put our data and a temp file to store the result
        tmp = _get_random_tmp_file()
        othertmp = _get_random_tmp_file()

        # Store our data in the temp file
        self.export(tmp.name, format="WAV")

        # Write the command to sox
        stdout = stderr = subprocess.PIPE if console_output else subprocess.DEVNULL
        command = cmd.format(inputfile=tmp.name, outputfile=othertmp.name)
        res = subprocess.call(command.split(' '), stdout=stdout, stderr=stderr)
        assert res == 0, "Sox did not work as intended, or perhaps you don't have Sox installed?"

        # Create a new AudioSegment from the other temp file (where Sox put the result)
        other = AudioSegment(pydub.AudioSegment.from_wav(othertmp.name), self.name)

        # Clean up the temp files
        if on_windows:
            os.remove(tmp.name)
            os.remove(othertmp.name)
        else:
            tmp.close()
            othertmp.close()

        return other

Executes a Sox command in a platform - independent manner .


def filter_silence(self, duration_s=1, threshold_percentage=1, console_output=False):
        """
        Returns a copy of this AudioSegment, but whose silence has been removed.

        .. note:: This method requires that you have the program 'sox' installed.

        .. warning:: This method uses the program 'sox' to perform the task. While this is very fast for a single
                     function call, the IO may add up for large numbers of AudioSegment objects.

        :param duration_s: The number of seconds of "silence" that must be present in a row to
                           be stripped.
        :param threshold_percentage: Silence is defined as any samples whose absolute value is below
                                     `threshold_percentage * max(abs(samples in this segment))`.
        :param console_output: If True, will pipe all sox output to the console.
        :returns: A copy of this AudioSegment, but whose silence has been removed.
        """
        command = "sox {inputfile} -t wav {outputfile} silence -l 1 0.1 "\
            + str(threshold_percentage) + "% -1 " + str(float(duration_s)) + " " + str(threshold_percentage) + "%"
        try:
            result = self._execute_sox_cmd(command)
        except pydub.exceptions.CouldntDecodeError:
            warnings.warn("After silence filtering, the resultant WAV file is corrupted, and so its data cannot be retrieved. Perhaps try a smaller threshold value.", stacklevel=2)
            # Return a copy of us
            result = AudioSegment(self.seg, self.name)
        return result

Returns a copy of this AudioSegment but whose silence has been removed .