我们从Python开源项目中，提取了以下31个代码示例，用于说明如何使用keras.backend.dropout()。
def get_constants(self, inputs, training=None): constants = self.recurrent_layer.get_constants( inputs=inputs, training=training ) if 0 < self.dense_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.recurrent_layer.units)) def dropped_inputs(): return K.dropout(ones, self.dense_dropout) out_dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training)] constants.append(out_dp_mask) else: constants.append([K.cast_to_floatx(1.)]) return constants
def get_constants(self, inputs, training=None): constants = [] '''if 0 < self.dropout_U < 1: ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)] constants.append(B_U) else: constants.append([K.cast_to_floatx(1.) for _ in range(3)]) if 0 < self.dropout_W < 1: input_shape = K.int_shape(x) input_dim = input_shape[-1] ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)] constants.append(B_W) else:''' constants.append([K.cast_to_floatx(1.) for _ in range(3)]) return constants
def step(self, inputs, states): h_tm1 = states[0] # previous memory #B_U = states[1] # dropout matrices for recurrent units #B_W = states[2] h_tm1a = K.dot(h_tm1, self.Wa) eij = K.dot(K.tanh(h_tm1a + K.dot(inputs[:, :self.h_dim], self.Ua)), self.Va) eijs = K.repeat_elements(eij, self.h_dim, axis=1) #alphaij = K.softmax(eijs) # batchsize * lenh h batchsize * lenh * ndim #ci = K.permute_dimensions(K.permute_dimensions(self.h, [2,0,1]) * alphaij, [1,2,0]) #cisum = K.sum(ci, axis=1) cisum = eijs*inputs[:, :self.h_dim] #print(K.shape(cisum), cisum.shape, ci.shape, self.h.shape, alphaij.shape, x.shape) zr = K.sigmoid(K.dot(inputs[:, self.h_dim:], self.Wzr) + K.dot(h_tm1, self.Uzr) + K.dot(cisum, self.Czr)) zi = zr[:, :self.units] ri = zr[:, self.units: 2 * self.units] si_ = K.tanh(K.dot(inputs[:, self.h_dim:], self.W) + K.dot(ri*h_tm1, self.U) + K.dot(cisum, self.C)) si = (1-zi) * h_tm1 + zi * si_ return si, [si] #h_tm1, [h_tm1]
def preprocess_input(self, inputs, training=None): if self.window_size > 1: inputs = K.temporal_padding(inputs, (self.window_size-1, 0)) inputs = K.expand_dims(inputs, 2) # add a dummy dimension output = K.conv2d(inputs, self.kernel, strides=self.strides, padding='valid', data_format='channels_last') output = K.squeeze(output, 2) # remove the dummy dimension if self.use_bias: output = K.bias_add(output, self.bias, data_format='channels_last') if self.dropout is not None and 0. < self.dropout < 1.: z = output[:, :, :self.units] f = output[:, :, self.units:2 * self.units] o = output[:, :, 2 * self.units:] f = K.in_train_phase(1 - _dropout(1 - f, self.dropout), f, training=training) return K.concatenate([z, f, o], -1) else: return output
def get_config(self): config = {'units': self.units, 'window_size': self.window_size, 'stride': self.strides[0], 'return_sequences': self.return_sequences, 'go_backwards': self.go_backwards, 'stateful': self.stateful, 'unroll': self.unroll, 'use_bias': self.use_bias, 'dropout': self.dropout, 'activation': activations.serialize(self.activation), 'kernel_initializer': initializers.serialize(self.kernel_initializer), 'bias_initializer': initializers.serialize(self.bias_initializer), 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 'activity_regularizer': regularizers.serialize(self.activity_regularizer), 'kernel_constraint': constraints.serialize(self.kernel_constraint), 'bias_constraint': constraints.serialize(self.bias_constraint), 'input_dim': self.input_dim, 'input_length': self.input_length} base_config = super(QRNN, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def step(self, inputs, states): if 0 < self.dropout < 1: h = ternarize_dot(inputs * states[1], self.kernel) else: h = ternarize_dot(inputs, self.kernel) if self.bias is not None: h = K.bias_add(h, self.bias) prev_output = states[0] if 0 < self.recurrent_dropout < 1: prev_output *= states[2] output = h + ternarize_dot(prev_output, self.recurrent_kernel) if self.activation is not None: output = self.activation(output) # Properly set learning phase on output tensor. if 0 < self.dropout + self.recurrent_dropout: output._uses_learning_phase = True return output, [output]
def get_config(self): config = {'units': self.units, 'activation': activations.serialize(self.activation), 'recurrent_activation': activations.serialize(self.recurrent_activation), 'use_bias': self.use_bias, 'kernel_initializer': initializers.serialize(self.kernel_initializer), 'recurrent_initializer': initializers.serialize(self.recurrent_initializer), 'bias_initializer': initializers.serialize(self.bias_initializer), 'unit_forget_bias': self.unit_forget_bias, 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 'recurrent_regularizer': regularizers.serialize(self.recurrent_regularizer), 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 'activity_regularizer': regularizers.serialize(self.activity_regularizer), 'kernel_constraint': constraints.serialize(self.kernel_constraint), 'recurrent_constraint': constraints.serialize(self.recurrent_constraint), 'bias_constraint': constraints.serialize(self.bias_constraint), 'dropout': self.dropout, 'recurrent_dropout': self.recurrent_dropout} base_config = super(MultiplicativeLSTM, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def preprocess_input(self, inputs, training=None): if self.implementation == 0: input_shape = K.int_shape(inputs) input_dim = input_shape[2] timesteps = input_shape[1] x_i = _time_distributed_dense(inputs, self.kernel_i, self.bias_i, self.dropout, input_dim, self.units, timesteps, training=training) x_f = _time_distributed_dense(inputs, self.kernel_f, self.bias_f, self.dropout, input_dim, self.units, timesteps, training=training) x_c = _time_distributed_dense(inputs, self.kernel_c, self.bias_c, self.dropout, input_dim, self.units, timesteps, training=training) x_o = _time_distributed_dense(inputs, self.kernel_o, self.bias_o, self.dropout, input_dim, self.units, timesteps, training=training) return K.concatenate([x_i, x_f, x_c, x_o], axis=2) else: return inputs
def get_config(self): config = {'units': self.units, 'activation': activations.serialize(self.activation), 'recurrent_activation': activations.serialize(self.recurrent_activation), 'use_bias': self.use_bias, 'kernel_initializer': initializers.serialize(self.kernel_initializer), 'recurrent_initializer': initializers.serialize(self.recurrent_initializer), 'bias_initializer': initializers.serialize(self.bias_initializer), 'unit_forget_bias': self.unit_forget_bias, 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 'recurrent_regularizer': regularizers.serialize(self.recurrent_regularizer), 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 'activity_regularizer': regularizers.serialize(self.activity_regularizer), 'kernel_constraint': constraints.serialize(self.kernel_constraint), 'recurrent_constraint': constraints.serialize(self.recurrent_constraint), 'bias_constraint': constraints.serialize(self.bias_constraint), 'dropout': self.dropout, 'recurrent_dropout': self.recurrent_dropout} base_config = super(PhasedLSTM, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def dropout(self): return self.recurrent_layer.dropout + self.dense_dropout
def time_distributed_dense(x, w, b=None, dropout=None, input_dim=None, units=None, timesteps=None): """Apply `y . w + b` for every temporal slice y of x. # Arguments x: input tensor. w: weight matrix. b: optional bias vector. dropout: wether to apply dropout (same dropout mask for every temporal slice of the input). input_dim: integer; optional dimensionality of the input. units: integer; optional dimensionality of the output. timesteps: integer; optional number of timesteps. # Returns Output tensor. """ if not input_dim: input_dim = K.shape(x)[2] if not timesteps: timesteps = K.shape(x)[1] if not units: units = K.shape(w)[1] if dropout is not None and 0. < dropout < 1.: # apply the same dropout pattern at every timestep ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim))) dropout_matrix = K.dropout(ones, dropout) expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps) x = K.in_train_phase(x * expanded_dropout_matrix, x) # collapse time dimension and batch dimension together x = K.reshape(x, (-1, input_dim)) x = K.dot(x, w) if b: x += b # reshape to 3D tensor if K.backend() == 'tensorflow': x = K.reshape(x, K.stack([-1, timesteps, units])) x.set_shape([None, None, units]) else: x = K.reshape(x, (-1, timesteps, units)) return x
def call(self, inputs, training=None): if 0. < self.rate < 1.: noise_shape = self._get_noise_shape(inputs) def dropped_inputs(): return K.dropout(inputs, self.rate, noise_shape, seed=self.seed) * (1 - self.rate) return K.in_train_phase(dropped_inputs, inputs, training=training) return inputs
def _dropout(x, level, noise_shape=None, seed=None): x = K.dropout(x, level, noise_shape, seed) x *= (1. - level) # compensate for the scaling by the dropout return x
def __init__(self, units, window_size=2, stride=1, return_sequences=False, go_backwards=False, stateful=False, unroll=False, activation='tanh', kernel_initializer='uniform', bias_initializer='zero', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, dropout=0, use_bias=True, input_dim=None, input_length=None, **kwargs): self.return_sequences = return_sequences self.go_backwards = go_backwards self.stateful = stateful self.unroll = unroll self.units = units self.window_size = window_size self.strides = (stride, 1) self.use_bias = use_bias self.activation = activations.get(activation) self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = dropout self.supports_masking = True self.input_spec = [InputSpec(ndim=3)] self.input_dim = input_dim self.input_length = input_length if self.input_dim: kwargs['input_shape'] = (self.input_length, self.input_dim) super(QRNN, self).__init__(**kwargs)
def get_constants(self, inputs, training=None): constants = [] if 0 < self.dropout < 1: input_shape = K.int_shape(inputs) input_dim = input_shape[-1] ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) def dropped_inputs(): return K.dropout(ones, self.dropout) dp_mask = K.in_train_phase(dropped_inputs, ones, training=training) constants.append(dp_mask) else: constants.append(K.cast_to_floatx(1.)) if 0 < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = K.in_train_phase(dropped_inputs, ones, training=training) constants.append(rec_dp_mask) else: constants.append(K.cast_to_floatx(1.)) return constants # Aliases
def call(self, inputs, mask=None): if 0. < self.rate < 1.: noise_shape = self._get_noise_shape(inputs) inputs = K.in_train_phase( K.dropout(inputs, self.rate, noise_shape) * (1. - self.rate), inputs)# multiplied by (1. - self.rate) for compensation return inputs
def get_constants(self, inputs, training=None): constants = [] if self.implementation != 0 and 0 < self.dropout < 1: input_shape = K.int_shape(inputs) input_dim = input_shape[-1] ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) def dropped_inputs(): return K.dropout(ones, self.dropout) dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(5)] constants.append(dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(5)]) if 0 < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(5)] constants.append(rec_dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(5)]) return constants
def step(self, inputs, states): h_tm1 = states[0] c_tm1 = states[1] dp_mask = states[2] rec_dp_mask = states[3] if self.implementation == 2: z = K.dot(inputs * dp_mask[0], self.kernel) z += z * K.dot(h_tm1 * rec_dp_mask[0], self.recurrent_kernel) # applies m instead of h_tm1 to z if self.use_bias: z = K.bias_add(z, self.bias) z0 = z[:, :self.units] z1 = z[:, self.units: 2 * self.units] z2 = z[:, 2 * self.units: 3 * self.units] z3 = z[:, 3 * self.units: 4 * self.units] z4 = z[:, 4 * self.units:] # just elementwise multiplication, no activation functions i = self.recurrent_activation(z0) f = self.recurrent_activation(z1) c = f * c_tm1 + i * self.activation(z2) o = self.recurrent_activation(z3) else: if self.implementation == 1: x_i = K.dot(inputs * dp_mask[0], self.kernel_i) + self.bias_i x_f = K.dot(inputs * dp_mask[1], self.kernel_f) + self.bias_f x_c = K.dot(inputs * dp_mask[2], self.kernel_c) + self.bias_c x_o = K.dot(inputs * dp_mask[3], self.kernel_o) + self.bias_o x_m = K.dot(inputs * dp_mask[4], self.kernel_m) + self.bias_m else: raise ValueError('Unknown `implementation` mode.') m = x_m * K.dot(h_tm1 * rec_dp_mask[4], self.recurrent_kernel_m) # elementwise multiplication m i = self.recurrent_activation(x_i + K.dot(m * rec_dp_mask[0], self.recurrent_kernel_i)) f = self.recurrent_activation(x_f + K.dot(m * rec_dp_mask[1], self.recurrent_kernel_f)) c = f * c_tm1 + i * self.activation(x_c + K.dot(m * rec_dp_mask[2], self.recurrent_kernel_c)) o = self.recurrent_activation(x_o + K.dot(m * rec_dp_mask[3], self.recurrent_kernel_o)) h = o * self.activation(c) if 0 < self.dropout + self.recurrent_dropout: h._uses_learning_phase = True return h, [h, c]
def call(self, x, mask=None): if 0. < self.p < 1.: noise_shape = self._get_noise_shape(x) x = K.dropout(x, self.p, noise_shape) return x
def zoneout(self, v, prev_v, pr=0.): diff = v - prev_v diff = K.in_train_phase(K.dropout(diff, pr, noise_shape=(self.output_dim,)), diff) # In testing, always return v * (1-pr) + prev_v * pr # In training when K.dropout returns 0, return prev_v # when K.dropout returns diff/(1-pr), return v return prev_v + diff * (1-pr)
def call(self, x, mask=None): if 0. < self.W_dropout < 1.: def dropped_inputs(): return K.dropout(x, self.W_dropout) x_dp = K.in_train_phase(dropped_inputs, x) else: x_dp = x uit = dot_product(x_dp, self.W) if self.bias: uit += self.b uit = K.tanh(uit) if 0. < self.u_dropout < 1.: def dropped_inputs(): return K.dropout(uit, self.u_dropout) u_dp = K.in_train_phase(dropped_inputs, uit) else: u_dp = uit ait = dot_product(u_dp, self.u) a = K.exp(ait) a = K.expand_dims(a) a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) weighted_input = x * a output = K.sum(weighted_input, axis=1) return output
def call(self, inputs, training=None): if 0. < self.rate < 1.: symbolic_shape = K.shape(inputs) noise_shape = [shape if shape > 0 else symbolic_shape[axis] for axis, shape in enumerate(self.noise_shape)] noise_shape = tuple(noise_shape) def dropped_inputs(): return K.dropout(inputs, self.rate, noise_shape, seed=self.seed) return K.in_train_phase(dropped_inputs, inputs, training=training) return inputs
def get_constants(self, inputs, training=None): constants = [] if self.implementation == 0 and 0 < self.dropout < 1: input_shape = K.int_shape(inputs) input_dim = input_shape[-1] ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, int(input_dim))) def dropped_inputs(): return K.dropout(ones, self.dropout) dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(4)] constants.append(dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) if 0 < self.recurrent_dropout < 1: ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1))) ones = K.tile(ones, (1, self.units)) def dropped_inputs(): return K.dropout(ones, self.recurrent_dropout) rec_dp_mask = [K.in_train_phase(dropped_inputs, ones, training=training) for _ in range(4)] constants.append(rec_dp_mask) else: constants.append([K.cast_to_floatx(1.) for _ in range(4)]) return constants
def step(self, inputs, states): h_tm1 = states[0] # previous memory #B_U = states[1] # dropout matrices for recurrent units #B_W = states[2] h_tm1a = K.dot(h_tm1, self.Wa) eij = K.dot(K.tanh(K.repeat(h_tm1a, K.shape(self.h)[1]) + self.ha), self.Va) eijs = K.squeeze(eij, -1) alphaij = K.softmax(eijs) # batchsize * lenh h batchsize * lenh * ndim ci = K.permute_dimensions(K.permute_dimensions(self.h, [2,0,1]) * alphaij, [1,2,0]) cisum = K.sum(ci, axis=1) #print(K.shape(cisum), cisum.shape, ci.shape, self.h.shape, alphaij.shape, x.shape) zr = K.sigmoid(K.dot(inputs, self.Wzr) + K.dot(h_tm1, self.Uzr) + K.dot(cisum, self.Czr)) zi = zr[:, :self.units] ri = zr[:, self.units: 2 * self.units] si_ = K.tanh(K.dot(inputs, self.W) + K.dot(ri*h_tm1, self.U) + K.dot(cisum, self.C)) si = (1-zi) * h_tm1 + zi * si_ return si, [si] #h_tm1, [h_tm1] '''if self.consume_less == 'gpu': matrix_x = K.dot(x * B_W[0], self.W) + self.b matrix_inner = K.dot(h_tm1 * B_U[0], self.U[:, :2 * self.units]) x_z = matrix_x[:, :self.units] x_r = matrix_x[:, self.units: 2 * self.units] inner_z = matrix_inner[:, :self.units] inner_r = matrix_inner[:, self.units: 2 * self.units] z = self.inner_activation(x_z + inner_z) r = self.inner_activation(x_r + inner_r) x_h = matrix_x[:, 2 * self.units:] inner_h = K.dot(r * h_tm1 * B_U[0], self.U[:, 2 * self.units:]) hh = self.activation(x_h + inner_h) else: if self.consume_less == 'cpu': x_z = x[:, :self.units] x_r = x[:, self.units: 2 * self.units] x_h = x[:, 2 * self.units:] elif self.consume_less == 'mem': x_z = K.dot(x * B_W[0], self.W_z) + self.b_z x_r = K.dot(x * B_W[1], self.W_r) + self.b_r x_h = K.dot(x * B_W[2], self.W_h) + self.b_h else: raise ValueError('Unknown `consume_less` mode.') z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z)) r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r)) hh = self.activation(x_h + K.dot(r * h_tm1 * B_U[2], self.U_h)) h = z * h_tm1 + (1 - z) * hh return h, [h]'''
def step(self, inputs, states): h_tm1 = states[0] # previous memory #B_U = states[1] # dropout matrices for recurrent units #B_W = states[2] h_tm1a = K.dot(h_tm1, self.Wa) eij = K.dot(K.tanh(h_tm1a + self.ha), self.Va) eijs = K.repeat_elements(eij, self.h_dim, axis=1) #alphaij = K.softmax(eijs) # batchsize * lenh h batchsize * lenh * ndim #ci = K.permute_dimensions(K.permute_dimensions(self.h, [2,0,1]) * alphaij, [1,2,0]) #cisum = K.sum(ci, axis=1) cisum = eijs*self.h #print(K.shape(cisum), cisum.shape, ci.shape, self.h.shape, alphaij.shape, x.shape) zr = K.sigmoid(K.dot(inputs, self.Wzr) + K.dot(h_tm1, self.Uzr) + K.dot(cisum, self.Czr)) zi = zr[:, :self.units] ri = zr[:, self.units: 2 * self.units] si_ = K.tanh(K.dot(inputs, self.W) + K.dot(ri*h_tm1, self.U) + K.dot(cisum, self.C)) si = (1-zi) * h_tm1 + zi * si_ return si, [si] #h_tm1, [h_tm1] '''if self.consume_less == 'gpu': matrix_x = K.dot(x * B_W[0], self.W) + self.b matrix_inner = K.dot(h_tm1 * B_U[0], self.U[:, :2 * self.units]) x_z = matrix_x[:, :self.units] x_r = matrix_x[:, self.units: 2 * self.units] inner_z = matrix_inner[:, :self.units] inner_r = matrix_inner[:, self.units: 2 * self.units] z = self.inner_activation(x_z + inner_z) r = self.inner_activation(x_r + inner_r) x_h = matrix_x[:, 2 * self.units:] inner_h = K.dot(r * h_tm1 * B_U[0], self.U[:, 2 * self.units:]) hh = self.activation(x_h + inner_h) else: if self.consume_less == 'cpu': x_z = x[:, :self.units] x_r = x[:, self.units: 2 * self.units] x_h = x[:, 2 * self.units:] elif self.consume_less == 'mem': x_z = K.dot(x * B_W[0], self.W_z) + self.b_z x_r = K.dot(x * B_W[1], self.W_r) + self.b_r x_h = K.dot(x * B_W[2], self.W_h) + self.b_h else: raise ValueError('Unknown `consume_less` mode.') z = self.inner_activation(x_z + K.dot(h_tm1 * B_U[0], self.U_z)) r = self.inner_activation(x_r + K.dot(h_tm1 * B_U[1], self.U_r)) hh = self.activation(x_h + K.dot(r * h_tm1 * B_U[2], self.U_h)) h = z * h_tm1 + (1 - z) * hh return h, [h]'''
def call(self, inputs, mask=None, initial_state=None, training=None): # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): initial_states = inputs[1:] inputs = inputs[0] elif initial_state is not None: pass elif self.stateful: initial_states = self.states else: initial_states = self.get_initial_states(inputs) if len(initial_states) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_states)) + ' initial states.') input_shape = K.int_shape(inputs) if self.unroll and input_shape[1] is None: raise ValueError('Cannot unroll a RNN if the ' 'time dimension is undefined. \n' '- If using a Sequential model, ' 'specify the time dimension by passing ' 'an `input_shape` or `batch_input_shape` ' 'argument to your first layer. If your ' 'first layer is an Embedding, you can ' 'also use the `input_length` argument.\n' '- If using the functional API, specify ' 'the time dimension by passing a `shape` ' 'or `batch_shape` argument to your Input layer.') constants = self.get_constants(inputs, training=None) preprocessed_input = self.preprocess_input(inputs, training=None) last_output, outputs, states = K.rnn(self.step, preprocessed_input, initial_states, go_backwards=self.go_backwards, mask=mask, constants=constants, unroll=self.unroll, input_length=input_shape[1]) if self.stateful: updates = [] for i in range(len(states)): updates.append((self.states[i], states[i])) self.add_update(updates, inputs) # Properly set learning phase if 0 < self.dropout < 1: last_output._uses_learning_phase = True outputs._uses_learning_phase = True if self.return_sequences: return outputs else: return last_output
def __init__(self, units, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., implementation=1, **kwargs): super(MultiplicativeLSTM, self).__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_spec = [InputSpec(shape=(None, self.units)), InputSpec(shape=(None, self.units))] self.state_size = (self.units, self.units) self.implementation = implementation
def _time_distributed_dense(x, w, b=None, dropout=None, input_dim=None, output_dim=None, timesteps=None, training=None): """Apply `y . w + b` for every temporal slice y of x. # Arguments x: input tensor. w: weight matrix. b: optional bias vector. dropout: wether to apply dropout (same dropout mask for every temporal slice of the input). input_dim: integer; optional dimensionality of the input. output_dim: integer; optional dimensionality of the output. timesteps: integer; optional number of timesteps. training: training phase tensor or boolean. # Returns Output tensor. """ if not input_dim: input_dim = K.shape(x)[2] if not timesteps: timesteps = K.shape(x)[1] if not output_dim: output_dim = K.shape(w)[1] if dropout is not None and 0. < dropout < 1.: # apply the same dropout pattern at every timestep ones = K.ones_like(K.reshape(x[:, 0, :], (-1, input_dim))) dropout_matrix = K.dropout(ones, dropout) expanded_dropout_matrix = K.repeat(dropout_matrix, timesteps) x = K.in_train_phase(x * expanded_dropout_matrix, x, training=training) # collapse time dimension and batch dimension together x = K.reshape(x, (-1, input_dim)) x = K.dot(x, w) if b is not None: x = K.bias_add(x, b) # reshape to 3D tensor if K.backend() == 'tensorflow': x = K.reshape(x, K.stack([-1, timesteps, output_dim])) x.set_shape([None, None, output_dim]) else: x = K.reshape(x, (-1, timesteps, output_dim)) return x
def __init__(self, units, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, timegate_initializer=_timegate_init, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, timegate_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, timegate_constraint='non_neg', dropout=0., recurrent_dropout=0., alpha=0.001, **kwargs): super(PhasedLSTM, self).__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.timegate_initializer = initializers.get(timegate_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.timegate_regularizer = regularizers.get(timegate_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.timegate_constraint = constraints.get(timegate_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.alpha = alpha
