我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.reduce_max()。
def log_variable(variable, gradient=None): r''' We introduce a function for logging a tensor variable's current state. It logs scalar values for the mean, standard deviation, minimum and maximum. Furthermore it logs a histogram of its state and (if given) of an optimization gradient. ''' name = variable.name mean = tf.reduce_mean(variable) tf.summary.scalar(name='%s/mean' % name, tensor=mean) tf.summary.scalar(name='%s/sttdev' % name, tensor=tf.sqrt(tf.reduce_mean(tf.square(variable - mean)))) tf.summary.scalar(name='%s/max' % name, tensor=tf.reduce_max(variable)) tf.summary.scalar(name='%s/min' % name, tensor=tf.reduce_min(variable)) tf.summary.histogram(name=name, values=variable) if gradient is not None: if isinstance(gradient, tf.IndexedSlices): grad_values = gradient.values else: grad_values = gradient if grad_values is not None: tf.summary.histogram(name='%s/gradients' % name, values=grad_values)
def _activation_summary(self, x, layer_name): """Helper to create summaries for activations. Args: x: layer output tensor layer_name: name of the layer Returns: nothing """ with tf.variable_scope('activation_summary') as scope: tf.summary.histogram( 'activation_summary/'+layer_name, x) tf.summary.scalar( 'activation_summary/'+layer_name+'/sparsity', tf.nn.zero_fraction(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/average', tf.reduce_mean(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/max', tf.reduce_max(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/min', tf.reduce_min(x))
def calculate_loss(self, predictions, labels, **unused_params): with tf.name_scope("loss_xent"): epsilon = 10e-6 vocab_size = predictions.get_shape().as_list()[1] float_labels = tf.cast(labels, tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss) neg_labels = 1 - float_labels predictions_pos = predictions*float_labels+10*neg_labels predictions_minpos = tf.reduce_min(predictions_pos,axis=1,keep_dims=True) predictions_neg = predictions*neg_labels-10*float_labels predictions_maxneg = tf.reduce_max(predictions_neg,axis=1,keep_dims=True) mask_1 = tf.cast(tf.greater_equal(predictions_neg, predictions_minpos),dtype=tf.float32) mask_2 = tf.cast(tf.less_equal(predictions_pos, predictions_maxneg),dtype=tf.float32) cross_entropy_loss = cross_entropy_loss*(mask_1+mask_2)*10 + cross_entropy_loss return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss(self, predictions, labels, **unused_params): bound = FLAGS.softmax_bound vocab_size_1 = bound with tf.name_scope("loss_softmax"): epsilon = 10e-8 float_labels = tf.cast(labels, tf.float32) labels_1 = float_labels[:,:vocab_size_1] predictions_1 = predictions[:,:vocab_size_1] cross_entropy_loss = CrossEntropyLoss().calculate_loss(predictions_1,labels_1) lables_2 = float_labels[:,vocab_size_1:] predictions_2 = predictions[:,vocab_size_1:] # l1 normalization (labels are no less than 0) label_rowsum = tf.maximum( tf.reduce_sum(lables_2, 1, keep_dims=True), epsilon) label_append = 1.0-tf.reduce_max(lables_2, 1, keep_dims=True) norm_float_labels = tf.concat((tf.div(lables_2, label_rowsum),label_append),axis=1) predictions_append = 1.0-tf.reduce_sum(predictions_2, 1, keep_dims=True) softmax_outputs = tf.concat((predictions_2,predictions_append),axis=1) softmax_loss = norm_float_labels * tf.log(softmax_outputs + epsilon) + ( 1 - norm_float_labels) * tf.log(1 - softmax_outputs + epsilon) softmax_loss = tf.negative(tf.reduce_sum(softmax_loss, 1)) return tf.reduce_mean(softmax_loss) + cross_entropy_loss
def resolution(self, model_input_raw, num_frames, resolution, method="SELECT"): frame_dim = len(model_input_raw.get_shape()) - 2 feature_dim = len(model_input_raw.get_shape()) - 1 max_frames = model_input_raw.get_shape().as_list()[frame_dim] num_features = model_input_raw.get_shape().as_list()[feature_dim] if resolution > 1: new_max_frames = max_frames / resolution cut_frames = new_max_frames * resolution model_input_raw = model_input_raw[:, :cut_frames, :] model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features]) if method == "MEAN": model_input_raw = tf.reduce_mean(model_input_raw, axis=2) elif method == "MAX": model_input_raw = tf.reduce_max(model_input_raw, axis=2) elif method == "SELECT": model_input_raw = model_input_raw[:,:,resolution-1,:] model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) num_frames = num_frames / resolution else: model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) return model_input, num_frames
def FramePooling(frames, method, **unused_params): """Pools over the frames of a video. Args: frames: A tensor with shape [batch_size, num_frames, feature_size]. method: "average", "max", "attention", or "none". Returns: A tensor with shape [batch_size, feature_size] for average, max, or attention pooling. A tensor with shape [batch_size*num_frames, feature_size] for none pooling. Raises: ValueError: if method is other than "average", "max", "attention", or "none". """ if method == "average": return tf.reduce_mean(frames, 1) elif method == "max": return tf.reduce_max(frames, 1) elif method == "none": feature_size = frames.shape_as_list()[2] return tf.reshape(frames, [-1, feature_size]) else: raise ValueError("Unrecognized pooling method: %s" % method)
def get_image_summary(img, idx=0): """ Make an image summary for 4d tensor image with index idx """ V = tf.slice(img, (0, 0, 0, idx), (1, -1, -1, 1)) V -= tf.reduce_min(V) V /= tf.reduce_max(V) V *= 255 img_w = tf.shape(img)[1] img_h = tf.shape(img)[2] V = tf.reshape(V, tf.stack((img_w, img_h, 1))) V = tf.transpose(V, (2, 0, 1)) V = tf.reshape(V, tf.stack((-1, img_w, img_h, 1))) return V
def inference(self): """main computation graph here: 1. embeddding layer, 2.Bi-LSTM layer, 3.max pooling, 4.FC layer 5.softmax """ #1.get emebedding of words in the sentence self.embedded_words = tf.nn.embedding_lookup(self.Embedding,self.input_x) #shape:[None,sentence_length,embed_size] #2. Bi-lstm layer output_conv=self.conv_layer_with_recurrent_structure() #shape:[None,sentence_length,embed_size*3] #3. max pooling #print("output_conv:",output_conv) #(3, 5, 8, 100) output_pooling=tf.reduce_max(output_conv,axis=1) #shape:[None,embed_size*3] #print("output_pooling:",output_pooling) #(3, 8, 100) #4. logits(use linear layer) with tf.name_scope("dropout"): h_drop=tf.nn.dropout(output_pooling,keep_prob=self.dropout_keep_prob) #[None,num_filters_total] with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. logits = tf.matmul(h_drop, self.W_projection) + self.b_projection # [batch_size,num_classes] return logits
def inference(self): """main computation graph here: 1. embeddding layer, 2.Bi-LSTM layer, 3.max pooling, 4.FC layer 5.softmax """ #1.get emebedding of words in the sentence self.embedded_words = tf.nn.embedding_lookup(self.Embedding,self.input_x) #shape:[None,sentence_length,embed_size] #2. Bi-lstm layer output_conv=self.conv_layer_with_recurrent_structure() #shape:[None,sentence_length,embed_size*3] #2.1 apply nolinearity #b = tf.get_variable("b", [self.embed_size*3]) #h = tf.nn.relu(tf.nn.bias_add(output_conv, b), "relu") #3. max pooling output_pooling=tf.reduce_max(output_conv,axis=1) #shape:[None,embed_size*3] #4. logits(use linear layer) with tf.name_scope("dropout"): h_drop=tf.nn.dropout(output_pooling,keep_prob=self.dropout_keep_prob) #[None,embed_size*3] with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. logits = tf.matmul(h_drop, self.W_projection) + self.b_projection #shape:[batch_size,num_classes]<-----h_drop:[None,embed_size*3];b_projection:[hidden_size*3, self.num_classes] return logits
def inference2(self): """main computation graph here: 1. embeddding layer, 2.Bi-LSTM layer, 3.max pooling, 4.FC layer 5.softmax """ #1.get emebedding of words in the sentence self.embedded_words = tf.nn.embedding_lookup(self.Embedding,self.input_x) #shape:[None,sentence_length,embed_size] #2. Bi-lstm layer output_conv=self.conv_layer_with_recurrent_structure() #shape:[None,sentence_length,embed_size*3] #3. max pooling #print("output_conv:",output_conv) #(3, 5, 8, 100) output_pooling=tf.reduce_max(output_conv,axis=1) #shape:[None,embed_size*3] #print("output_pooling:",output_pooling) #(3, 8, 100) #4. logits(use linear layer) with tf.name_scope("dropout_rcnn"): h_drop=tf.nn.dropout(output_pooling,keep_prob=self.dropout_keep_prob) #[None,embed_size*3] #with tf.name_scope("output"): #inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. logits = tf.matmul(h_drop, self.W_projection_rcnn) + self.b_projection_rcnn # [batch_size,num_classes] return logits
def model(inputs, is_training=False): # Multiple parallel convolutions block1 = multi_conv(inputs, [1, 2, 3, 4, 5, 6, 7, 8], [512, 512, 512, 512, 512, 512, 512, 512], name='block1', is_training=is_training) net = tf.reduce_max(block1, axis=1, name='maxpool') # Fully connected hidden layer (dense -> batch norm -> relu -> dropout) net = tf.layers.dense(net, 4096, kernel_regularizer=kernel_regularizer, name='fc1') net = tf.layers.batch_normalization(net, training=is_training, name='fc1/batch_normalization') net = tf.nn.relu(net, name='fc1/relu') net = tf.layers.dropout(net, rate=0.5, training=is_training, name='fc1/dropout') # Fully connected output layer net = tf.layers.dense(net, 4716, kernel_regularizer=kernel_regularizer, name='fc2') tf.summary.histogram('summary/fc2', tf.nn.sigmoid(net)) return net
def define_model(x, keep_prob, number_of_classes, number_of_filters, number_of_fc_features): splitted = tf.unpack(x, axis=4) branches = [] with tf.variable_scope('branches') as scope: for index, tensor_slice in enumerate(splitted): branches.append(single_branch(splitted[index], number_of_filters, number_of_fc_features)) if (index == 0): scope.reuse_variables() concatenated = tf.pack(branches, axis=2) ti_pooled = tf.reduce_max(concatenated, reduction_indices=[2]) drop = tf.nn.dropout(ti_pooled, keep_prob) with tf.variable_scope('fc2'): logits = fc(drop, [number_of_fc_features, number_of_classes], [number_of_classes]) return logits
def __call__(self, x, scope=None): with tf.variable_scope(scope or type(self).__name__): # Check if the input size exist. input_size = x.get_shape().with_rank(2)[1] if input_size is None: raise ValueError("Expecting input_size to be set.") maxout_Wo = tf.get_variable(name='Wo', shape=(input_size, 2*self._units), initializer=gaussian_initializer(mean=0.0, std=0.01)) maxout_b = tf.get_variable(name='b', shape=(2*self._units,), initializer=tf.constant_initializer(0.0)) # 1st. Compute on all the 2 channels and reshape. t = tf.matmul(x, maxout_Wo) + maxout_b t = tf.reshape(t, shape=(-1, self._units, 2)) # 2nd. Do maxout op, now has shape: (None, self._units) maxout_t = tf.reduce_max(t, axis=-1) return maxout_t
def gumbel_softmax(logits, temperature, hard=False): """Sample from the Gumbel-Softmax distribution and optionally discretize. Args: logits: [batch_size, n_class] unnormalized log-probs temperature: non-negative scalar hard: if True, take argmax, but differentiate w.r.t. soft sample y Returns: [batch_size, n_class] sample from the Gumbel-Softmax distribution. If hard=True, then the returned sample will be one-hot, otherwise it will be a probabilitiy distribution that sums to 1 across classes """ y = gumbel_softmax_sample(logits, temperature) #if hard: # k = tf.shape(logits)[-1] # #y_hard = tf.cast(tf.one_hot(tf.argmax(y,1),k), y.dtype) # y_hard = tf.cast(tf.equal(y,tf.reduce_max(y,1,keep_dims=True)),y.dtype) # y = tf.stop_gradient(y_hard - y) + y return y
def bag_hinge_loss(config, preds, sent_mask, flip_sent_mask, hete_mask, sent_trgt, sent_num): """ HINGE LOSS: DEFINED AS: MAX(0, M - MIN(SENT+) - MAX(SENT-)) THIS ONLY APPLIES TO HETE BAGS. """ flip_sent_trgt = \ tf.constant(1, shape=[config.batch_size,sent_num], dtype=config.data_type) - \ sent_trgt pos_preds = preds + flip_sent_trgt + flip_sent_mask # [batch_size, sent_num] neg_preds = preds * flip_sent_trgt * sent_mask # [batch_size, sent_num] min_pos_pred = tf.reduce_min(pos_preds, 1) # min_pos_pred = tf.Print(min_pos_pred, [min_pos_pred], message='min_pos_pred') max_neg_pred = tf.reduce_max(neg_preds, 1) # max_neg_pred = tf.Print(max_neg_pred, [max_neg_pred], message='max_neg_pred') hinge_loss = hete_mask * tf.reduce_max(tf.pack( [tf.constant(0, shape=[config.batch_size], dtype=config.data_type), (0.20 - min_pos_pred + max_neg_pred)], axis=1), 1) # [batch_size] # hinge_loss = tf.Print(hinge_loss, [hinge_loss], message='hinge_loss', summarize=20) avg_hinge_loss = tf.reduce_sum(hinge_loss) / (tf.reduce_sum(hete_mask) + 1e-12) return avg_hinge_loss
def segment_softmax(scores, segment_ids): """Given scores and a partition, converts scores to probs by performing softmax over all rows within a partition.""" # Subtract max num_segments = tf.reduce_max(segment_ids) + 1 if len(scores.get_shape()) == 2: max_per_partition = tf.unsorted_segment_max(tf.reduce_max(scores, axis=1), segment_ids, num_segments) scores -= tf.expand_dims(tf.gather(max_per_partition, segment_ids), axis=1) else: max_per_partition = tf.unsorted_segment_max(scores, segment_ids, num_segments) scores -= tf.gather(max_per_partition, segment_ids) # Compute probs scores_exp = tf.exp(scores) if len(scores.get_shape()) == 2: scores_exp_sum_per_partition = tf.unsorted_segment_sum(tf.reduce_sum(scores_exp, axis=1), segment_ids, num_segments) probs = scores_exp / tf.expand_dims(tf.gather(scores_exp_sum_per_partition, segment_ids), axis=1) else: scores_exp_sum_per_partition = tf.unsorted_segment_sum(scores_exp, segment_ids, num_segments) probs = scores_exp / tf.gather(scores_exp_sum_per_partition, segment_ids) return probs
def global_attention(state, hidden_states, encoder, encoder_input_length, scope=None, context=None, **kwargs): with tf.variable_scope(scope or 'attention_{}'.format(encoder.name)): if context is not None and encoder.use_context: state = tf.concat([state, context], axis=1) if encoder.attn_filters: e = compute_energy_with_filter(hidden_states, state, attn_size=encoder.attn_size, attn_filters=encoder.attn_filters, attn_filter_length=encoder.attn_filter_length, **kwargs) else: e = compute_energy(hidden_states, state, attn_size=encoder.attn_size, attn_keep_prob=encoder.attn_keep_prob, pervasive_dropout=encoder.pervasive_dropout, layer_norm=encoder.layer_norm, mult_attn=encoder.mult_attn, **kwargs) e -= tf.reduce_max(e, axis=1, keep_dims=True) mask = tf.sequence_mask(encoder_input_length, maxlen=tf.shape(hidden_states)[1], dtype=tf.float32) T = encoder.attn_temperature or 1.0 exp = tf.exp(e / T) * mask weights = exp / tf.reduce_sum(exp, axis=-1, keep_dims=True) weighted_average = tf.reduce_sum(tf.expand_dims(weights, 2) * hidden_states, axis=1) return weighted_average, weights
def compute_loss(self): """ ??loss Return: loss: scalar """ if not self._use_crf: labels = tf.reshape( tf.contrib.layers.one_hot_encoding( tf.reshape(self.input_label_ph, [-1]), num_classes=self._nb_classes), shape=[-1, self._sequence_length, self._nb_classes]) cross_entropy = -tf.reduce_sum(labels * tf.log(self.logits), axis=2) mask = tf.sign(tf.reduce_max(tf.abs(labels), axis=2)) cross_entropy_masked = tf.reduce_sum( cross_entropy*mask, axis=1) / tf.cast(self.sequence_actual_length, tf.float32) return tf.reduce_mean(cross_entropy_masked) else: log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood( self.logits, self.input_label_ph, self.sequence_actual_length) return tf.reduce_mean(-log_likelihood)
def log_sum_exp(x, axis=None, keep_dims=False): """ Deprecated: Use tf.reduce_logsumexp(). Tensorflow numerically stable log sum of exps across the `axis`. :param x: A Tensor or numpy array. :param axis: An int or list or tuple. The dimensions to reduce. If `None` (the default), reduces all dimensions. :param keep_dims: Bool. If true, retains reduced dimensions with length 1. Default to be False. :return: A Tensor after the computation of log sum exp along given axes of x. """ x = tf.cast(x, dtype=tf.float32) x_max = tf.reduce_max(x, axis=axis, keep_dims=True) ret = tf.log(tf.reduce_sum(tf.exp(x - x_max), axis=axis, keep_dims=True)) + x_max if not keep_dims: ret = tf.reduce_sum(ret, axis=axis) return ret
def log_mean_exp(x, axis=None, keep_dims=False): """ Tensorflow numerically stable log mean of exps across the `axis`. :param x: A Tensor or numpy array. :param axis: An int or list or tuple. The dimensions to reduce. If `None` (the default), reduces all dimensions. :param keep_dims: Bool. If true, retains reduced dimensions with length 1. Default to be False. :return: A Tensor after the computation of log mean exp along given axes of x. """ x = tf.cast(x, dtype=tf.float32) x_max = tf.reduce_max(x, axis=axis, keep_dims=True) ret = tf.log(tf.reduce_mean(tf.exp(x - x_max), axis=axis, keep_dims=True)) + x_max if not keep_dims: ret = tf.reduce_mean(ret, axis=axis) return ret
def variable_summaries(var, name, collections=None): """Attach a lot of summaries to a Tensor (for TensorBoard visualization). Args: - var: Tensor for variable from which we want to log. - name: Variable name. - collections: List of collections to save the summary to. """ with tf.name_scope(name): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean, collections) num_params = tf.reduce_prod(tf.shape(var)) tf.summary.scalar('num_params', num_params, collections) with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev, collections) tf.summary.scalar('max', tf.reduce_max(var), collections) tf.summary.scalar('min', tf.reduce_min(var), collections) tf.summary.histogram('histogram', var, collections) tf.summary.scalar('sparsity', tf.nn.zero_fraction(var), collections)
def _conform_kernel_to_tensor(kernel, tensor, shape): """ Re-shape a convolution kernel to match the given tensor's color dimensions. """ l = len(kernel) channels = shape[-1] temp = np.repeat(kernel, channels) temp = tf.reshape(temp, (l, l, channels, 1)) temp = tf.cast(temp, tf.float32) temp /= tf.maximum(tf.reduce_max(temp), tf.reduce_min(temp) * -1) return temp
def _joint_positions(self): highest_activation = tf.reduce_max(self.sigm_network, [1, 2]) x = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 1), 1) y = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 2), 1) x = tf.cast(x, tf.float32) y = tf.cast(y, tf.float32) a = tf.cast(highest_activation, tf.float32) scale_coef = (self.image_size / self.heatmap_size) x *= scale_coef y *= scale_coef out = tf.stack([y, x, a]) return out
def euclidean_distance(self): x = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 1), 1) y = tf.argmax(tf.reduce_max(self.smoothed_sigm_network, 2), 1) x = tf.cast(x, tf.float32) y = tf.cast(y, tf.float32) dy = tf.squeeze(self.desired_points[:, 0, :]) dx = tf.squeeze(self.desired_points[:, 1, :]) sx = tf.squared_difference(x, dx) sy = tf.squared_difference(y, dy) l2_dist = tf.sqrt(sx + sy) return l2_dist
def global_max_pool(incoming, name="GlobalMaxPool"): """ Global Max Pooling. Input: 4-D Tensor [batch, height, width, in_channels]. Output: 2-D Tensor [batch, pooled dim] Arguments: incoming: `Tensor`. Incoming 4-D Tensor. name: A name for this layer (optional). Default: 'GlobalMaxPool'. """ input_shape = utils.get_incoming_shape(incoming) assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" with tf.name_scope(name): inference = tf.reduce_max(incoming, [1, 2]) # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference) return inference
def _build_loss(self): config = self.config JX = tf.shape(self.x)[2] M = tf.shape(self.x)[1] JQ = tf.shape(self.q)[1] loss_mask = tf.reduce_max(tf.cast(self.q_mask, 'float'), 1) losses = tf.nn.softmax_cross_entropy_with_logits( self.logits, tf.cast(tf.reshape(self.y, [-1, M * JX]), 'float')) ce_loss = tf.reduce_mean(loss_mask * losses) tf.add_to_collection('losses', ce_loss) ce_loss2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( self.logits2, tf.cast(tf.reshape(self.y2, [-1, M * JX]), 'float'))) tf.add_to_collection("losses", ce_loss2) self.loss = tf.add_n(tf.get_collection('losses', scope=self.scope), name='loss') tf.scalar_summary(self.loss.op.name, self.loss) tf.add_to_collection('ema/scalar', self.loss)
def build_graph(self, q_network, config): self.ph_reward = tf.placeholder(tf.float32, [None]) self.ph_action = tf.placeholder(tf.int32, [None]) self.ph_terminal = tf.placeholder(tf.int32, [None]) self.ph_q_next_target = tf.placeholder(tf.float32, [None, config.output.action_size]) self.ph_q_next = tf.placeholder(tf.float32, [None, config.output.action_size]) action_one_hot = tf.one_hot(self.ph_action, config.output.action_size) q_action = tf.reduce_sum(tf.multiply(q_network.node, action_one_hot), axis=1) if config.double_dqn: q_max = tf.reduce_sum(self.ph_q_next_target * tf.one_hot(tf.argmax(self.ph_q_next, axis=1), config.output.action_size), axis=1) else: q_max = tf.reduce_max(self.ph_q_next_target, axis=1) y = self.ph_reward + tf.cast(1 - self.ph_terminal, tf.float32) * tf.scalar_mul(config.rewards_gamma, q_max) return tf.losses.absolute_difference(q_action, y)
def attentive_pooling(self,input_left,input_right): Q = tf.reshape(input_left,[self.batch_size,self.max_input_left,len(self.filter_sizes) * self.num_filters],name = 'Q') A = tf.reshape(input_right,[self.batch_size,self.max_input_right,len(self.filter_sizes) * self.num_filters],name = 'A') # G = tf.tanh(tf.matmul(tf.matmul(Q,self.U),\ # A,transpose_b = True),name = 'G') first = tf.matmul(tf.reshape(Q,[-1,len(self.filter_sizes) * self.num_filters]),self.U) second_step = tf.reshape(first,[self.batch_size,-1,len(self.filter_sizes) * self.num_filters]) result = tf.matmul(second_step,tf.transpose(A,perm = [0,2,1])) G = tf.tanh(result) # column-wise pooling ,row-wise pooling row_pooling = tf.reduce_max(G,1,True,name = 'row_pooling') col_pooling = tf.reduce_max(G,2,True,name = 'col_pooling') attention_q = tf.nn.softmax(col_pooling,1,name = 'attention_q') attention_a = tf.nn.softmax(row_pooling,name = 'attention_a') R_q = tf.reshape(tf.matmul(Q,attention_q,transpose_a = 1),[self.batch_size,self.num_filters * len(self.filter_sizes),-1],name = 'R_q') R_a = tf.reshape(tf.matmul(attention_a,A),[self.batch_size,self.num_filters * len(self.filter_sizes),-1],name = 'R_a') return R_q,R_a
def gradient_binarizing_scalers(grads_and_vars, clip_factor): """ Get the scalers.""" gradients, variables = zip(*grads_and_vars) scalers = [] for gradient in gradients: if gradient is None: scalers.append(None) continue if(clip_factor > 1.0e-5): mean_gradient = tf.reduce_mean(gradient) stddev_gradient = tf.sqrt(tf.reduce_mean(tf.square(gradient - mean_gradient))) scalers.append(clip_factor * stddev_gradient) else: scalers.append(tf.reduce_max(tf.abs(gradient))) return list(zip(scalers, variables))
def average_precision_voc07(precision, recall, name=None): """Compute (interpolated) average precision from precision and recall Tensors. The implementation follows Pascal 2007 guidelines. See also: https://sanchom.wordpress.com/tag/average-precision/ """ with tf.name_scope(name, 'average_precision_voc07', [precision, recall]): # Convert to float64 to decrease error on cumulated sums. precision = tf.cast(precision, dtype=tf.float64) recall = tf.cast(recall, dtype=tf.float64) # Add zero-limit value to avoid any boundary problem... precision = tf.concat([precision, [0.]], axis=0) recall = tf.concat([recall, [np.inf]], axis=0) # Split the integral into 10 bins. l_aps = [] for t in np.arange(0., 1.1, 0.1): mask = tf.greater_equal(recall, t) v = tf.reduce_max(tf.boolean_mask(precision, mask)) l_aps.append(v / 11.) ap = tf.add_n(l_aps) return ap
def ctc_label_dense_to_sparse(labels, label_lengths, batch_size): # The second dimension of labels must be equal to the longest label length in the batch correct_shape_assert = tf.assert_equal(tf.shape(labels)[1], tf.reduce_max(label_lengths)) with tf.control_dependencies([correct_shape_assert]): labels = tf.identity(labels) label_shape = tf.shape(labels) num_batches_tns = tf.stack([label_shape[0]]) max_num_labels_tns = tf.stack([label_shape[1]]) def range_less_than(previous_state, current_input): return tf.expand_dims(tf.range(label_shape[1]), 0) < current_input init = tf.cast(tf.fill(max_num_labels_tns, 0), tf.bool) init = tf.expand_dims(init, 0) dense_mask = tf.scan(range_less_than, label_lengths, initializer=init, parallel_iterations=1) dense_mask = dense_mask[:, 0, :] label_array = tf.reshape(tf.tile(tf.range(0, label_shape[1]), num_batches_tns), label_shape) label_ind = tf.boolean_mask(label_array, dense_mask) batch_array = tf.transpose(tf.reshape(tf.tile(tf.range(0, label_shape[0]), max_num_labels_tns), tf.reverse(label_shape, [0]))) batch_ind = tf.boolean_mask(batch_array, dense_mask) indices = tf.transpose(tf.reshape(tf.concat([batch_ind, label_ind], 0), [2, -1])) shape = [batch_size, tf.reduce_max(label_lengths)] vals_sparse = gather_nd(labels, indices, shape) return tf.SparseTensor(tf.to_int64(indices), vals_sparse, tf.to_int64(label_shape)) # Validate and normalize transcriptions. Returns a cleaned version of the label # or None if it's invalid.
def create_model(self, model_input, vocab_size, num_frames, **unused_params): shape = model_input.get_shape().as_list() frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2) frames_true = tf.ones(tf.shape(frames_sum)) frames_false = tf.zeros(tf.shape(frames_sum)) frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1]) activation_1 = tf.reduce_max(model_input, axis=1) activation_2 = tf.reduce_sum(model_input*frames_bool, axis=1)/(tf.reduce_sum(frames_bool, axis=1)+1e-6) activation_3 = tf.reduce_min(model_input, axis=1) model_input_1, final_probilities_1 = self.sub_moe(activation_1,vocab_size,scopename="_max") model_input_2, final_probilities_2 = self.sub_moe(activation_2,vocab_size,scopename="_mean") model_input_3, final_probilities_3 = self.sub_moe(activation_3,vocab_size,scopename="_min") final_probilities = tf.stack((final_probilities_1,final_probilities_2,final_probilities_3),axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[shape[2], 3, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) activations = tf.stack((model_input_1, model_input_2, model_input_3), axis=2) weight = tf.nn.softmax(tf.einsum("aij,ijk->ajk", activations, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, distill_labels=None, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input cnn_size = FLAGS.cnn_cells num_filters = [cnn_size, cnn_size, cnn_size*2] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale,vocab_size,distill_labels=distill_labels, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, lstm_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input num_filters = [256, 256, 512] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn_gate(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities, axis=1) moe_inputs = tf.stack(moe_inputs, axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size]) result["predictions"] = tf.reduce_mean(final_probilities, axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input num_filters = [256, 256, 512] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn_glu(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities, axis=1) moe_inputs = tf.stack(moe_inputs, axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size]) result["predictions"] = tf.reduce_mean(final_probilities, axis=1) return result
