我们从Python开源项目中，提取了以下37个代码示例，用于说明如何使用tensorflow.reduce_all()。
def categorical_accuracy_with_variable_timestep(y_true, y_pred): # Actually discarding is not needed if the dummy is an all-zeros array # (It is indeed encoded in an all-zeros array by # CaptionPreprocessing.preprocess_batch) y_true = y_true[:, :-1, :] # Discard the last timestep/word (dummy) y_pred = y_pred[:, :-1, :] # Discard the last timestep/word (dummy) # Flatten the timestep dimension shape = tf.shape(y_true) y_true = tf.reshape(y_true, [-1, shape[-1]]) y_pred = tf.reshape(y_pred, [-1, shape[-1]]) # Discard rows that are all zeros as they represent dummy or padding words. is_zero_y_true = tf.equal(y_true, 0) is_zero_row_y_true = tf.reduce_all(is_zero_y_true, axis=-1) y_true = tf.boolean_mask(y_true, ~is_zero_row_y_true) y_pred = tf.boolean_mask(y_pred, ~is_zero_row_y_true) accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y_true, axis=1), tf.argmax(y_pred, axis=1)), dtype=tf.float32)) return accuracy # As Keras stores a function's name as its metric's name
def is_same_dynamic_shape(x, y): """ Whether `x` and `y` has the same dynamic shape. :param x: A Tensor. :param y: A Tensor. :return: A scalar Tensor of `bool`. """ # There is a BUG of Tensorflow for not doing static shape inference # right in nested tf.cond()'s, so we are not comparing x and y's # shape directly but working with their concatenations. return tf.cond( tf.equal(tf.rank(x), tf.rank(y)), lambda: tf.reduce_all(tf.equal( tf.concat([tf.shape(x), tf.shape(y)], 0), tf.concat([tf.shape(y), tf.shape(x)], 0))), lambda: tf.convert_to_tensor(False, tf.bool))
def preturn_network(rewards, discounts, values): # First reward must be zero, first discount must be one first_reward = tf.Assert( tf.reduce_all(tf.equal(rewards[:, 0, :], 0.0)), [rewards[:, 0, :]]) first_discount = tf.Assert( tf.reduce_all(tf.equal(discounts[:, 0, :], 1.0)), [discounts[:, 0, :]]) with tf.control_dependencies([first_reward, first_discount]): with tf.variable_scope('preturn'): accum_value_discounts = tf.cumprod(discounts, axis=1, exclusive=False) accum_reward_discounts = tf.cumprod(discounts, axis=1, exclusive=True) discounted_values = values * accum_value_discounts discounted_rewards = rewards * accum_reward_discounts cumulative_rewards = tf.cumsum(discounted_rewards, axis=1) preturns = cumulative_rewards + discounted_values util.activation_summary(preturns) return preturns
def all(x, axis=None, keepdims=False): '''Bitwise reduction (logical AND). Returns an uint8 tensor ''' axis = _normalize_axis(axis, ndim(x)) x = tf.cast(x, tf.bool) x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims) return tf.cast(x, tf.uint8)
def barycentric(verts, p): ab = verts[2] - verts[0] ac = verts[1] - verts[0] pa = verts[0] - p u = utils.tri_cross( [ab[0], ac[0], pa[:, 0]], [ab[1], ac[1], pa[:, 1]]) v = [u[0] / u[2], u[1] / u[2]] bc = [1. - v[0] - v[1], v[1], v[0]] valid = tf.logical_and( tf.abs(u[2]) >= 1.0, tf.reduce_all(tf.stack(bc, axis=1) >= 0, axis=1)) return bc, valid
def per_sentence_accuracy(targets, predictions, weights=None): """Computes the per-sentence accuracy. Given a set of ground truth values and a set of predicted labels as tensors of the same shape, it returns a tensor of rank equal to the ground truth values tensor minus 1, with values 1.0 if all the predicted values along the -1 axis are correct, 0.0 otherwise. So, if the grount truth is [[1, 2, 3], [0, 9, 23]] and the predicted labels are [[1, 2, 3], [9, 0, 23]] the result will be: [1,0]. Arguments: target: the gold truth values `Tensor`, with `tf.int32` as `dtype`. It has rank `[d_0, d_1, ..., d_{r-1}]` and the last value is supposed to range between `0` and `num_classes - 1`, where `num_classes` is the number of possible classes. predictions: the predicted values `Tensor` with `tf.float32` as `dtype`. It can have shape `[d_0, d_1, ..., d_{r-1}, num_classes]` and dtype `float32` and represents the probability distribution across the output classes generated by the model -- so that the predicted label is the one coming from argmax over the last dimension. Alternatively it can be of the same shape, `dtype` and format of `target`, and it will considered as the predicted labels. weights: coefficients for the metric. This must be scalar or of same rank as `target`. Returns: values: a `Tensor` of `dtype=tf.float32` and of [d_0, d_1, ..., d_{r-2}] representing the accuracy per sentence, i.e. for all the elements of the -1 axis, weighted according to the input argument `weights` weights: a `Tensor` of `dtype=tf.float32` and of the same shape of `values` representing the weighted scheme for the streaming average on `values`, which is the same tensor of the input `weights` argument. """ values, weights = accuracy(targets, predictions, weights) values = tf.cast(values, tf.bool) if weights is not None: weights = tf.cast(weights, tf.bool) values = ops.logical_impl(weights, values) values = tf.reduce_all(values, axis=-1) return tf.cast(values, tf.float32), tf.cast(tf.ones_like(values), tf.float32)
def all(x, axis=None, keepdims=False): """Bitwise reduction (logical AND). # Arguments x: input tensor. axis: axis along which to perform the reduction. keepdims: whether the drop or broadcast the reduction axes. # Returns A uint8 tensor (0s and 1s). """ axis = _normalize_axis(axis, ndim(x)) x = tf.cast(x, tf.bool) x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims) return tf.cast(x, tf.uint8)
def _leapfrog_step(xs, ps, epsilon, max_iterations, logprob_grads_fn): def update_xs(ps_values): return _map(lambda x, p: x.assign_add(epsilon * p), xs, ps_values) def whether_proceed(grads): finits = _map(lambda grad: tf.reduce_all(tf.is_finite(grad)), grads) return tf.reduce_all(finits) def cond(i, proceed, _ps, _xs): return tf.logical_and(proceed, i < max_iterations) def body(i, _proceed, ps, _xs): xs_new = update_xs(ps) with tf.control_dependencies(xs_new): _, grads = logprob_grads_fn() proceed = whether_proceed(grads) def ps_step(): with tf.control_dependencies(grads): return _update_ps(ps, grads, epsilon) def ps_no_step(): with tf.control_dependencies(grads): return ps ps_new = tf.cond(proceed, ps_step, ps_no_step, strict=True) return i + 1, proceed, ps_new, xs_new result = _while_loop(cond, body, [0, True, ps, xs]) _i, proceed_out, ps_out, xs_out = result deps = _flat([proceed_out], ps_out, xs_out) with tf.control_dependencies(deps): logprob_out, grads_out = logprob_grads_fn() return proceed_out, xs_out, ps_out, logprob_out, grads_out
def initialize(self, name=None): with tf.name_scope(name, "TrainingHelperInitialize"): finished = tf.equal(0, self._sequence_length) all_finished = tf.reduce_all(finished) next_inputs = tf.cond( all_finished, lambda: self._zero_inputs, lambda: nest.map_structure(lambda inp: inp.read(0), self._input_tas)) return (finished, next_inputs)
def next_inputs(self, time, outputs, state, name=None, **unused_kwargs): """next_inputs_fn for TrainingHelper.""" with tf.name_scope(name, "TrainingHelperNextInputs", [time, outputs, state]): next_time = time + 1 finished = (next_time >= self._sequence_length) all_finished = tf.reduce_all(finished) def read_from_ta(inp): return inp.read(next_time) next_inputs = tf.cond( all_finished, lambda: self._zero_inputs, lambda: nest.map_structure(read_from_ta, self._input_tas)) return (finished, next_inputs, state)
def next_inputs(self, time, outputs, state, sample_ids, name=None): with tf.name_scope(name, "ScheduledEmbeddingTrainingHelperSample", [time, outputs, state, sample_ids]): (finished, base_next_inputs, state) = ( super(ScheduledEmbeddingTrainingHelper, self).next_inputs( time=time, outputs=outputs, state=state, sample_ids=sample_ids, name=name)) def maybe_sample(): """Perform scheduled sampling.""" where_sampling = tf.cast( tf.where(sample_ids > -1), tf.int32) where_not_sampling = tf.cast( tf.where(sample_ids <= -1), tf.int32) where_sampling_flat = tf.reshape(where_sampling, [-1]) where_not_sampling_flat = tf.reshape( where_not_sampling, [-1]) sample_ids_sampling = tf.gather( sample_ids, where_sampling_flat) inputs_not_sampling = tf.gather( base_next_inputs, where_not_sampling_flat) sampled_next_inputs = self._embedding_fn(sample_ids_sampling) base_shape = tf.shape(base_next_inputs) return (tf.scatter_nd(indices=where_sampling, updates=sampled_next_inputs, shape=base_shape) + tf.scatter_nd(indices=where_not_sampling, updates=inputs_not_sampling, shape=base_shape)) all_finished = tf.reduce_all(finished) next_inputs = tf.cond( all_finished, lambda: base_next_inputs, maybe_sample) return (finished, next_inputs, state)
def test_name(self): result_lt = ops.reduce_all(self.bool_lt, {'channel'}) self.assertIn('lt_reduce_all', result_lt.name)
def test(self): result_lt = ops.reduce_all(self.bool_lt, {'channel'}) golden_lt = core.LabeledTensor( tf.reduce_all(self.bool_tensor, 1), [self.a0, self.a2, self.a3]) self.assertLabeledTensorsEqual(result_lt, golden_lt)
def colors_to_dimensions(image_tensor, colors): logger = get_logger() single_label_tensors = [] for single_label_color in colors: is_color = tf.reduce_all( tf.equal(image_tensor, single_label_color), axis=-1 ) single_label_tensor = tf.where( is_color, tf.fill(is_color.shape, 1.0), tf.fill(is_color.shape, 0.0) ) single_label_tensors.append(single_label_tensor) return tf.stack(single_label_tensors, axis=-1)
def replace_black_with_white_color(image_tensor): is_black = tf.reduce_all( tf.equal(image_tensor, (0, 0, 0)), axis=-1 ) is_black = tf.stack([is_black] * 3, axis=-1) return tf.where( is_black, 255 * tf.ones_like(image_tensor), image_tensor )
def lambda_preturn_network(preturns, lambdas): # Final lamdba must be zero final_lambda = tf.Assert( tf.reduce_all(tf.equal(lambdas[:, -1, :], 0.0)), [lambdas[:, -1, :]]) with tf.control_dependencies([final_lambda]): with tf.variable_scope('lambda_preturn'): accum_lambda = tf.cumprod(lambdas, axis=1, exclusive=True) lambda_bar = (1 - lambdas) * accum_lambda # This should always sum to 1 lambda_preturn = tf.reduce_sum( lambda_bar * preturns, reduction_indices=1) util.activation_summary(lambda_preturn) return lambda_preturn
def all(x, axis=None, keepdims=False): """Bitwise reduction (logical AND). # Arguments x: Tensor or variable. axis: axis along which to perform the reduction. keepdims: whether the drop or broadcast the reduction axes. # Returns A uint8 tensor (0s and 1s). """ axis = _normalize_axis(axis, ndim(x)) x = tf.cast(x, tf.bool) return tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
def next_inputs(self, time, outputs, state, sample_ids, name=None): '''Stop on EOS. Otherwise, pass the last output as the next input and pass through state.''' with tf.name_scope('TacoTestHelper'): finished = tf.reduce_all(tf.equal(outputs, self._end_token), axis=1) # Feed last output frame as next input. outputs is [N, output_dim * r] next_inputs = outputs[:, -self._output_dim:] return (finished, next_inputs, state)
def test_All(self): t = tf.reduce_all(self.random(3, 4, 5), reduction_indices=[0, 1], keep_dims=True) self.check(t) if td._tf_version[:3] >= (0, 12, 0): t = tf.reduce_all(self.random(3, 4, 5), axis=[0, 1], keep_dims=True) self.check(t)
def loop_fn_transition(time,previous_output,previous_state,previous_loop_state): #print time elements_finished = (time >= decoder_lengths) def next_input(): prev_out_with_weights = tf.matmul(previous_output,w['score']) prev_out_with_weights = tf.reshape(prev_out_with_weights,[-1,final_hidden_units,1]) score = tf.matmul(encoder_outputs,prev_out_with_weights) score = tf.reshape(score,[-1,num_steps]) attention = tf.nn.softmax(score) attention = tf.reshape(attention,[-1,1,num_steps]) ct = tf.matmul(attention,encoder_outputs) ct = tf.reshape(ct,[-1,final_hidden_units]) ctht = tf.concat((ct,previous_output),1) ht_dash = tf.nn.tanh(tf.add(tf.matmul(ctht,w['hdash']),b['hdash'])) pred = tf.nn.softmax(tf.add(tf.matmul(ctht,w['decoder']),b['decoder'])) prediction = tf.argmax(pred,axis=1) inputn = tf.nn.embedding_lookup(embeddings,prediction) return inputn finished = tf.reduce_all(elements_finished) next_input = tf.cond(finished,lambda:pad_embedded,next_input) state = previous_state output = previous_output #print output.shape loop_state = None return (elements_finished, next_input, state, output, loop_state) # In[31]:
def evaluation(logits, labels): prediction = tf.argmax(logits, 2) prediction = tf.cast(prediction, tf.int32) equal = tf.equal(prediction, labels) equal_all = tf.reduce_all(equal, axis=1) accuracy = tf.reduce_mean(tf.cast(equal_all, tf.float32), name="accuracy") return accuracy
def _init_decoder(self): """ Creates decoder attributes. We cannot simply use a dynamic_rnn since we are feeding the outputs of the decoder back into the inputs. Therefore we use a raw_rnn and emulate a dynamic_rnn with this behavior. (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/rnn.py) """ # EOS token added self.decoder_inputs_length = self.encoder_inputs_length + 1 def loop_fn_initial(time, cell_output, cell_state, loop_state): elements_finished = (time >= self.decoder_inputs_length) # EOS token (0 + self.EOS) initial_input = tf.zeros([self.batch_size, self.decoder_cell.output_size], dtype=tf.float32) + self.EOS initial_cell_state = self.encoder_final_state initial_loop_state = None # we don't need to pass any additional information return (elements_finished, initial_input, initial_cell_state, None, # cell output is dummy here initial_loop_state) def loop_fn(time, cell_output, cell_state, loop_state): if cell_output is None: # time == 0 return loop_fn_initial(time, cell_output, cell_state, loop_state) cell_output.set_shape([self.batch_size, self.decoder_cell.output_size]) emit_output = cell_output next_cell_state = cell_state elements_finished = (time >= self.decoder_inputs_length) finished = tf.reduce_all(elements_finished) next_input = tf.cond( finished, lambda: tf.zeros([self.batch_size, self.decoder_cell.output_size], dtype=tf.float32), # self.PAD lambda: cell_output # Use the input from the previous cell ) next_loop_state = None return ( elements_finished, next_input, next_cell_state, emit_output, next_loop_state ) decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(self.decoder_cell, loop_fn) self.decoder_outputs = decoder_outputs_ta.stack() self.decoder_outputs = tf.transpose(self.decoder_outputs, [1, 0, 2]) with tf.variable_scope('DecoderOutputProjection') as scope: self.decoder_outputs = self.projection(self.decoder_outputs, self.seq_width, scope)
def _log_prob(self, given): logits = self.logits def _broadcast(given, logits): # static shape has been checked in base class. ones_ = tf.ones(tf.shape(logits)[:-1], self.dtype) if logits.get_shape(): ones_.set_shape(logits.get_shape()[:-1]) given *= ones_ logits *= tf.ones_like(tf.expand_dims(given, -1), self.param_dtype) return given, logits def _is_same_dynamic_shape(given, logits): return tf.cond( tf.equal(tf.rank(given), tf.rank(logits) - 1), lambda: tf.reduce_all(tf.equal( tf.concat([tf.shape(given), tf.shape(logits)[:-1]], 0), tf.concat([tf.shape(logits)[:-1], tf.shape(given)], 0))), lambda: tf.convert_to_tensor(False, tf.bool)) if not (given.get_shape() and logits.get_shape()): given, logits = _broadcast(given, logits) else: if given.get_shape().ndims != logits.get_shape().ndims - 1: given, logits = _broadcast(given, logits) elif given.get_shape().is_fully_defined() and \ logits.get_shape()[:-1].is_fully_defined(): if given.get_shape() != logits.get_shape()[:-1]: given, logits = _broadcast(given, logits) else: # Below code seems to induce a BUG when this function is # called in HMC. Probably due to tensorflow's not supporting # control flow edge from an op inside the body to outside. # We should further fix this. # # given, logits = tf.cond( # is_same_dynamic_shape(given, logits), # lambda: (given, logits), # lambda: _broadcast(given, logits, 'given', 'logits')) given, logits = _broadcast(given, logits) # `labels` type of `sparse_softmax_cross_entropy_with_logits` must be # int32 or int64 if self.dtype == tf.float32: given = tf.cast(given, dtype=tf.int32) elif self.dtype == tf.float64: given = tf.cast(given, dtype=tf.int64) elif self.dtype not in [tf.int32, tf.int64]: given = tf.cast(given, tf.int32) log_p = -tf.nn.sparse_softmax_cross_entropy_with_logits(labels=given, logits=logits) if given.get_shape() and logits.get_shape(): log_p.set_shape(tf.broadcast_static_shape(given.get_shape(), logits.get_shape()[:-1])) return log_p
def _jsma_impl(model, x, yind, epochs, eps, clip_min, clip_max, score_fn): def _cond(i, xadv): return tf.less(i, epochs) def _body(i, xadv): ybar = model(xadv) dy_dx = tf.gradients(ybar, xadv)[0] # gradients of target w.r.t input yt = tf.gather_nd(ybar, yind) dt_dx = tf.gradients(yt, xadv)[0] # gradients of non-targets w.r.t input do_dx = dy_dx - dt_dx c0 = tf.logical_or(eps < 0, xadv < clip_max) c1 = tf.logical_or(eps > 0, xadv > clip_min) cond = tf.reduce_all([dt_dx >= 0, do_dx <= 0, c0, c1], axis=0) cond = tf.to_float(cond) # saliency score for each pixel score = cond * score_fn(dt_dx, do_dx) shape = score.get_shape().as_list() dim = _prod(shape[1:]) score = tf.reshape(score, [-1, dim]) # find the pixel with the highest saliency score ind = tf.argmax(score, axis=1) dx = tf.one_hot(ind, dim, on_value=eps, off_value=0.0) dx = tf.reshape(dx, [-1] + shape[1:]) xadv = tf.stop_gradient(xadv + dx) xadv = tf.clip_by_value(xadv, clip_min, clip_max) return i+1, xadv _, xadv = tf.while_loop(_cond, _body, (0, tf.identity(x)), back_prop=False, name='_jsma_batch') return xadv
def _jsma2_impl(model, x, yind, epochs, eps, clip_min, clip_max, score_fn): def _cond(k, xadv): return tf.less(k, epochs) def _body(k, xadv): ybar = model(xadv) dy_dx = tf.gradients(ybar, xadv)[0] # gradients of target w.r.t input yt = tf.gather_nd(ybar, yind) dt_dx = tf.gradients(yt, xadv)[0] # gradients of non-targets w.r.t input do_dx = dy_dx - dt_dx c0 = tf.logical_or(eps < 0, xadv < clip_max) c1 = tf.logical_or(eps > 0, xadv > clip_min) cond = tf.reduce_all([dt_dx >= 0, do_dx <= 0, c0, c1], axis=0) cond = tf.to_float(cond) # saliency score for each pixel score = cond * score_fn(dt_dx, do_dx) shape = score.get_shape().as_list() dim = _prod(shape[1:]) score = tf.reshape(score, [-1, dim]) a = tf.expand_dims(score, axis=1) b = tf.expand_dims(score, axis=2) score2 = tf.reshape(a + b, [-1, dim*dim]) ij = tf.argmax(score2, axis=1) i = tf.to_int32(ij / dim) j = tf.to_int32(ij) % dim dxi = tf.one_hot(i, dim, on_value=eps, off_value=0.0) dxj = tf.one_hot(j, dim, on_value=eps, off_value=0.0) dx = tf.reshape(dxi + dxj, [-1] + shape[1:]) xadv = tf.stop_gradient(xadv + dx) xadv = tf.clip_by_value(xadv, clip_min, clip_max) return k+1, xadv _, xadv = tf.while_loop(_cond, _body, (0, tf.identity(x)), back_prop=False, name='_jsma2_batch') return xadv
def split_proposals(proposals, proposals_num, gt, gt_num, iou, scores, cross_boundary_mask): '''Generate batches from proposals and ground truth boxes Idea is to drastically reduce number of proposals to evaluate. So, we find those proposals that have IoU > 0.7 with _any_ ground truth and mark them as positive samples. Proposals with IoU < 0.3 with _all_ ground truth boxes are considered negative. All other proposals are discarded. We generate batch with at most half of examples being positive. We also pad them with negative have we not enough positive proposals. proposals: N x 4 tensor proposal_num: N gt: M x 4 tensor gt_num: M iou: N x M tensor of IoU between every proposal and ground truth scores: N x 2 tensor with scores object/not-object cross_boundary_mask: N x 1 Tensor masking out-of-image proposals ''' # now let's get rid of non-positive and non-negative samples # Sample is considered positive if it has IoU > 0.7 with _any_ ground truth box # XXX: maximal IoU ground truth proposal should be treated as positive positive_mask = tf.reduce_any(tf.greater(iou, 0.7), axis=1) & cross_boundary_mask # Sample would be considered negative if _all_ ground truch box # have iou less than 0.3 negative_mask = tf.reduce_all(tf.less(iou, 0.3), axis=1) & cross_boundary_mask # Select only positive boxes and their corresponding predicted scores positive_boxes = tf.boolean_mask(proposals, positive_mask) positive_scores = tf.boolean_mask(scores, positive_mask) positive_labels = tf.reduce_mean(tf.ones_like(positive_scores), axis=1) # Same for negative negative_boxes = tf.boolean_mask(proposals, negative_mask) negative_scores = tf.boolean_mask(scores, negative_mask) negative_labels = tf.reduce_mean(tf.zeros_like(negative_scores), axis=1) return ( (positive_boxes, positive_scores, positive_labels), (negative_boxes, negative_scores, negative_labels) )
def merge(tensors_list, mode, axis=1, name='merge', outputs_collections=None, **kwargs): """ Merge op Args: tensor_list: A list `Tensors` to merge mode: str, available modes are ['concat', 'elemwise_sum', 'elemwise_mul', 'sum', 'mean', 'prod', 'max', 'min', 'and', 'or'] name: a optional scope/name of the layer outputs_collections: The collections to which the outputs are added. Returns: A `Tensor` representing the results of the repetition operation. Raises: ValueError: If 'kernel_size' is not a 2-D list """ assert len(tensors_list) > 1, "Merge required 2 or more tensors." with tf.name_scope(name): tensors = [l for l in tensors_list] if mode == 'concat': output = tf.concat(tensors, axis=axis) elif mode == 'elemwise_sum': output = tensors[0] for i in range(1, len(tensors)): output = tf.add(output, tensors[i]) elif mode == 'elemwise_mul': output = tensors[0] for i in range(1, len(tensors)): output = tf.multiply(output, tensors[i]) elif mode == 'sum': output = tf.reduce_sum(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'mean': output = tf.reduce_mean(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'prod': output = tf.reduce_prod(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'max': output = tf.reduce_max(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'min': output = tf.reduce_min(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'and': output = tf.reduce_all(tf.concat(tensors, axis=axis), axis=axis) elif mode == 'or': output = tf.reduce_any(tf.concat(tensors, axis=axis), axis=axis) else: raise Exception("Unknown merge mode", str(mode)) return _collect_named_outputs(outputs_collections, name, output) return output
def next_inputs(self, time, outputs, state, sample_ids, name=None): with tf.name_scope(name, "ScheduledOutputTrainingHelperNextInputs", [time, outputs, state, sample_ids]): (finished, base_next_inputs, state) = ( super(ScheduledOutputTrainingHelper, self).next_inputs( time=time, outputs=outputs, state=state, sample_ids=sample_ids, name=name)) def maybe_sample(): """Perform scheduled sampling.""" def maybe_concatenate_auxiliary_inputs(outputs_, indices=None): """Concatenate outputs with auxiliary inputs, if they exist.""" if self._auxiliary_input_tas is None: return outputs_ next_time = time + 1 auxiliary_inputs = nest.map_structure( lambda ta: ta.read(next_time), self._auxiliary_input_tas) if indices is not None: auxiliary_inputs = tf.gather_nd( auxiliary_inputs, indices) return nest.map_structure( lambda x, y: tf.concat((x, y), -1), outputs_, auxiliary_inputs) if self._next_input_layer is None: return tf.where( sample_ids, maybe_concatenate_auxiliary_inputs(outputs), base_next_inputs) where_sampling = tf.cast( tf.where(sample_ids), tf.int32) where_not_sampling = tf.cast( tf.where(tf.logical_not(sample_ids)), tf.int32) outputs_sampling = tf.gather_nd(outputs, where_sampling) inputs_not_sampling = tf.gather_nd(base_next_inputs, where_not_sampling) sampled_next_inputs = maybe_concatenate_auxiliary_inputs( self._next_input_layer(outputs_sampling), where_sampling) base_shape = tf.shape(base_next_inputs) return (tf.scatter_nd(indices=where_sampling, updates=sampled_next_inputs, shape=base_shape) + tf.scatter_nd(indices=where_not_sampling, updates=inputs_not_sampling, shape=base_shape)) all_finished = tf.reduce_all(finished) next_inputs = tf.cond( all_finished, lambda: base_next_inputs, maybe_sample) return (finished, next_inputs, state)
def decode(self, enc_outputs, enc_final_state): with tf.variable_scope(self.decoder.scope): def condition(time, all_outputs: tf.TensorArray, inputs, states): def check_outputs_ends(): def has_end_word(t): return tf.reduce_any(tf.equal(t, ANSWER_MAX)) output_label = tf.arg_max(all_outputs.stack(), 2) output_label = tf.Print(output_label, [output_label], "Output Labels: ") # The outputs are time-major, which means time is the first # dimension. Here I need to check whether all the generated # answers are ends with "</s>", so we need to transpose it # to batch-major. Because `map_fn` only map function by the # first dimension. batch_major_outputs = tf.transpose(output_label, (1, 0)) all_outputs_ends = tf.reduce_all(tf.map_fn(has_end_word, batch_major_outputs, dtype=tf.bool)) return all_outputs_ends # If the TensorArray has 0 size, stack() will trigger error, # so I have to use condition function to check whether the # size is 0. all_ends = tf.cond(tf.equal(all_outputs.size(), 0), lambda: tf.constant(False, tf.bool), check_outputs_ends) condition_result = tf.logical_and(tf.logical_not(all_ends), tf.less(time, ANSWER_MAX)) return condition_result def body(time, all_outputs, inputs, state): dec_outputs, dec_state, output_logits, next_input = self.decoder.step(inputs, state) all_outputs = all_outputs.write(time, output_logits) return time + 1, all_outputs, next_input, dec_state output_ta = tensor_array_ops.TensorArray(dtype=tf.float32, size=0, dynamic_size=True, element_shape=(None, config.DEC_VOCAB), clear_after_read=False) # with time-major data input, the batch size is the second dimension batch_size = tf.shape(enc_outputs)[1] zero_input = tf.ones(tf.expand_dims(batch_size, axis=0), dtype=tf.int32) * ANSWER_START res = control_flow_ops.while_loop( condition, body, loop_vars=[0, output_ta, self.decoder.zero_input(zero_input), enc_final_state], ) final_outputs = res[1].stack() final_outputs = tf.Print(final_outputs, [final_outputs], "Final Output: ") final_state = res[3] return final_outputs, final_state
def naive_decoder(cell, enc_states, targets, start_token, end_token, feed_previous=True, training=True, scope='naive_decoder.0'): init_state = enc_states[-1] timesteps = tf.shape(enc_states)[0] # targets time major targets_tm = tf.transpose(targets, [1,0,2]) states = tf.TensorArray(dtype=tf.float32, size=timesteps+1, name='states', clear_after_read=False) outputs = tf.TensorArray(dtype=tf.float32, size=timesteps+1, name='outputs', clear_after_read=False) def step(i, states, outputs): # run one step # read from TensorArray (states) state_prev = states.read(i) if is_lstm(cell): # previous state <tensor> -> <LSTMStateTuple> c, h = tf.unstack(state_prev) state_prev = rnn.LSTMStateTuple(c,h) if feed_previous: input_ = outputs.read(i) else: input_ = targets_tm[i] output, state = cell(input_, state_prev) # add state, output to list states = states.write(i+1, state) outputs = outputs.write(i+1, output) i = tf.add(i,1) return i, states, outputs with tf.variable_scope(scope): # initial state states = states.write(0, init_state) # initial input outputs = outputs.write(0, start_token) i = tf.constant(0) # Stop loop condition if training: c = lambda x, y, z : tf.less(x, timesteps) else: c = lambda x, y, z : tf.reduce_all(tf.not_equal(tf.argmax(z.read(x), axis=-1), end_token)) # body b = lambda x, y, z : step(x, y, z) # execution _, fstates, foutputs = tf.while_loop(c,b, [i, states, outputs]) return foutputs.stack()[1:] # add states; but why?
def build_acquisition(self, Xcand): outdim = tf.shape(self.data[1])[1] num_cells = tf.shape(self.pareto.bounds.lb)[0] N = tf.shape(Xcand)[0] # Extended Pareto front pf_ext = tf.concat([-np.inf * tf.ones([1, outdim], dtype=float_type), self.pareto.front, self.reference], 0) # Predictions for candidates, concatenate columns preds = [m.build_predict(Xcand) for m in self.models] candidate_mean, candidate_var = (tf.concat(moment, 1) for moment in zip(*preds)) candidate_var = tf.maximum(candidate_var, stability) # avoid zeros # Calculate the cdf's for all candidates for every predictive distribution in the data points normal = tf.contrib.distributions.Normal(candidate_mean, tf.sqrt(candidate_var)) Phi = tf.transpose(normal.cdf(tf.expand_dims(pf_ext, 1)), [1, 0, 2]) # N x pf_ext_size x outdim # tf.gather_nd indices for bound points col_idx = tf.tile(tf.range(outdim), (num_cells,)) ub_idx = tf.stack((tf.reshape(self.pareto.bounds.ub, [-1]), col_idx), axis=1) # (num_cells*outdim x 2) lb_idx = tf.stack((tf.reshape(self.pareto.bounds.lb, [-1]), col_idx), axis=1) # (num_cells*outdim x 2) # Calculate PoI P1 = tf.transpose(tf.gather_nd(tf.transpose(Phi, perm=[1, 2, 0]), ub_idx)) # N x num_cell*outdim P2 = tf.transpose(tf.gather_nd(tf.transpose(Phi, perm=[1, 2, 0]), lb_idx)) # N x num_cell*outdim P = tf.reshape(P1 - P2, [N, num_cells, outdim]) PoI = tf.reduce_sum(tf.reduce_prod(P, axis=2), axis=1, keep_dims=True) # N x 1 # Calculate Hypervolume contribution of points Y ub_points = tf.reshape(tf.gather_nd(pf_ext, ub_idx), [num_cells, outdim]) lb_points = tf.reshape(tf.gather_nd(pf_ext, lb_idx), [num_cells, outdim]) splus_valid = tf.reduce_all(tf.tile(tf.expand_dims(ub_points, 1), [1, N, 1]) > candidate_mean, axis=2) # num_cells x N splus_idx = tf.expand_dims(tf.cast(splus_valid, dtype=float_type), -1) # num_cells x N x 1 splus_lb = tf.tile(tf.expand_dims(lb_points, 1), [1, N, 1]) # num_cells x N x outdim splus_lb = tf.maximum(splus_lb, candidate_mean) # num_cells x N x outdim splus_ub = tf.tile(tf.expand_dims(ub_points, 1), [1, N, 1]) # num_cells x N x outdim splus = tf.concat([splus_idx, splus_ub - splus_lb], axis=2) # num_cells x N x (outdim+1) Hv = tf.transpose(tf.reduce_sum(tf.reduce_prod(splus, axis=2), axis=0, keep_dims=True)) # N x 1 # return HvPoI return tf.multiply(Hv, PoI)
