我们从Python开源项目中，提取了以下37个代码示例，用于说明如何使用tensorflow.sparse_tensor_dense_matmul()。
def test_custom_repr_graph(self): # Define a custom representation function graph def build_tanh_representation_graph(tf_features, n_components, n_features, node_name_ending): tf_tanh_weights = tf.Variable(tf.random_normal([n_features, n_components], stddev=.5), name='tanh_weights_%s' % node_name_ending) tf_repr = tf.nn.tanh(tf.sparse_tensor_dense_matmul(tf_features, tf_tanh_weights)) # Return repr layer and variables return tf_repr, [tf_tanh_weights] # Build a model with the custom representation function model = TensorRec(user_repr_graph=build_tanh_representation_graph, item_repr_graph=build_tanh_representation_graph) self.assertIsNotNone(model)
def dot(x, y): '''Multiplies 2 tensors. When attempting to multiply a ND tensor with a ND tensor, reproduces the Theano behavior (e.g. (2, 3).(4, 3, 5) = (2, 4, 5)) ''' if ndim(x) is not None and (ndim(x) > 2 or ndim(y) > 2): x_shape = (-1,) + int_shape(x)[1:] y_shape = int_shape(y) y_permute_dim = list(range(ndim(y))) y_permute_dim = [y_permute_dim.pop(-2)] + y_permute_dim xt = tf.reshape(x, [-1, x_shape[-1]]) yt = tf.reshape(tf.transpose(y, perm=y_permute_dim), [y_shape[-2], -1]) return tf.reshape(tf.matmul(xt, yt), x_shape[:-1] + y_shape[:-2] + y_shape[-1:]) if is_sparse(x): out = tf.sparse_tensor_dense_matmul(x, y) else: out = tf.matmul(x, y) return out
def DizzyLayerV2(X, rot_list, n): n_prime = int(n*(n-1)/2) thetas = tf.Variable(tf.random_uniform([n_prime, 1], 0, 2*math.pi), name="thetas") results = [X] k = 0 for sublist in rot_list: indices = [] values = [] for (a, b) in sublist: c = tf.cos(thetas[k]) s = tf.sin(thetas[k]) indices = indices + [[a, a], [a, b], [b, a], [b, b]] values = values + [c, s, -s, c] k += 1 shape = [n, n] v = tf.pack(tf.squeeze(values)) R = tf.SparseTensor(indices, v, shape) results.append(tf.sparse_tensor_dense_matmul(R, results[-1])) return results[-1]
def __init__(self, model_inputs, rouge_scorer, hps): self.word_embedding = tf.get_variable( "word_embedding", [hps.vocab_size, hps.word_embedding_size]) self.article_inputs = tf.nn.embedding_lookup(self.word_embedding, model_inputs.sliced_article) self.stopworded_abstract_bag = tf.transpose( tf.sparse_tensor_dense_matmul( rouge_scorer.stem_projector_stopworded, tf.to_float(model_inputs.abstract_bag), adjoint_a=True, adjoint_b=True)) with tf.variable_scope("article_enc"): article_outs = shared_util.deep_birnn(hps, self.article_inputs, model_inputs.article_len) self.article_feats = shared_util.relu(article_outs, hps.hidden_size) with tf.variable_scope("scorer"): self.word_logits = tf.reshape( shared_util.linear(self.article_feats, 1), [hps.batch_size, -1])
def relu_representation_graph(tf_features, n_components, n_features, node_name_ending): relu_size = 4 * n_components # Create variable nodes tf_relu_weights = tf.Variable(tf.random_normal([n_features, relu_size], stddev=.5), name='relu_weights_%s' % node_name_ending) tf_relu_biases = tf.Variable(tf.zeros([1, relu_size]), name='relu_biases_%s' % node_name_ending) tf_linear_weights = tf.Variable(tf.random_normal([relu_size, n_components], stddev=.5), name='linear_weights_%s' % node_name_ending) # Create ReLU layer tf_relu = tf.nn.relu(tf.add(tf.sparse_tensor_dense_matmul(tf_features, tf_relu_weights), tf_relu_biases)) tf_repr = tf.matmul(tf_relu, tf_linear_weights) # Return repr layer and variables return tf_repr, [tf_relu_weights, tf_linear_weights, tf_relu_biases]
def call(self, x, mask=None): #sys.stderr.write("sparse fuylly connected layer input data %s type:%s\n" % (x.name, K.type(x))) #sys.stderr.write("sparse fuylly connected layer weight type:%s\n" % (K.type(self.W))) print(str(K.ndim(x))) return self.activation(tf.sparse_tensor_dense_matmul(x, self.W) + self.b)
def _call(self, inputs): x = inputs x = tf.nn.dropout(x, 1-self.dropout) x = tf.matmul(x, self.vars['weights']) x = tf.sparse_tensor_dense_matmul(self.adj, x) outputs = self.act(x) return outputs
def _call(self, inputs): x = inputs x = dropout_sparse(x, 1-self.dropout, self.features_nonzero) x = tf.sparse_tensor_dense_matmul(x, self.vars['weights']) x = tf.sparse_tensor_dense_matmul(self.adj, x) outputs = self.act(x) return outputs
def dot(x, y, sparse=False): if sparse: return tf.sparse_tensor_dense_matmul(x, y) else: return tf.matmul(x, y)
def batch_sparse_tensor_dense_matmul(sp_a, b): '''Multiply sp_a by every row of b.''' return tf.map_fn(lambda b_i: tf.sparse_tensor_dense_matmul(sp_a, b_i), b)
def __init__(self, length, k, lam): with tf.variable_scope(type(self).__name__): self.length = length self.k = k self.lam = lam self.D = tf_get_delta(get_sparse_penalty_matrix((length,)), k) self.samples = tf.placeholder(tf.int32, [None]) self.y = tf.one_hot(self.samples, length) self.q = tf.Variable([1.]*length) self.yhat = tf.nn.softmax(self.q, name='yhat') self.acc = tf.reduce_mean(-tf.reduce_sum(self.y * tf.log(tf.clip_by_value(self.yhat, 1e-10, 1.0)) + (1 - self.y) * tf.log(tf.clip_by_value(1 - self.yhat, 1e-10, 1.0)), axis=[1])) self.reg = tf.reduce_sum(tf.abs(tf.sparse_tensor_dense_matmul(self.D, tf.expand_dims(self.q,-1)))) self.loss = self.acc + self.lam * self.reg
def collect_messages(self, messages): e_forward_mtr = self.graph_representation.get_entity_forward_v_by_m(normalized=True) e_backward_mtr = self.graph_representation.get_entity_backward_v_by_m(normalized=True) r_forward_mtr = self.graph_representation.get_relation_forward_v_by_m(normalized=True) r_backward_mtr = self.graph_representation.get_relation_backward_v_by_m(normalized=True) collected_e_messages = tf.sparse_tensor_dense_matmul(r_forward_mtr, messages[2]) collected_e_messages += tf.sparse_tensor_dense_matmul(r_backward_mtr, messages[3]) collected_r_messages = tf.sparse_tensor_dense_matmul(e_forward_mtr, messages[0]) collected_r_messages += tf.sparse_tensor_dense_matmul(e_backward_mtr, messages[1]) return collected_e_messages, collected_r_messages
def combine_messages(self, forward_messages, backward_messages, self_loop_messages, previous_code, mode='train'): mtr_f = self.get_graph().forward_incidence_matrix(normalization=('global', 'recalculated')) mtr_b = self.get_graph().backward_incidence_matrix(normalization=('global', 'recalculated')) collected_messages_f = tf.sparse_tensor_dense_matmul(mtr_f, forward_messages) collected_messages_b = tf.sparse_tensor_dense_matmul(mtr_b, backward_messages) new_embedding = self_loop_messages + collected_messages_f + collected_messages_b + self.b if self.use_nonlinearity: new_embedding = tf.nn.relu(new_embedding) return new_embedding
def combine_messages(self, forward_messages, backward_messages, self_loop_messages, previous_code, mode='train'): mtr_f = self.get_graph().forward_incidence_matrix(normalization=('global', 'recalculated')) mtr_b = self.get_graph().backward_incidence_matrix(normalization=('global', 'recalculated')) collected_messages_f = tf.sparse_tensor_dense_matmul(mtr_f, forward_messages) collected_messages_b = tf.sparse_tensor_dense_matmul(mtr_b, backward_messages) updated_vertex_embeddings = collected_messages_f + collected_messages_b if self.use_nonlinearity: activated = tf.nn.relu(updated_vertex_embeddings) else: activated = updated_vertex_embeddings return activated
def combine_messages(self, forward_messages, backward_messages, self_loop_messages, previous_code, mode='train'): mtr_f = self.get_graph().forward_incidence_matrix(normalization=('global', 'recalculated')) mtr_b = self.get_graph().backward_incidence_matrix(normalization=('global', 'recalculated')) collected_messages_f = tf.sparse_tensor_dense_matmul(mtr_f, forward_messages) collected_messages_b = tf.sparse_tensor_dense_matmul(mtr_b, backward_messages) updated_vertex_embeddings = collected_messages_f + collected_messages_b if self.use_nonlinearity: activated = tf.nn.relu(updated_vertex_embeddings + self_loop_messages) else: activated = updated_vertex_embeddings + self_loop_messages return activated
def doRotations(X, rotations): # print('number of rotations in doRotations: %d' % len(rotations)) with vs.variable_scope("Do_Rotations"): for sparse_rot in rotations: X = tf.sparse_tensor_dense_matmul(sparse_rot, X) return X
def doRotationsSigmas(X, rotations, num_units): with vs.variable_scope("Do_Rotations"): sigma = vs.get_variable( "Sigma", [num_units,1], dtype=tf.float32, initializer=init_ops.constant_initializer(value=1.0, dtype=tf.float32)) sigma_spot = int(len(rotations)/2) for i, sparse_rot in enumerate(rotations): if i == sigma_spot: X = X * sigma X = tf.sparse_tensor_dense_matmul(sparse_rot, X) return X, sigma
def rotationTransform(X, n, scope, num_rots=None): num_rots = num_rots or (n-1) n_prime = int(n*(n-1)//2*num_rots/(n-1)) outputs = [] with vs.variable_scope(scope or "RotationTransform"): for i, (name, x) in enumerate(X): (indices, values_idxs) = rotationPreprocess(n, num_rots) thetas = vs.get_variable(initializer=tf.random_uniform([n_prime, 1], 0, 2*math.pi), name="Thetas"+str(i)+name, dtype=tf.float32) cos = tf.cos(thetas) sin = tf.sin(thetas) nsin = tf.neg(sin) thetas_concat = tf.concat(0, [cos,sin,nsin]) gathered_values = tf.squeeze(tf.gather(thetas_concat, values_idxs)) shape = tf.constant([n, n], dtype=tf.int64) splt_values = tf.split(0, num_rots, gathered_values) splt_indices = tf.split(0, num_rots, indices) shape = tf.constant([n,n], dtype=tf.int64) for i in range(num_rots): curr_indices = splt_indices[i] curr_values = splt_values[i] sparse_rot = tf.SparseTensor(indices=curr_indices, values=curr_values, shape=shape) x = tf.sparse_tensor_dense_matmul(sparse_rot, x) outputs.append(x) return outputs
def chebyshev5(self, x, L, Fout, K): N, M, Fin = x.get_shape() N, M, Fin = int(N), int(M), int(Fin) # Rescale Laplacian and store as a TF sparse tensor. Copy to not modify the shared L. L = scipy.sparse.csr_matrix(L) L = graph.rescale_L(L, lmax=2) L = L.tocoo() indices = np.column_stack((L.row, L.col)) L = tf.SparseTensor(indices, L.data, L.shape) L = tf.sparse_reorder(L) # Transform to Chebyshev basis x0 = tf.transpose(x, perm=[1, 2, 0]) # M x Fin x N x0 = tf.reshape(x0, [M, Fin*N]) # M x Fin*N x = tf.expand_dims(x0, 0) # 1 x M x Fin*N def concat(x, x_): x_ = tf.expand_dims(x_, 0) # 1 x M x Fin*N return tf.concat(0, [x, x_]) # K x M x Fin*N if K > 1: x1 = tf.sparse_tensor_dense_matmul(L, x0) x = concat(x, x1) for k in range(2, K): x2 = 2 * tf.sparse_tensor_dense_matmul(L, x1) - x0 # M x Fin*N x = concat(x, x2) x0, x1 = x1, x2 x = tf.reshape(x, [K, M, Fin, N]) # K x M x Fin x N x = tf.transpose(x, perm=[3,1,2,0]) # N x M x Fin x K x = tf.reshape(x, [N*M, Fin*K]) # N*M x Fin*K # Filter: Fin*Fout filters of order K, i.e. one filterbank per feature pair. W = self._weight_variable([Fin*K, Fout], regularization=False) x = tf.matmul(x, W) # N*M x Fout return tf.reshape(x, [N, M, Fout]) # N x M x Fout
def chebyshev5(self, x, L, Fout, K, regularization=False): N, M, Fin = x.get_shape() N, M, Fin = int(N), int(M), int(Fin) # Rescale Laplacian and store as a TF sparse tensor. Copy to not modify the shared L. L = scipy.sparse.csr_matrix(L) L = graph.rescale_L(L, lmax=2) L = L.tocoo() indices = np.column_stack((L.row, L.col)) L = tf.SparseTensor(indices, L.data, L.shape) L = tf.sparse_reorder(L) # Transform to Chebyshev basis x0 = tf.transpose(x, perm=[1, 2, 0]) # M x Fin x N x0 = tf.reshape(x0, [M, Fin*N]) # M x Fin*N x = tf.expand_dims(x0, 0) # 1 x M x Fin*N def concat(x, x_): x_ = tf.expand_dims(x_, 0) # 1 x M x Fin*N return tf.concat(0, [x, x_]) # K x M x Fin*N if K > 1: x1 = tf.sparse_tensor_dense_matmul(L, x0) x = concat(x, x1) for k in range(2, K): x2 = 2 * tf.sparse_tensor_dense_matmul(L, x1) - x0 # M x Fin*N x = concat(x, x2) x0, x1 = x1, x2 x = tf.reshape(x, [K, M, Fin, N]) # K x M x Fin x N x = tf.transpose(x, perm=[3,1,2,0]) # N x M x Fin x K x = tf.reshape(x, [N*M, Fin*K]) # N*M x Fin*K # Filter: Fin*Fout filters of order K, i.e. one filterbank per feature pair. W = self._weight_variable([Fin*K, Fout], regularization=regularization) x = tf.matmul(x, W) # N*M x Fout return tf.reshape(x, [N, M, Fout]) # N x M x Fout
def graph_convolution_layer(self, node_emb, scope, edge_emb=None): # Path to hyperparameters and configuration settings for the graph convolutional layers prefix = 'model/graph_conv_layers' with tf.variable_scope(scope, reuse=not self.is_training): # Compute the extended node embedding as the concatenation of the original node embedding and the sum of # the node embeddings of all distance-one neighbors in the graph. ext_node_emb = tf.concat([node_emb, tf.sparse_tensor_dense_matmul(self.input['adj_mat'], node_emb)], axis=1) # If edge labels are to be considered by the model, concatenate as well the (pre-computed) sum of the # feature vectors labelling all edges connected to each node if edge_emb is not None: ext_node_emb = tf.concat([ext_node_emb, edge_emb], axis=1) # Compute output by applying a fully connected layer to the extended node embedding out = tf.contrib.layers.fully_connected(inputs=ext_node_emb, num_outputs=self.getitem('config', 'num_outputs', prefix), activation_fn=self.string_to_tf_act(self.getitem('config', 'activation_fn', prefix)), weights_initializer=self.weights_initializer_graph_conv, weights_regularizer=self.weights_regularizer_graph_conv, biases_initializer=tf.constant_initializer(0.1, tf.float32), normalizer_fn=self.normalizer_fn_graph_conv, normalizer_params=self.normalizer_params_graph_conv, trainable=self.getitem('config', 'trainable', prefix)) # Apply dropout (if necessary). Alternatively, could have also forced keep_prob to 1.0 when is_training is # False if self.is_training: out = tf.nn.dropout(out, self.getitem('config', 'keep_prob', prefix)) return out
def build_graph_fingerprint(self): # Total number of graph convolution layers n_layers = self.getitem('config', 'model/graph_conv_layers/n_layers') # Create output dictionaries for graph convolutional layers and fingerprint output layers self.output['graph_conv_layers'] = {} self.output['fingerprint_output_layers'] = {} # Input node embeddings node_emb = self.input['node_features'] # Pre-compute the sum of the feature vectors labelling all edges connected to each node (if necessary) self.output['graph_conv_layers']['edge_emb'] = None if self.getitem('config', 'model/input/edge_features/use') and self.num_edge_features > 0: self.output['graph_conv_layers']['edge_emb'] = tf.sparse_tensor_dense_matmul(self.input['inc_mat'], self.input['edge_features']) # Compute node and graph level fingerprints for the input layer graph_fp, node_fp = self.output_embedding_layer(node_emb, 'output_embedding_layer_0') # List of node-level embeddings per layer (output of graph convolutional layers), node-level fingerprints per # layer (per-node output of fingerprint output layers) and graph-level fingerprints per layer (total output of # fingerprint output layers) self.output['graph_conv_layers']['node_emb'] = [node_emb] self.output['fingerprint_output_layers']['node_fp'] = [node_fp] self.output['fingerprint_output_layers']['graph_fp'] = [graph_fp] # Create all graph convolutional layers and their respective fingerprint output layers for layer_idx in xrange(1, n_layers+1): node_emb = self.graph_convolution_layer(node_emb=self.output['graph_conv_layers']['node_emb'][-1], scope='graph_conv_layer_%d' % layer_idx, edge_emb=self.output['graph_conv_layers']['edge_emb']) graph_fp, node_fp = self.output_embedding_layer(node_emb=self.output['graph_conv_layers']['node_emb'][-1], scope='output_embedding_layer_%d' % layer_idx) # Append outputs to lists self.output['graph_conv_layers']['node_emb'].append(node_emb) self.output['fingerprint_output_layers']['node_fp'].append(node_fp) self.output['fingerprint_output_layers']['graph_fp'].append(graph_fp) # Obtain graph fingerprint as the sum of the graph activations across all layers self.output['fingerprint_output_layers']['fingerprint'] = tf.add_n(self.output['fingerprint_output_layers']['graph_fp'])
def get_rouge_recall_suff_stats(self, pred_counts, gold_counts): """Get overlapping predicted counts and gold counts for pair of bags.""" # Map words to their stems. pred_stems = tf.transpose( tf.sparse_tensor_dense_matmul( self.stem_projector_stopworded, pred_counts, adjoint_a=True, adjoint_b=True)) # <UNK> tokens count as always missing from predicted counts but not # from gold counts to avoid overly optimistic evaluation. gold_stems = tf.transpose( tf.sparse_tensor_dense_matmul( self.stem_projector_stopworded_keep_unk, gold_counts, adjoint_a=True, adjoint_b=True)) # Only count max of 1 point for each overlapping word type pred_stems = tf.minimum(1.0, pred_stems) gold_stems = tf.minimum(1.0, gold_stems) overlaps = tf.reduce_sum(tf.minimum(pred_stems, gold_stems), 1) gold_counts = tf.reduce_sum(gold_stems, 1) return overlaps, gold_counts
def __init__(self, input_dim=None, output_dim=1, init_path=None, opt_algo='gd', learning_rate=1e-2, l2_weight=0, random_seed=None): Model.__init__(self) init_vars = [('w', [input_dim, output_dim], 'xavier', dtype), ('b', [output_dim], 'zero', dtype)] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = tf.sparse_placeholder(dtype) self.y = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w = self.vars['w'] b = self.vars['b'] xw = tf.sparse_tensor_dense_matmul(self.X, w) logits = tf.reshape(xw + b, [-1]) self.y_prob = tf.sigmoid(logits) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y, logits=logits)) + \ l2_weight * tf.nn.l2_loss(xw) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, input_dim=None, output_dim=1, factor_order=10, init_path=None, opt_algo='gd', learning_rate=1e-2, l2_w=0, l2_v=0, random_seed=None): Model.__init__(self) init_vars = [('w', [input_dim, output_dim], 'xavier', dtype), ('v', [input_dim, factor_order], 'xavier', dtype), ('b', [output_dim], 'zero', dtype)] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = tf.sparse_placeholder(dtype) self.y = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w = self.vars['w'] v = self.vars['v'] b = self.vars['b'] X_square = tf.SparseTensor(self.X.indices, tf.square(self.X.values), tf.to_int64(tf.shape(self.X))) xv = tf.square(tf.sparse_tensor_dense_matmul(self.X, v)) p = 0.5 * tf.reshape( tf.reduce_sum(xv - tf.sparse_tensor_dense_matmul(X_square, tf.square(v)), 1), [-1, output_dim]) xw = tf.sparse_tensor_dense_matmul(self.X, w) logits = tf.reshape(xw + b + p, [-1]) self.y_prob = tf.sigmoid(logits) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=self.y)) + \ l2_w * tf.nn.l2_loss(xw) + \ l2_v * tf.nn.l2_loss(xv) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def _train_fprop(self, state_below): idx, val = state_below X = tf.SparseTensor(tf.cast(idx, 'int64'), val, shape=[self.batchsize, self.prev_dim]) X_order = tf.sparse_reorder(X) XW = tf.sparse_tensor_dense_matmul(X_order, self.W, adjoint_a=False, adjoint_b=False) return tf.add(XW, self.b)
def linear_representation_graph(tf_features, n_components, n_features, node_name_ending): # Create variable nodes tf_linear_weights = tf.Variable(tf.random_normal([n_features, n_components], stddev=.5), name='linear_weights_%s' % node_name_ending) tf_repr = tf.sparse_tensor_dense_matmul(tf_features, tf_linear_weights) # Return repr layer and variables return tf_repr, [tf_linear_weights]
def calc_log_loss(self, Pairwise, Question, Answer, Review, TermtoTermR, TermtoTermP, Question_I, Answer_I, Review_I): #print 'Doing for item %d'%(i) shape1 = tf.shape(Pairwise) shape2 = tf.shape(Answer) nq = shape1[0] nr = shape1[1] na = shape2[1] pairwise = tf.reshape(Pairwise, [-1, self.PairwiseDim]) pairwise = tf.reshape(tf.matmul(pairwise, self.theta), [nq, nr]) termTotermR = tf.sparse_reshape(TermtoTermR, [-1, self.V]) termTotermR = tf.reshape(tf.sparse_tensor_dense_matmul(termTotermR, self.RelvPar), [nq, nr]) QProj = tf.sparse_tensor_dense_matmul(Question_I, self.A) RProjR = tf.sparse_tensor_dense_matmul(Review_I, self.B) BilinearR = tf.matmul(QProj, tf.transpose(RProjR)) Relevance = tf.nn.softmax(pairwise + termTotermR + BilinearR) termTotermP = tf.sparse_reshape(TermtoTermP, [-1, self.V]) termTotermP = tf.reshape(tf.sparse_tensor_dense_matmul(termTotermP, self.PredPar), [nq, na, nr]) AProj = tf.sparse_tensor_dense_matmul(tf.sparse_reshape(Answer_I, [-1, self.V]), self.X) RProjP = tf.sparse_tensor_dense_matmul(Review_I, self.Y) BilinearP = tf.reshape(tf.matmul(AProj, tf.transpose(RProjP)), [nq, na, nr]) Prediction = BilinearP + termTotermP Prediction = tf.expand_dims(Prediction[:,0,:], 1) - Prediction Prediction = Prediction[:,1:,:] Prediction = tf.sigmoid(Prediction) MoE = tf.reduce_sum(tf.multiply(Prediction, tf.expand_dims(Relevance, axis = 1)), axis = 2) accuracy_count = tf.cast(tf.shape(tf.where(MoE > 0.5))[0], tf.float64) count = nq * na log_likelihood = tf.reduce_sum(tf.log(MoE)) R1 = tf.reduce_sum(tf.square(self.A)) + tf.reduce_sum(tf.square(self.B)) R2 = tf.reduce_sum(tf.square(self.X)) + tf.reduce_sum(tf.square(self.Y)) log_likelihood -= self.Lambda * (R1 + R2) return -1*log_likelihood, MoE, Relevance
def matmul(a, b, benchmark=True, name='mul'): # TODO maybe put inside dot """ Interface function for matmul that works also with sparse tensors :param a: :param b: :param benchmark: :param name: :return: """ a_is_sparse = isinstance(a, tf.SparseTensor) with tf.name_scope(name): if a_is_sparse: mul = wsr(tf.matmul(tf.sparse_tensor_to_dense(a, default_value=0.), b)) if benchmark: mul_ops = [wsr(tf.sparse_tensor_dense_matmul(a, b)), mul, # others ? # wsr(tf.nn.embedding_lookup_sparse()) # I couldn't figure out how this works...... ] def _avg_exe_times(op, repetitions): from time import time ex_times = [] for _ in range(repetitions): st = time() op.eval() ex_times.append(time() - st) return np.mean(ex_times[1:]), np.max(ex_times), np.min(ex_times) with tf.Session(config=CONFIG_GPU_GROWTH).as_default(): tf.global_variables_initializer().run() # TODO here should only initialize necessary variable # (downstream in the computation graph) statistics = {op: _avg_exe_times(op, repetitions=4) for op in mul_ops} [print(k, v) for k, v in statistics.items()] mul = sorted(statistics.items(), key=lambda v: v[1][0])[0][0] # returns best one w.r.t. avg exe time print(mul, 'selected') else: mul = wsr(tf.matmul(a, b)) return mul # Define a context manager to suppress stdout and stderr.
def affine(input_tensor, output_size, bias=True, bias_start=0.0, input_size=None, scope="affine", sparse_input=False): """Add an affine transformation of `input_tensor` to the current graph. Note: This op is loosely based on tensorflow.python.ops.rnn_cell.linear. An affine transformation is a linear transformation with a shift, `t = tf.matmul(input_tensor, W) + b`. Parameters ---------- input_tensor : tensorflow Tensor object, rank 2 Input tensor to be transformed. output_size : int The output will be size [a, output_size] where `input_tensor` has shape [a, b]. bias : bool, optional If True, apply a bias to the transformation. If False, only a linear transformation is applied (i.e., `t = tf.matmul(W, input_tensor)`). bias_start : float, optional The initial value for the bias `b`. input_size : int, optional Second dimension of the rank 2 input tensor. Required for sparse input tensors. sparse_input : bool, optional Set to True if `input_tensor` is sparse. Returns ------- t : tensorflow tensor object The affine transformation of `input_tensor`. """ # The input size is needed for sparse matrices. if input_size is None: input_size = input_tensor.get_shape().as_list()[1] with tf.variable_scope(scope): W_0 = tf.get_variable( "weights0", [input_size, output_size]) # If the input is sparse, then use a special matmul routine. matmul = tf.sparse_tensor_dense_matmul if sparse_input else tf.matmul t = matmul(input_tensor, W_0) if bias: b_0 = tf.get_variable( "bias0", [output_size], initializer=tf.constant_initializer(bias_start)) t = tf.add(t, b_0) return t
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) node_in = num_inputs * embed_size for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) l = xw for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] print(l.shape, wi.shape, bi.shape) l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, filter_sizes=None, layer_acts=None, drop_out=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) init_vars.append(('f1', [embed_size, filter_sizes[0], 1, 2], 'xavier', dtype)) init_vars.append(('f2', [embed_size, filter_sizes[1], 2, 2], 'xavier', dtype)) init_vars.append(('w1', [2 * 3 * embed_size, 1], 'xavier', dtype)) init_vars.append(('b1', [1], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) l = xw l = tf.transpose(tf.reshape(l, [-1, num_inputs, embed_size, 1]), [0, 2, 1, 3]) f1 = self.vars['f1'] l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), int(num_inputs / 2)), [0, 1, 3, 2]) f2 = self.vars['f2'] l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), 3), [0, 1, 3, 2]) l = tf.nn.dropout( utils.activate( tf.reshape(l, [-1, embed_size * 3 * 2]), layer_acts[0]), self.layer_keeps[0]) w1 = self.vars['w1'] b1 = self.vars['b1'] l = tf.matmul(l, w1) + b1 l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
