我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用torch.ones()。
def calc_gradient_penalty(self, netD, real_data, fake_data): alpha = torch.rand(1, 1) alpha = alpha.expand(real_data.size()) alpha = alpha.cuda() interpolates = alpha * real_data + ((1 - alpha) * fake_data) interpolates = interpolates.cuda() interpolates = Variable(interpolates, requires_grad=True) disc_interpolates = netD.forward(interpolates) gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates, grad_outputs=torch.ones(disc_interpolates.size()).cuda(), create_graph=True, retain_graph=True, only_inputs=True)[0] gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * self.LAMBDA return gradient_penalty
def bn_hat_z_layers(self, hat_z_layers, z_pre_layers): # TODO: Calculate batchnorm using GPU Tensors. assert len(hat_z_layers) == len(z_pre_layers) hat_z_layers_normalized = [] for i, (hat_z, z_pre) in enumerate(zip(hat_z_layers, z_pre_layers)): if self.use_cuda: ones = Variable(torch.ones(z_pre.size()[0], 1).cuda()) else: ones = Variable(torch.ones(z_pre.size()[0], 1)) mean = torch.mean(z_pre, 0) noise_var = np.random.normal(loc=0.0, scale=1 - 1e-10, size=z_pre.size()) if self.use_cuda: var = np.var(z_pre.data.cpu().numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1]) else: var = np.var(z_pre.data.numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1]) var = Variable(torch.FloatTensor(var)) if self.use_cuda: hat_z = hat_z.cpu() ones = ones.cpu() mean = mean.cpu() hat_z_normalized = torch.div(hat_z - ones.mm(mean), ones.mm(torch.sqrt(var + 1e-10))) if self.use_cuda: hat_z_normalized = hat_z_normalized.cuda() hat_z_layers_normalized.append(hat_z_normalized) return hat_z_layers_normalized
def __init__(self, nOutput, eps=1e-5, momentum=0.1, affine=True): super(BatchNormalization, self).__init__() assert nOutput != 0 self.affine = affine self.eps = eps self.train = True self.momentum = momentum self.running_mean = torch.zeros(nOutput) self.running_var = torch.ones(nOutput) self.save_mean = None self.save_std = None if self.affine: self.weight = torch.Tensor(nOutput) self.bias = torch.Tensor(nOutput) self.gradWeight = torch.Tensor(nOutput) self.gradBias = torch.Tensor(nOutput) self.reset() else: self.weight = None self.bias = None self.gradWeight = None self.gradBias = None
def _test_pool(self): def do_test(): p = mp.Pool(2) for proc in p._pool: lc.check_pid(proc.pid) buffers = (torch.zeros(2, 2) for i in range(4)) results = p.map(simple_pool_fill, buffers, 1) for r in results: self.assertEqual(r, torch.ones(2, 2) * 5, 0) self.assertEqual(len(results), 4) p.close() p.join() with leak_checker(self) as lc: do_test()
def test_gpu(self): compile_extension( name='gpulib', header=test_dir + '/ffi/src/cuda/cudalib.h', sources=[ test_dir + '/ffi/src/cuda/cudalib.c', ], with_cuda=True, verbose=False, ) import gpulib tensor = torch.ones(2, 2).float() gpulib.good_func(tensor, 2, 1.5) self.assertEqual(tensor, torch.ones(2, 2) * 2 + 1.5) ctensor = tensor.cuda().fill_(1) gpulib.cuda_func(ctensor, 2, 1.5) self.assertEqual(ctensor, torch.ones(2, 2) * 2 + 1.5) self.assertRaises(TypeError, lambda: gpulib.cuda_func(tensor, 2, 1.5)) self.assertRaises(TypeError, lambda: gpulib.cuda_func(ctensor.storage(), 2, 1.5))
def test_L1Penalty(self): weight = 1 m = nn.L1Penalty(weight, False, False) input = torch.rand(2,10).add_(-0.5) input[0][0] = 0 m.forward(input) grad = m.backward(input, torch.ones(input.size())) self.assertEqual(input.abs().sum() * weight, m.loss) true_grad = (input.gt(0).type_as(grad) + input.lt(0).type_as(grad).mul_(-1)).mul_(weight) self.assertEqual(true_grad, grad) # Check that these don't raise errors m.__repr__() str(m)
def test_all_any(self): def test(size): x = torch.ones(*size).byte() self.assertTrue(x.all()) self.assertTrue(x.any()) x[3] = 0 self.assertFalse(x.all()) self.assertTrue(x.any()) x.zero_() self.assertFalse(x.all()) self.assertFalse(x.any()) x.fill_(2) self.assertTrue(x.all()) self.assertTrue(x.any()) test((10,)) test((5, 5))
def test_accuracy_computation(self): accuracy = BooleanAccuracy() predictions = torch.Tensor([[0, 1], [2, 3], [4, 5], [6, 7]]) targets = torch.Tensor([[0, 1], [2, 2], [4, 5], [7, 7]]) accuracy(predictions, targets) assert accuracy.get_metric() == 2. / 4 mask = torch.ones(4, 2) mask[1, 1] = 0 accuracy(predictions, targets, mask) assert accuracy.get_metric() == 5. / 8 targets[1, 1] = 3 accuracy(predictions, targets) assert accuracy.get_metric() == 8. / 12 accuracy.reset() accuracy(predictions, targets) assert accuracy.get_metric() == 3. / 4
def setUp(self): super(TestEncoderBase, self).setUp() self.lstm = LSTM(bidirectional=True, num_layers=3, input_size=3, hidden_size=7, batch_first=True) self.encoder_base = _EncoderBase(stateful=True) tensor = Variable(torch.rand([5, 7, 3])) tensor[1, 6:, :] = 0 tensor[3, 2:, :] = 0 self.tensor = tensor mask = Variable(torch.ones(5, 7)) mask[1, 6:] = 0 mask[2, :] = 0 # <= completely masked mask[3, 2:] = 0 mask[4, :] = 0 # <= completely masked self.mask = mask self.batch_size = 5 self.num_valid = 3 sequence_lengths = get_lengths_from_binary_sequence_mask(mask) _, _, restoration_indices, sorting_indices = sort_batch_by_length(tensor, sequence_lengths) self.sorting_indices = sorting_indices self.restoration_indices = restoration_indices
def test_forward_works_even_with_empty_sequences(self): lstm = LSTM(bidirectional=True, num_layers=3, input_size=3, hidden_size=11, batch_first=True) encoder = PytorchSeq2VecWrapper(lstm) tensor = torch.autograd.Variable(torch.rand([5, 7, 3])) tensor[1, 6:, :] = 0 tensor[2, :, :] = 0 tensor[3, 2:, :] = 0 tensor[4, :, :] = 0 mask = torch.autograd.Variable(torch.ones(5, 7)) mask[1, 6:] = 0 mask[2, :] = 0 mask[3, 2:] = 0 mask[4, :] = 0 results = encoder(tensor, mask) for i in (0, 1, 3): assert not (results[i] == 0.).data.all() for i in (2, 4): assert (results[i] == 0.).data.all()
def test_forward_works_even_with_empty_sequences(self): lstm = LSTM(bidirectional=True, num_layers=3, input_size=3, hidden_size=7, batch_first=True) encoder = PytorchSeq2SeqWrapper(lstm) tensor = torch.autograd.Variable(torch.rand([5, 7, 3])) tensor[1, 6:, :] = 0 tensor[2, :, :] = 0 tensor[3, 2:, :] = 0 tensor[4, :, :] = 0 mask = torch.autograd.Variable(torch.ones(5, 7)) mask[1, 6:] = 0 mask[2, :] = 0 mask[3, 2:] = 0 mask[4, :] = 0 results = encoder(tensor, mask) for i in (0, 1, 3): assert not (results[i] == 0.).data.all() for i in (2, 4): assert (results[i] == 0.).data.all()
def test_forward_pulls_out_correct_tensor_with_sequence_lengths(self): lstm = LSTM(bidirectional=True, num_layers=3, input_size=3, hidden_size=7, batch_first=True) encoder = PytorchSeq2SeqWrapper(lstm) tensor = torch.rand([5, 7, 3]) tensor[1, 6:, :] = 0 tensor[2, 4:, :] = 0 tensor[3, 2:, :] = 0 tensor[4, 1:, :] = 0 mask = torch.ones(5, 7) mask[1, 6:] = 0 mask[2, 4:] = 0 mask[3, 2:] = 0 mask[4, 1:] = 0 input_tensor = Variable(tensor) mask = Variable(mask) sequence_lengths = get_lengths_from_binary_sequence_mask(mask) packed_sequence = pack_padded_sequence(input_tensor, sequence_lengths.data.tolist(), batch_first=True) lstm_output, _ = lstm(packed_sequence) encoder_output = encoder(input_tensor, mask) lstm_tensor, _ = pad_packed_sequence(lstm_output, batch_first=True) assert_almost_equal(encoder_output.data.numpy(), lstm_tensor.data.numpy())
def test_wrapper_stateful(self): lstm = LSTM(bidirectional=True, num_layers=2, input_size=3, hidden_size=7, batch_first=True) encoder = PytorchSeq2SeqWrapper(lstm, stateful=True) # To test the stateful functionality we need to call the encoder multiple times. # Different batch sizes further tests some of the logic. batch_sizes = [5, 10, 8] sequence_lengths = [4, 6, 7] states = [] for batch_size, sequence_length in zip(batch_sizes, sequence_lengths): tensor = Variable(torch.rand([batch_size, sequence_length, 3])) mask = Variable(torch.ones(batch_size, sequence_length)) mask.data[0, 3:] = 0 encoder_output = encoder(tensor, mask) states.append(encoder._states) # pylint: disable=protected-access # Check that the output is masked properly. assert_almost_equal(encoder_output[0, 3:, :].data.numpy(), numpy.zeros((4, 14))) for k in range(2): assert_almost_equal( states[-1][k][:, -2:, :].data.numpy(), states[-2][k][:, -2:, :].data.numpy() )
def test_wrapper_stateful_single_state_gru(self): gru = GRU(bidirectional=True, num_layers=2, input_size=3, hidden_size=7, batch_first=True) encoder = PytorchSeq2SeqWrapper(gru, stateful=True) batch_sizes = [10, 5] states = [] for batch_size in batch_sizes: tensor = Variable(torch.rand([batch_size, 5, 3])) mask = Variable(torch.ones(batch_size, 5)) mask.data[0, 3:] = 0 encoder_output = encoder(tensor, mask) states.append(encoder._states) # pylint: disable=protected-access assert_almost_equal(encoder_output[0, 3:, :].data.numpy(), numpy.zeros((2, 14))) assert_almost_equal( states[-1][0][:, -5:, :].data.numpy(), states[-2][0][:, -5:, :].data.numpy() )
def __call__(self, # type: ignore logits: torch.Tensor, mask: Optional[torch.Tensor] = None): """ Parameters ---------- logits : ``torch.Tensor``, required. A tensor of unnormalized log probabilities of shape (batch_size, ..., num_classes). mask: ``torch.Tensor``, optional (default = None). A masking tensor of shape (batch_size, ...). """ # Get the data from the Variables. logits, mask = self.unwrap_to_tensors(logits, mask) if mask is None: mask = torch.ones(logits.size()[:-1]) log_probs = torch.nn.functional.log_softmax(Variable(logits), dim=-1).data probabilities = torch.exp(log_probs) * mask.unsqueeze(-1) weighted_negative_likelihood = - log_probs * probabilities entropy = weighted_negative_likelihood.sum(-1) self._entropy += entropy.sum() / mask.sum() self._count += 1
def test_mlpg_gradcheck(): # MLPG is performed dimention by dimention, so static_dim 1 is enough, # 2 just for in case. static_dim = 2 T = 10 for windows in _get_windows_set(): torch.manual_seed(1234) means = Variable(torch.rand(T, static_dim * len(windows)), requires_grad=True) inputs = (means,) # Unit variances case variances = torch.ones(static_dim * len(windows) ).expand(T, static_dim * len(windows)) assert gradcheck(MLPG(variances, windows), inputs, eps=1e-3, atol=1e-3) # Rand variances case variances = torch.rand(static_dim * len(windows) ).expand(T, static_dim * len(windows)) assert gradcheck(MLPG(variances, windows), inputs, eps=1e-3, atol=1e-3)
def forward(self, inputs, batch_size, hidden_cell=None): if hidden_cell is None: # then must init with zeros if use_cuda: hidden = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size).cuda()) cell = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size).cuda()) else: hidden = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size)) cell = Variable(torch.zeros(2, batch_size, hp.enc_hidden_size)) hidden_cell = (hidden, cell) _, (hidden,cell) = self.lstm(inputs.float(), hidden_cell) # hidden is (2, batch_size, hidden_size), we want (batch_size, 2*hidden_size): hidden_forward, hidden_backward = torch.split(hidden,1,0) hidden_cat = torch.cat([hidden_forward.squeeze(0), hidden_backward.squeeze(0)],1) # mu and sigma: mu = self.fc_mu(hidden_cat) sigma_hat = self.fc_sigma(hidden_cat) sigma = torch.exp(sigma_hat/2.) # N ~ N(0,1) z_size = mu.size() if use_cuda: N = Variable(torch.normal(torch.zeros(z_size),torch.ones(z_size)).cuda()) else: N = Variable(torch.normal(torch.zeros(z_size),torch.ones(z_size))) z = mu + sigma*N # mu and sigma_hat are needed for LKL loss return z, mu, sigma_hat
def model(data): # Create unit normal priors over the parameters mu = Variable(torch.zeros(p, 1)).type_as(data) sigma = Variable(torch.ones(p, 1)).type_as(data) bias_mu = Variable(torch.zeros(1)).type_as(data) bias_sigma = Variable(torch.ones(1)).type_as(data) w_prior, b_prior = Normal(mu, sigma), Normal(bias_mu, bias_sigma) priors = {'linear.weight': w_prior, 'linear.bias': b_prior} # lift module parameters to random variables sampled from the priors lifted_module = pyro.random_module("module", regression_model, priors) # sample a regressor (which also samples w and b) lifted_reg_model = lifted_module() with pyro.iarange("map", N, subsample=data): x_data = data[:, :-1] y_data = data[:, -1] # run the regressor forward conditioned on inputs prediction_mean = lifted_reg_model(x_data).squeeze() pyro.observe("obs", Normal(prediction_mean, Variable(torch.ones(data.size(0))).type_as(data)), y_data.squeeze())
def guide(data): w_mu = Variable(torch.randn(p, 1).type_as(data.data), requires_grad=True) w_log_sig = Variable((-3.0 * torch.ones(p, 1) + 0.05 * torch.randn(p, 1)).type_as(data.data), requires_grad=True) b_mu = Variable(torch.randn(1).type_as(data.data), requires_grad=True) b_log_sig = Variable((-3.0 * torch.ones(1) + 0.05 * torch.randn(1)).type_as(data.data), requires_grad=True) # register learnable params in the param store mw_param = pyro.param("guide_mean_weight", w_mu) sw_param = softplus(pyro.param("guide_log_sigma_weight", w_log_sig)) mb_param = pyro.param("guide_mean_bias", b_mu) sb_param = softplus(pyro.param("guide_log_sigma_bias", b_log_sig)) # gaussian guide distributions for w and b w_dist = Normal(mw_param, sw_param) b_dist = Normal(mb_param, sb_param) dists = {'linear.weight': w_dist, 'linear.bias': b_dist} # overloading the parameters in the module with random samples from the guide distributions lifted_module = pyro.random_module("module", regression_model, dists) # sample a regressor return lifted_module() # instantiate optim and inference objects
def setUp(self): pyro.clear_param_store() def model(): mu = pyro.sample("mu", Normal(Variable(torch.zeros(1)), Variable(torch.ones(1)))) xd = Normal(mu, Variable(torch.ones(1)), batch_size=50) pyro.observe("xs", xd, self.data) return mu def guide(): return pyro.sample("mu", Normal(Variable(torch.zeros(1)), Variable(torch.ones(1)))) # data self.data = Variable(torch.zeros(50, 1)) self.mu_mean = Variable(torch.zeros(1)) self.mu_stddev = torch.sqrt(Variable(torch.ones(1)) / 51.0) # model and guide self.model = model self.guide = guide
def setUp(self): # normal-normal; known covariance def model_dup(): pyro.param("mu_q", Variable(torch.ones(1), requires_grad=True)) pyro.sample("mu_q", dist.normal, ng_zeros(1), ng_ones(1)) def model_obs_dup(): pyro.sample("mu_q", dist.normal, ng_zeros(1), ng_ones(1)) pyro.observe("mu_q", dist.normal, ng_zeros(1), ng_ones(1), ng_zeros(1)) def model(): pyro.sample("mu_q", dist.normal, ng_zeros(1), ng_ones(1)) def guide(): p = pyro.param("p", Variable(torch.ones(1), requires_grad=True)) pyro.sample("mu_q", dist.normal, ng_zeros(1), p) pyro.sample("mu_q_2", dist.normal, ng_zeros(1), p) self.duplicate_model = model_dup self.duplicate_obs = model_obs_dup self.model = model self.guide = guide
def setUp(self): # Simple model with 1 continuous + 1 discrete + 1 continuous variable. def model(): p = Variable(torch.Tensor([0.5])) mu = Variable(torch.zeros(1)) sigma = Variable(torch.ones(1)) x = pyro.sample("x", Normal(mu, sigma)) # Before the discrete variable. y = pyro.sample("y", Bernoulli(p)) z = pyro.sample("z", Normal(mu, sigma)) # After the discrete variable. return dict(x=x, y=y, z=z) self.sites = ["x", "y", "z", "_INPUT", "_RETURN"] self.model = model self.queue = Queue() self.queue.put(poutine.Trace())
def __init__(self, shared_resources: SharedResources): super(FastQAPyTorchModule, self).__init__() self._shared_resources = shared_resources input_size = shared_resources.config["repr_dim_input"] size = shared_resources.config["repr_dim"] self._size = size self._with_char_embeddings = self._shared_resources.config.get("with_char_embeddings", False) # modules & parameters if self._with_char_embeddings: self._conv_char_embedding = embedding.ConvCharEmbeddingModule( len(shared_resources.char_vocab), size) self._embedding_projection = nn.Linear(size + input_size, size) self._embedding_highway = Highway(size, 1) self._v_wiq_w = nn.Parameter(torch.ones(1, 1, input_size + size)) input_size = size else: self._v_wiq_w = nn.Parameter(torch.ones(1, 1, input_size)) self._bilstm = BiLSTM(input_size + 2, size) self._answer_layer = FastQAAnswerModule(shared_resources) # [size, 2 * size] self._question_projection = nn.Parameter(torch.cat([torch.eye(size), torch.eye(size)], dim=1)) self._support_projection = nn.Parameter(torch.cat([torch.eye(size), torch.eye(size)], dim=1))
def compute_cov_a(a, classname, layer_info, fast_cnn): batch_size = a.size(0) if classname == 'Conv2d': if fast_cnn: a = _extract_patches(a, *layer_info) a = a.view(a.size(0), -1, a.size(-1)) a = a.mean(1) else: a = _extract_patches(a, *layer_info) a = a.view(-1, a.size(-1)).div_(a.size(1)).div_(a.size(2)) elif classname == 'AddBias': is_cuda = a.is_cuda a = torch.ones(a.size(0), 1) if is_cuda: a = a.cuda() return a.t() @ (a / batch_size)
def calc_gradient_penalty(D, real_data, fake_data): """Calculatge gradient penalty for WGAN-GP.""" alpha = torch.rand(params.batch_size, 1) alpha = alpha.expand(real_data.size()) alpha = make_cuda(alpha) interpolates = make_variable(alpha * real_data + ((1 - alpha) * fake_data)) interpolates.requires_grad = True disc_interpolates = D(interpolates) gradients = grad(outputs=disc_interpolates, inputs=interpolates, grad_outputs=make_cuda( torch.ones(disc_interpolates.size())), create_graph=True, retain_graph=True, only_inputs=True)[0] gradient_penalty = params.penalty_lambda * \ ((gradients.norm(2, dim=1) - 1) ** 2).mean() return gradient_penalty
def __init__(self, num_features, max_length, eps=1e-5, momentum=0.1, affine=True): """ Most parts are copied from torch.nn.modules.batchnorm._BatchNorm. """ super(SeparatedBatchNorm1d, self).__init__() self.num_features = num_features self.max_length = max_length self.affine = affine self.eps = eps self.momentum = momentum if self.affine: self.weight = nn.Parameter(torch.FloatTensor(num_features)) self.bias = nn.Parameter(torch.FloatTensor(num_features)) else: self.register_parameter('weight', None) self.register_parameter('bias', None) for i in range(max_length): self.register_buffer( 'running_mean_{}'.format(i), torch.zeros(num_features)) self.register_buffer( 'running_var_{}'.format(i), torch.ones(num_features)) self.reset_parameters()
def baseline_search(self, input, beam_size=None): # This is the simple greedy search batch_size = input.size(0) hidden_feat = self.lstm_im(input.view(1, input.size()[0], input.size()[1]))[1] x = Variable(torch.ones(1, batch_size,).type(torch.LongTensor) * self.start, requires_grad=False).cuda() # <start> output = [] flag = torch.ones(batch_size) for i in range(self.nseq): input_x = self.encoder(x.view(1, -1)) output_feature, hidden_feat = self.lstm_word(input_x, hidden_feat) output_t = self.decoder(output_feature.view(-1, output_feature.size(2))) output_t = F.log_softmax(output_t) logprob, x = output_t.max(1) output.append(x) flag[x.cpu().eq(self.end).data] = 0 if flag.sum() == 0: break output = torch.stack(output, 0).squeeze().transpose(0, 1).cpu().data return output
def __init__(self, num_features, max_len, eps=1e-5, momentum=0.1, affine=True): super(recurrent_BatchNorm, self).__init__() self.num_features = num_features self.affine = affine self.max_len = max_len self.eps = eps self.momentum = momentum if self.affine: self.weight = nn.Parameter(torch.Tensor(num_features)) self.register_parameter('weight', self.weight) self.bias = nn.Parameter(torch.Tensor(num_features)) self.register_parameter('bias', self.bias) else: self.register_parameter('weight', None) self.register_parameter('bias', None) for i in xrange(max_len): self.register_buffer('running_mean_{}'.format(i), torch.zeros(num_features)) self.register_buffer('running_var_{}'.format(i), torch.ones(num_features)) self.reset_parameters()
def __init__(self, X, Y, hidden_layer_sizes): super(Net, self).__init__() # Initialize linear layer with least squares solution X_ = np.hstack([X, np.ones((X.shape[0],1))]) Theta = np.linalg.solve(X_.T.dot(X_), X_.T.dot(Y)) self.lin = nn.Linear(X.shape[1], Y.shape[1]) W,b = self.lin.parameters() W.data = torch.Tensor(Theta[:-1,:].T) b.data = torch.Tensor(Theta[-1,:]) # Set up non-linear network of # Linear -> BatchNorm -> ReLU -> Dropout layers layer_sizes = [X.shape[1]] + hidden_layer_sizes layers = reduce(operator.add, [[nn.Linear(a,b), nn.BatchNorm1d(b), nn.ReLU(), nn.Dropout(p=0.2)] for a,b in zip(layer_sizes[0:-1], layer_sizes[1:])]) layers += [nn.Linear(layer_sizes[-1], Y.shape[1])] self.net = nn.Sequential(*layers) self.sig = Parameter(torch.ones(1, Y.shape[1]).cuda())
def __init__(self, params, eps=1e-2): super(SolveNewsvendor, self).__init__() k = len(params['d']) self.Q = Variable(torch.diag(torch.Tensor( [params['c_quad']] + [params['b_quad']]*k + [params['h_quad']]*k)) \ .cuda()) self.p = Variable(torch.Tensor( [params['c_lin']] + [params['b_lin']]*k + [params['h_lin']]*k) \ .cuda()) self.G = Variable(torch.cat([ torch.cat([-torch.ones(k,1), -torch.eye(k), torch.zeros(k,k)], 1), torch.cat([torch.ones(k,1), torch.zeros(k,k), -torch.eye(k)], 1), -torch.eye(1 + 2*k)], 0).cuda()) self.h = Variable(torch.Tensor( np.concatenate([-params['d'], params['d'], np.zeros(1+ 2*k)])).cuda()) self.one = Variable(torch.Tensor([1])).cuda() self.eps_eye = eps * Variable(torch.eye(1 + 2*k).cuda()).unsqueeze(0)
def forward(self, y): nBatch, k = y.size() Q_scale = torch.cat([torch.diag(torch.cat( [self.one, y[i], y[i]])).unsqueeze(0) for i in range(nBatch)], 0) Q = self.Q.unsqueeze(0).expand_as(Q_scale).mul(Q_scale) p_scale = torch.cat([Variable(torch.ones(nBatch,1).cuda()), y, y], 1) p = self.p.unsqueeze(0).expand_as(p_scale).mul(p_scale) G = self.G.unsqueeze(0).expand(nBatch, self.G.size(0), self.G.size(1)) h = self.h.unsqueeze(0).expand(nBatch, self.h.size(0)) e = Variable(torch.Tensor().cuda()).double() out = QPFunction(verbose=False)\ (Q.double(), p.double(), G.double(), h.double(), e, e).float() return out[:,:1]
def uniform_weights(n_sample): """Return uniform weights (almost for debug). EXAMPLE ------- >>> weights = uniform_weights(3) >>> print(weights) <BLANKLINE> 0.3333 0.3333 0.3333 [torch.FloatTensor of size 3] <BLANKLINE> :return: """ weights = torch.ones(n_sample) return weights / weights.sum()
def __init__(self, nFeatures, args): super().__init__() nHidden, neq, nineq = 2*nFeatures-1,0,2*nFeatures-2 assert(neq==0) # self.fc1 = nn.Linear(nFeatures, nHidden) self.M = Variable(torch.tril(torch.ones(nHidden, nHidden)).cuda()) Q = 1e-8*torch.eye(nHidden) Q[:nFeatures,:nFeatures] = torch.eye(nFeatures) self.L = Variable(torch.potrf(Q)) self.D = Parameter(0.3*torch.randn(nFeatures-1, nFeatures)) # self.lam = Parameter(20.*torch.ones(1)) self.h = Variable(torch.zeros(nineq)) self.nFeatures = nFeatures self.nHidden = nHidden self.neq = neq self.nineq = nineq self.args = args
def forward(self, x): nBatch = x.size(0) L = self.M*self.L Q = L.mm(L.t()) + self.args.eps*Variable(torch.eye(self.nHidden)).cuda() Q = Q.unsqueeze(0).expand(nBatch, self.nHidden, self.nHidden) nI = Variable(-torch.eye(self.nFeatures-1).type_as(Q.data)) G = torch.cat(( torch.cat(( self.D, nI), 1), torch.cat((-self.D, nI), 1) )) G = G.unsqueeze(0).expand(nBatch, self.nineq, self.nHidden) h = self.h.unsqueeze(0).expand(nBatch, self.nineq) e = Variable(torch.Tensor()) # p = torch.cat((-x, self.lam.unsqueeze(0).expand(nBatch, self.nFeatures-1)), 1) p = torch.cat((-x, Parameter(13.*torch.ones(nBatch, self.nFeatures-1).cuda())), 1) x = QPFunction()(Q.double(), p.double(), G.double(), h.double(), e, e).float() x = x[:,:self.nFeatures] return x
def get_sudoku_matrix(n): X = np.array([[cp.Variable(n**2) for i in range(n**2)] for j in range(n**2)]) cons = ([x >= 0 for row in X for x in row] + [cp.sum_entries(x) == 1 for row in X for x in row] + [sum(row) == np.ones(n**2) for row in X] + [sum([row[i] for row in X]) == np.ones(n**2) for i in range(n**2)] + [sum([sum(row[i:i+n]) for row in X[j:j+n]]) == np.ones(n**2) for i in range(0,n**2,n) for j in range(0, n**2, n)]) f = sum([cp.sum_entries(x) for row in X for x in row]) prob = cp.Problem(cp.Minimize(f), cons) A = np.asarray(prob.get_problem_data(cp.ECOS)["A"].todense()) A0 = [A[0]] rank = 1 for i in range(1,A.shape[0]): if np.linalg.matrix_rank(A0+[A[i]], tol=1e-12) > rank: A0.append(A[i]) rank += 1 return np.array(A0)
def __init__(self, n, Qpenalty, nineq): super().__init__() nx = (n**2)**3 self.Q = Variable(Qpenalty*torch.eye(nx).double().cuda()) self.G1 = Variable(-torch.eye(nx).double().cuda()) self.h1 = Variable(torch.zeros(nx).double().cuda()) # if trueInit: # self.A = Parameter(torch.DoubleTensor(get_sudoku_matrix(n)).cuda()) # else: # # t = get_sudoku_matrix(n) # # self.A = Parameter(torch.rand(t.shape).double().cuda()) # # import IPython, sys; IPython.embed(); sys.exit(-1) self.A = Parameter(torch.rand(50,nx).double().cuda()) self.G2 = Parameter(torch.Tensor(128, nx).uniform_(-1,1).double().cuda()) self.z2 = Parameter(torch.zeros(nx).double().cuda()) self.s2 = Parameter(torch.ones(128).double().cuda()) # self.b = Variable(torch.ones(self.A.size(0)).double().cuda())
def __init__(self, nHidden=50, nineq=200, neq=0, eps=1e-4): super(LenetOptNet, self).__init__() self.conv1 = nn.Conv2d(1, 20, kernel_size=5) self.conv2 = nn.Conv2d(20, 50, kernel_size=5) self.qp_o = nn.Linear(50*4*4, nHidden) self.qp_z0 = nn.Linear(50*4*4, nHidden) self.qp_s0 = nn.Linear(50*4*4, nineq) assert(neq==0) self.M = Variable(torch.tril(torch.ones(nHidden, nHidden)).cuda()) self.L = Parameter(torch.tril(torch.rand(nHidden, nHidden).cuda())) self.G = Parameter(torch.Tensor(nineq,nHidden).uniform_(-1,1).cuda()) # self.z0 = Parameter(torch.zeros(nHidden).cuda()) # self.s0 = Parameter(torch.ones(nineq).cuda()) self.nHidden = nHidden self.nineq = nineq self.neq = neq self.eps = eps
def __init__(self, nFeatures, nHidden, nCls, neq, Qpenalty=0.1, eps=1e-4): super().__init__() self.nFeatures = nFeatures self.nHidden = nHidden self.nCls = nCls self.fc1 = nn.Linear(nFeatures, nHidden) self.fc2 = nn.Linear(nHidden, nCls) self.Q = Variable(Qpenalty*torch.eye(nHidden).double().cuda()) self.G = Variable(-torch.eye(nHidden).double().cuda()) self.h = Variable(torch.zeros(nHidden).double().cuda()) self.A = Parameter(torch.rand(neq,nHidden).double().cuda()) self.b = Variable(torch.ones(self.A.size(0)).double().cuda()) self.neq = neq
def test_Conv2d_large_workspace(self): # These sizes require huge cuDNN workspaces. Make sure we choose a # reasonable algorithm that does not run out of memory sizes = [ (1, 256, 109, 175), (1, 256, 80, 128), (1, 256, 120, 192), ] dtype = torch.cuda.FloatTensor def run_test(benchmark): torch.backends.cudnn.benchmark = benchmark conv = torch.nn.Conv2d(256, 256, kernel_size=3, padding=1).type(dtype) for size in sizes: x = torch.randn(size).type(dtype) out = conv(Variable(x, requires_grad=True)) out.backward(torch.ones(out.size()).type(dtype)) b = torch.backends.cudnn.benchmark try: run_test(benchmark=False) run_test(benchmark=True) finally: torch.backends.cudnn.benchmark = b
def test_accumulate_grad(self): import sys grad_output = Variable(torch.ones(5, 5)) for start_volatile, end_volatile in product((True, False), repeat=2): go1 = grad_output.data if start_volatile else grad_output go2 = grad_output.data if end_volatile else grad_output x = Variable(torch.randn(5, 5), requires_grad=True) y = x + 2 y.backward(go1, retain_variables=True) x_grad = x.grad x_grad_clone = x.grad.data.clone() del x y.backward(go2) # That's the only case when we can accumulate in-place if start_volatile and end_volatile: expected_grad = x_grad_clone * 2 else: expected_grad = x_grad_clone self.assertEqual(x_grad.data, expected_grad)
def test_hessian_vector(self): x = Variable(torch.randn(2, 2), requires_grad=True) y = Variable(torch.randn(2, 2), requires_grad=True) z = x ** 2 + y * x + y ** 2 z.backward(Variable(torch.ones(2, 2), requires_grad=True), retain_variables=True) x_grad = 2 * x.data + y.data y_grad = x.data + 2 * y.data self.assertEqual(x.grad.data, x_grad) self.assertEqual(y.grad.data, y_grad) grad_sum = 2 * x.grad + y.grad grad_sum.backward(torch.ones(2, 2)) x_hv = torch.ones(2, 2) * 5 y_hv = torch.ones(2, 2) * 4 self.assertEqual(x.grad.data, x_grad + x_hv) self.assertEqual(y.grad.data, y_grad + y_hv)
def test_hooks_cycle(self): import gc counter = [0] class GradHook(object): def __init__(self, var): self.var = var def __del__(self): counter[0] += 1 def __call__(self, *args): pass def run_test(): x = Variable(torch.ones(5, 5), requires_grad=True) y = x * 2 x.register_hook(GradHook(x)) y.register_hook(GradHook(y)) y._backward_hooks[1] = GradHook(y) run_test() gc.collect() self.assertEqual(counter[0], 3)
def test_volatile(self): x = Variable(torch.ones(5, 5), requires_grad=True) y = Variable(torch.ones(5, 5) * 4, volatile=True) z = x ** 2 self.assertFalse(z.volatile) self.assertTrue(z.requires_grad) self.assertIsNotNone(z.grad_fn) z.backward(torch.ones(5, 5)) self.assertEqual(x.grad.data, torch.ones(5, 5) * 2) w = z + y self.assertTrue(w.volatile) self.assertFalse(w.requires_grad) self.assertRaises(RuntimeError, lambda: w.backward(torch.ones(5, 5))) self.assertIsNone(w.grad_fn)
def test_requires_grad(self): x = Variable(torch.randn(5, 5)) y = Variable(torch.randn(5, 5)) z = Variable(torch.randn(5, 5), requires_grad=True) a = x + y self.assertFalse(a.requires_grad) b = a + z self.assertTrue(b.requires_grad) def error(): raise RuntimeError # Make sure backward isn't called on these a._backward_hooks = OrderedDict() x._backward_hooks = OrderedDict() y._backward_hooks = OrderedDict() a._backward_hooks['test'] = error x._backward_hooks['test'] = error y._backward_hooks['test'] = error b.backward(torch.ones(5, 5))
def _test_setitem_tensor(self, size, index): x = Variable(torch.ones(*size), requires_grad=True) y = x + 2 y_version = y._version value = Variable(torch.Tensor(x[index].size()).fill_(7), requires_grad=True) y[index] = value self.assertNotEqual(y._version, y_version) y.backward(torch.ones(*size)) expected_grad_input = torch.ones(*size) if isinstance(index, Variable): index = index.data expected_grad_input[index] = 0 self.assertEqual(x.grad.data, expected_grad_input) self.assertEqual(value.grad.data, torch.ones(value.size())) # case when x is not same shape as y[1] x = Variable(torch.randn(1, 2), requires_grad=True) y = Variable(torch.zeros(10, 2)) y[1] = x y.backward(torch.randn(10, 2)) self.assertEqual(x.size(), x.grad.size())
def test_return_leaf_inplace(self): class Inplace(InplaceFunction): def forward(self, a, b): self.mark_dirty(a) return a.add_(b), b + 2 def backward(self, grad_a, grad_b): return grad_a, grad_a + grad_b x = Variable(torch.randn(5, 5)) y = Variable(torch.randn(5, 5), requires_grad=True) fn = Inplace(True) q, p = fn(x, y) self.assertIs(q, x) self.assertIs(q.grad_fn, fn) self.assertTrue(q.requires_grad) q.sum().backward() self.assertEqual(y.grad.data, torch.ones(5, 5))
def _test_pool(self, ctx=mp, repeat=1): def do_test(): p = ctx.Pool(2) for proc in p._pool: lc.check_pid(proc.pid) buffers = [torch.zeros(2, 2) for i in range(4)] results = p.map(simple_pool_fill, buffers, 1) self.assertEqual(len(results), len(buffers)) for r in results: self.assertEqual(r, torch.ones(2, 2) * 5, 0) for b in buffers: self.assertEqual(b, torch.ones(2, 2) * 4, 0) p.close() p.join() with leak_checker(self) as lc: for i in range(repeat): do_test()
def _test_autograd_sharing(self, var): ready = mp.Event() master_modified = mp.Event() queue = mp.Queue() p = mp.Process(target=autograd_sharing, args=(queue, ready, master_modified)) p.daemon = True p.start() var._grad = Variable(torch.zeros(5, 5), requires_grad=False) queue.put(var) ready.wait() var.data[0, 0] = 1000 var.grad.data[:] = torch.ones(5, 5) * 4 master_modified.set() worker_ok = queue.get() self.assertTrue(worker_ok) self.assertEqual(var.data, torch.ones(5, 5)) self.assertEqual(var.grad.data, torch.ones(5, 5) * 4) p.join(1) self.assertFalse(p.is_alive())
