我们从Python开源项目中，提取了以下22个代码示例，用于说明如何使用torchvision.datasets.MNIST。
def get_loader(config): """Builds and returns Dataloader for MNIST and SVHN dataset.""" transform = transforms.Compose([ transforms.Scale(config.image_size), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) svhn = datasets.SVHN(root=config.svhn_path, download=True, transform=transform) mnist = datasets.MNIST(root=config.mnist_path, download=True, transform=transform) svhn_loader = torch.utils.data.DataLoader(dataset=svhn, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) mnist_loader = torch.utils.data.DataLoader(dataset=mnist, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) return svhn_loader, mnist_loader
def get_mnist(train): """Get MNIST dataset loader.""" # image pre-processing pre_process = transforms.Compose([transforms.ToTensor(), transforms.Normalize( mean=params.dataset_mean, std=params.dataset_std)]) # dataset and data loader mnist_dataset = datasets.MNIST(root=params.data_root, train=train, transform=pre_process, download=True) mnist_data_loader = torch.utils.data.DataLoader( dataset=mnist_dataset, batch_size=params.batch_size, shuffle=True) return mnist_data_loader
def mnist(self): norm_mean = [0.1307] norm_std = [0.3081] train_loader = torch.utils.data.DataLoader( dsets.MNIST("/home/dataset/mnist", train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std) ])), batch_size=self.train_batch_size, shuffle=True, num_workers=self.n_threads, pin_memory=False ) test_loader = torch.utils.data.DataLoader( dsets.MNIST("/home/dataset/mnist", train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std) ])), batch_size=self.test_batch_size, shuffle=True, num_workers=self.n_threads, pin_memory=False ) return train_loader, test_loader
def train(rank, args, model): torch.manual_seed(args.seed + rank) train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batch_size, shuffle=True, num_workers=1) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batch_size, shuffle=True, num_workers=1) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) for epoch in range(1, args.epochs + 1): train_epoch(epoch, args, model, train_loader, optimizer) test_epoch(model, test_loader)
def __init__(self, batch_size): self.batch_size = batch_size train_dataset = dsets.MNIST(root="./data", train=True, transform=transforms.ToTensor(), download=True) test_dataset = dsets.MNIST(root="./data", train=False, transform=transforms.ToTensor()) self.train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) self.test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)
def get_loaders(loader_batchsize, **kwargs): arguments=kwargs['arguments'] data = arguments.data if data == 'mnist': kwargs = {'num_workers': 1, 'pin_memory': True} if arguments.cuda else {} train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), #transforms.Normalize((0,), (1,)) ])), batch_size=loader_batchsize, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), #transforms.Normalize((7,), (0.3081,)) ])), batch_size=loader_batchsize, shuffle=True, **kwargs) return train_loader, test_loader
def train(epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): if use_cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % 500 == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.data[0])) # # A simple test procedure to measure STN the performances on MNIST. #
def get_dataset(name, split='train', transform=None, target_transform=None, download=True, datasets_path=__DATASETS_DEFAULT_PATH): train = (split == 'train') root = os.path.join(datasets_path, name) if name == 'cifar10': return datasets.CIFAR10(root=root, train=train, transform=transform, target_transform=target_transform, download=download) elif name == 'cifar100': return datasets.CIFAR100(root=root, train=train, transform=transform, target_transform=target_transform, download=download) elif name == 'mnist': return datasets.MNIST(root=root, train=train, transform=transform, target_transform=target_transform, download=download) elif name == 'stl10': return datasets.STL10(root=root, split=split, transform=transform, target_transform=target_transform, download=download) elif name == 'imagenet': if train: root = os.path.join(root, 'train') else: root = os.path.join(root, 'val') return datasets.ImageFolder(root=root, transform=transform, target_transform=target_transform)
def get_loader(config): """Builds and returns Dataloader for MNIST and SVHN dataset.""" transform = transforms.Compose([ transforms.Scale(config.image_size), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) svhn = datasets.SVHN(root=config.svhn_path, download=True, transform=transform, split='train') mnist = datasets.MNIST(root=config.mnist_path, download=True, transform=transform, train=True) svhn_test = datasets.SVHN(root=config.svhn_path, download=True, transform=transform, split='test') mnist_test = datasets.MNIST(root=config.mnist_path, download=True, transform=transform, train=False) svhn_loader = torch.utils.data.DataLoader(dataset=svhn, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) mnist_loader = torch.utils.data.DataLoader(dataset=mnist, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) svhn_test_loader = torch.utils.data.DataLoader(dataset=svhn_test, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) mnist_test_loader = torch.utils.data.DataLoader(dataset=mnist_test, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) return svhn_loader, mnist_loader, svhn_test_loader, mnist_test_loader
def __init__(self, dataset, train_batch_size, test_batch_size, n_threads=4, ten_crop=False): self.dataset = dataset self.train_batch_size = train_batch_size self.test_batch_size = test_batch_size self.n_threads = n_threads self.ten_crop = ten_crop if self.dataset == "cifar10" or self.dataset == "cifar100": print "|===>Creating Cifar Data Loader" self.train_loader, self.test_loader = self.cifar(dataset=self.dataset) elif self.dataset == "mnist": print "|===>Creating MNIST Data Loader" self.train_loader, self.test_loader = self.mnist() else: assert False, "invalid data set"
def main(): if torch.cuda.is_available(): use_cuda = True else: use_cuda = False # Dataset trainset = datasets.MNIST('../../data', download=True,train=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])) train_loader = DataLoader(trainset, batch_size=128, shuffle=True, num_workers=4) # Model model = Net() # NLLLoss nllloss = nn.NLLLoss() #CrossEntropyLoss = log_softmax + NLLLoss # CenterLoss loss_weight = 1.0 centerloss = CenterLoss(10,2,loss_weight) if use_cuda: nllloss = nllloss.cuda() centerloss = centerloss.cuda() model = model.cuda() criterion = [nllloss, centerloss] # optimzer4nn optimizer4nn = optim.SGD(model.parameters(),lr=0.001,momentum=0.9, weight_decay=0.0005) sheduler = lr_scheduler.StepLR(optimizer4nn,20,gamma=0.8) # optimzer4center optimzer4center = optim.SGD(centerloss.parameters(), lr =0.5) for epoch in range(50): sheduler.step() # print optimizer4nn.param_groups[0]['lr'] train(train_loader, model, criterion, [optimizer4nn, optimzer4center], epoch+1, use_cuda)
def MNIST_loader(root, image_size, normalize=True): """ Function to load torchvision dataset object based on just image size Args: root = If your dataset is downloaded and ready to use, mention the location of this folder. Else, the dataset will be downloaded to this location image_size = Size of every image normalize = Requirement to normalize the image. Default is true """ transformations = [transforms.Scale(image_size), transforms.ToTensor()] if normalize == True: transformations.append(transforms.Normalize((0.5, ), (0.5, ))) mnist_data = dset.MNIST(root=root, download=True, transform=transforms.Compose(transformations)) return mnist_data
def get(batch_size, data_root='/tmp/public_dataset/pytorch', train=True, val=True, **kwargs): data_root = os.path.expanduser(os.path.join(data_root, 'mnist-data')) kwargs.pop('input_size', None) num_workers = kwargs.setdefault('num_workers', 1) print("Building MNIST data loader with {} workers".format(num_workers)) ds = [] if train: train_loader = torch.utils.data.DataLoader( datasets.MNIST(root=data_root, train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size, shuffle=True, **kwargs) ds.append(train_loader) if val: test_loader = torch.utils.data.DataLoader( datasets.MNIST(root=data_root, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size, shuffle=True, **kwargs) ds.append(test_loader) ds = ds[0] if len(ds) == 1 else ds return ds
def execute(self): batch_size = 50 train_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor()), batch_size=batch_size, shuffle=True) self.data = train_loader for port in self.outPorts: port.transferData(self.data)
def __init__(self, batchsize, train=True): Dataset.__init__(self) data_root = join(dirname(realpath(__file__)), 'MNIST_data') self.name = "mnist" self.range = [0.0, 1.0] self.data_dims = [1, 28, 28] self.batchsize = batchsize self.data = dsets.MNIST(root=data_root, download=True, train=train, transform=transforms.Compose([ transforms.ToTensor()])) self.dataloder = tdata.DataLoader(self.data, self.batchsize, shuffle=True) self.iter = iter(self.dataloder) self._index = 0
def __init__(self, mode, sup_num, use_cuda=True, *args, **kwargs): super(MNISTCached, self).__init__(train=mode in ["sup", "unsup", "valid"], *args, **kwargs) # transformations on MNIST data (normalization and one-hot conversion for labels) def transform(x): return fn_x_mnist(x, use_cuda) def target_transform(y): return fn_y_mnist(y, use_cuda) self.mode = mode assert mode in ["sup", "unsup", "test", "valid"], "invalid train/test option values" if mode in ["sup", "unsup", "valid"]: # transform the training data if transformations are provided if transform is not None: self.train_data = (transform(self.train_data.float())) if target_transform is not None: self.train_labels = (target_transform(self.train_labels)) if MNISTCached.train_data_sup is None: if sup_num is None: assert mode == "unsup" MNISTCached.train_data_unsup, MNISTCached.train_labels_unsup = \ self.train_data, self.train_labels else: MNISTCached.train_data_sup, MNISTCached.train_labels_sup, \ MNISTCached.train_data_unsup, MNISTCached.train_labels_unsup, \ MNISTCached.data_valid, MNISTCached.labels_valid = \ split_sup_unsup_valid(self.train_data, self.train_labels, sup_num) if mode == "sup": self.train_data, self.train_labels = MNISTCached.train_data_sup, MNISTCached.train_labels_sup elif mode == "unsup": self.train_data = MNISTCached.train_data_unsup # making sure that the unsupervised labels are not available to inference self.train_labels = (torch.Tensor( MNISTCached.train_labels_unsup.shape[0]).view(-1, 1)) * np.nan else: self.train_data, self.train_labels = MNISTCached.data_valid, MNISTCached.labels_valid else: # transform the testing data if transformations are provided if transform is not None: self.test_data = (transform(self.test_data.float())) if target_transform is not None: self.test_labels = (target_transform(self.test_labels))
def get_data(args, train_flag=True): transform = transforms.Compose([ transforms.Scale(args.image_size), transforms.CenterCrop(args.image_size), transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) if args.dataset in ['imagenet', 'folder', 'lfw']: dataset = dset.ImageFolder(root=args.dataroot, transform=transform) elif args.dataset == 'lsun': dataset = dset.LSUN(db_path=args.dataroot, classes=['bedroom_train'], transform=transform) elif args.dataset == 'cifar10': dataset = dset.CIFAR10(root=args.dataroot, download=True, train=train_flag, transform=transform) elif args.dataset == 'cifar100': dataset = dset.CIFAR100(root=args.dataroot, download=True, train=train_flag, transform=transform) elif args.dataset == 'mnist': dataset = dset.MNIST(root=args.dataroot, download=True, train=train_flag, transform=transform) elif args.dataset == 'celeba': imdir = 'train' if train_flag else 'val' dataroot = os.path.join(args.dataroot, imdir) if args.image_size != 64: raise ValueError('the image size for CelebA dataset need to be 64!') dataset = FolderWithImages(root=dataroot, input_transform=transforms.Compose([ ALICropAndScale(), transforms.ToTensor(), transforms.Normalize( (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]), target_transform=transforms.ToTensor() ) else: raise ValueError("Unknown dataset %s" % (args.dataset)) return dataset
def get_loaders(args): kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} if args.dataset == 'mnist': trainLoader = torch.utils.data.DataLoader( dset.MNIST('data/mnist', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batchSz, shuffle=True, **kwargs) testLoader = torch.utils.data.DataLoader( dset.MNIST('data/mnist', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batchSz, shuffle=False, **kwargs) elif args.dataset == 'cifar-10': normMean = [0.49139968, 0.48215827, 0.44653124] normStd = [0.24703233, 0.24348505, 0.26158768] normTransform = transforms.Normalize(normMean, normStd) trainTransform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normTransform ]) testTransform = transforms.Compose([ transforms.ToTensor(), normTransform ]) trainLoader = DataLoader( dset.CIFAR10(root='data/cifar', train=True, download=True, transform=trainTransform), batch_size=args.batchSz, shuffle=True, **kwargs) testLoader = DataLoader( dset.CIFAR10(root='data/cifar', train=False, download=True, transform=testTransform), batch_size=args.batchSz, shuffle=False, **kwargs) else: assert(False) return trainLoader, testLoader
def execute(self): model = MnistModel() batch_size = 50 train_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor()), batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=False, transform=transforms.ToTensor()), batch_size=1000) for p in model.parameters(): print(p.size()) optimizer = optim.Adam(model.parameters(), lr=0.0001) model.train() train_loss = [] train_accu = [] i = 0 for epoch in range(15): for data, target in train_loader: data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() # calc gradients train_loss.append(loss.data[0]) optimizer.step() # update gradients prediction = output.data.max(1)[1] # first column has actual prob. accuracy = prediction.eq(target.data).sum()/batch_size*100 train_accu.append(accuracy) if i % 1000 == 0: print('Train Step: {}\tLoss: {:.3f}\tAccuracy: {:.3f}'.format(i, loss.data[0], accuracy)) i += 1 plt.plot(np.arange(len(train_loss)), train_loss) plt.plot(np.arange(len(train_accu)), train_accu) model.eval() correct = 0 for data, target in test_loader: data, target = Variable(data, volatile=True), Variable(target) output = model(data) prediction = output.data.max(1)[1] correct += prediction.eq(target.data).sum() print('\nTest set: Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset))) self.data = np.random.rand(self.dim[0],self.dim[1]) for port in self.outPorts: port.transferData(self.data)
