我们从Python开源项目中，提取了以下15个代码示例，用于说明如何使用scipy.sparse.data()。
def chunkize_serial(iterable, chunksize, as_numpy=False): """ Return elements from the iterable in `chunksize`-ed lists. The last returned element may be smaller (if length of collection is not divisible by `chunksize`). >>> print(list(grouper(range(10), 3))) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] """ import numpy it = iter(iterable) while True: if as_numpy: # convert each document to a 2d numpy array (~6x faster when transmitting # chunk data over the wire, in Pyro) wrapped_chunk = [[numpy.array(doc) for doc in itertools.islice(it, int(chunksize))]] else: wrapped_chunk = [list(itertools.islice(it, int(chunksize)))] if not wrapped_chunk[0]: break # memory opt: wrap the chunk and then pop(), to avoid leaving behind a dangling reference yield wrapped_chunk.pop()
def chunkize_serial(iterable, chunksize, as_numpy=False): """ Return elements from the iterable in `chunksize`-ed lists. The last returned element may be smaller (if length of collection is not divisible by `chunksize`). >>> print(list(grouper(range(10), 3))) [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] """ it = iter(iterable) while True: if as_numpy: # convert each document to a 2d numpy array (~6x faster when transmitting # chunk data over the wire, in Pyro) wrapped_chunk = [[np.array(doc) for doc in itertools.islice(it, int(chunksize))]] else: wrapped_chunk = [list(itertools.islice(it, int(chunksize)))] if not wrapped_chunk[0]: break # memory opt: wrap the chunk and then pop(), to avoid leaving behind a dangling reference yield wrapped_chunk.pop()
def load(cls, fname, mmap=None): """ Load a previously saved object from file (also see `save`). If the object was saved with large arrays stored separately, you can load these arrays via mmap (shared memory) using `mmap='r'`. Default: don't use mmap, load large arrays as normal objects. """ logger.info("loading %s object from %s" % (cls.__name__, fname)) subname = lambda suffix: fname + '.' + suffix + '.npy' obj = unpickle(fname) for attrib in getattr(obj, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % (attrib, subname(attrib), mmap)) setattr(obj, attrib, numpy.load(subname(attrib), mmap_mode=mmap)) for attrib in getattr(obj, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % (attrib, subname(attrib), mmap)) sparse = unpickle(subname(attrib)) sparse.data = numpy.load(subname(attrib) + '.data.npy', mmap_mode=mmap) sparse.indptr = numpy.load(subname(attrib) + '.indptr.npy', mmap_mode=mmap) sparse.indices = numpy.load(subname(attrib) + '.indices.npy', mmap_mode=mmap) setattr(obj, attrib, sparse) for attrib in getattr(obj, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(obj, attrib, None) return obj
def mock_data_row(dim=1000, prob_nnz=0.5, lam=1.0): """ Create a random gensim sparse vector. Each coordinate is nonzero with probability `prob_nnz`, each non-zero coordinate value is drawn from a Poisson distribution with parameter lambda equal to `lam`. """ nnz = np.random.uniform(size=(dim,)) data = [(i, float(np.random.poisson(lam=lam) + 1.0)) for i in xrange(dim) if nnz[i] < prob_nnz] return data
def mock_data(n_items=1000, dim=1000, prob_nnz=0.5, lam=1.0): """ Create a random gensim-style corpus, as a list of lists of (int, float) tuples, to be used as a mock corpus. """ data = [mock_data_row(dim=dim, prob_nnz=prob_nnz, lam=lam) for _ in xrange(n_items)] return data
def _load_specials(self, fname, mmap, compress, subname): """ Loads any attributes that were stored specially, and gives the same opportunity to recursively included SaveLoad instances. """ mmap_error = lambda x, y: IOError( 'Cannot mmap compressed object %s in file %s. ' % (x, y) + 'Use `load(fname, mmap=None)` or uncompress files manually.') for attrib in getattr(self, '__recursive_saveloads', []): cfname = '.'.join((fname, attrib)) logger.info("loading %s recursively from %s.* with mmap=%s" % ( attrib, cfname, mmap)) getattr(self, attrib)._load_specials(cfname, mmap, compress, subname) for attrib in getattr(self, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) val = np.load(subname(fname, attrib))['val'] else: val = np.load(subname(fname, attrib), mmap_mode=mmap) setattr(self, attrib, val) for attrib in getattr(self, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) sparse = unpickle(subname(fname, attrib)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) with np.load(subname(fname, attrib, 'sparse')) as f: sparse.data = f['data'] sparse.indptr = f['indptr'] sparse.indices = f['indices'] else: sparse.data = np.load(subname(fname, attrib, 'data'), mmap_mode=mmap) sparse.indptr = np.load(subname(fname, attrib, 'indptr'), mmap_mode=mmap) sparse.indices = np.load(subname(fname, attrib, 'indices'), mmap_mode=mmap) setattr(self, attrib, sparse) for attrib in getattr(self, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(self, attrib, None)
def _load_specials(self, fname, mmap, compress, subname): """ Loads any attributes that were stored specially, and gives the same opportunity to recursively included SaveLoad instances. """ mmap_error = lambda x, y: IOError( 'Cannot mmap compressed object %s in file %s. ' % (x, y) + 'Use `load(fname, mmap=None)` or uncompress files manually.') for attrib in getattr(self, '__recursive_saveloads', []): cfname = '.'.join((fname, attrib)) logger.info("loading %s recursively from %s.* with mmap=%s" % ( attrib, cfname, mmap)) getattr(self, attrib)._load_specials(cfname, mmap, compress, subname) for attrib in getattr(self, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) val = numpy.load(subname(fname, attrib))['val'] else: val = numpy.load(subname(fname, attrib), mmap_mode=mmap) setattr(self, attrib, val) for attrib in getattr(self, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) sparse = unpickle(subname(fname, attrib)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) with numpy.load(subname(fname, attrib, 'sparse')) as f: sparse.data = f['data'] sparse.indptr = f['indptr'] sparse.indices = f['indices'] else: sparse.data = numpy.load(subname(fname, attrib, 'data'), mmap_mode=mmap) sparse.indptr = numpy.load(subname(fname, attrib, 'indptr'), mmap_mode=mmap) sparse.indices = numpy.load(subname(fname, attrib, 'indices'), mmap_mode=mmap) setattr(self, attrib, sparse) for attrib in getattr(self, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(self, attrib, None)
