/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); this.snapshotSize = keySize(); if (!this.cellSet.isEmpty()) { this.snapshot = this.cellSet; this.cellSet = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); this.snapshotAllocator = this.allocator; // Reset allocator so we get a fresh buffer for the new memstore if (allocator != null) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } timeOfOldestEdit = Long.MAX_VALUE; } } MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator), this.tagsPresent); this.tagsPresent = false; return memStoreSnapshot; }
private void verifySeeking(EncodedSeeker encodeSeeker, ByteBuffer encodedData, int batchId) { List<KeyValue> kvList = new ArrayList<KeyValue>(); for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { kvList.clear(); encodeSeeker.setCurrentBuffer(encodedData); KeyValue firstOnRow = KeyValueUtil.createFirstOnRow(getRowKey(batchId, i)); encodeSeeker.seekToKeyInBlock( new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(), firstOnRow.getKeyLength()), false); boolean hasMoreOfEncodeScanner = encodeSeeker.next(); CollectionBackedScanner collectionScanner = new CollectionBackedScanner( this.kvset); boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow); if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) { dumpInputKVSet(); fail("Get error result after seeking " + firstOnRow); } if (hasMoreOfEncodeScanner) { if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(), collectionScanner.peek()) != 0) { dumpInputKVSet(); fail("Expected " + collectionScanner.peek() + " actual " + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow); } } } }
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); this.snapshotSize = keySize(); if (!this.cellSet.isEmpty()) { this.snapshot = this.cellSet; this.cellSet = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); this.snapshotAllocator = this.allocator; // Reset allocator so we get a fresh buffer for the new memstore if (allocator != null) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[]{Configuration.class}, new Object[]{conf}); } timeOfOldestEdit = Long.MAX_VALUE; } } return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator)); }
private void verifySeeking(EncodedSeeker encodeSeeker, ByteBuffer encodedData, int batchId) { List<KeyValue> kvList = new ArrayList<KeyValue>(); for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { kvList.clear(); encodeSeeker.setCurrentBuffer(encodedData); KeyValue firstOnRow = KeyValue.createFirstOnRow(getRowKey(batchId, i)); encodeSeeker.seekToKeyInBlock(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(), firstOnRow.getKeyLength(), false); boolean hasMoreOfEncodeScanner = encodeSeeker.next(); CollectionBackedScanner collectionScanner = new CollectionBackedScanner( this.kvset); boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow); if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) { dumpInputKVSet(); fail("Get error result after seeking " + firstOnRow); } if (hasMoreOfEncodeScanner) { if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(), collectionScanner.peek()) != 0) { dumpInputKVSet(); fail("Expected " + collectionScanner.peek() + " actual " + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow); } } } }
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTimeMillis(); this.snapshotSize = keySize(); if (!this.kvset.isEmpty()) { this.snapshot = this.kvset; this.kvset = new KeyValueSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); this.snapshotAllocator = this.allocator; // Reset allocator so we get a fresh buffer for the new memstore if (allocator != null) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } timeOfOldestEdit = Long.MAX_VALUE; } } return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator)); }
@Override public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId, TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status) throws IOException { ArrayList<Path> result = new ArrayList<Path>(); if (snapshot.size() == 0) return result; // don't flush if there are no entries // Use a store scanner to find which rows to flush. long smallestReadPoint = store.getSmallestReadPoint(); KeyValueScanner memstoreScanner = new CollectionBackedScanner(snapshot, store.getComparator()); InternalScanner scanner = preCreateCoprocScanner(memstoreScanner); if (scanner == null) { scanner = createStoreScanner(smallestReadPoint, memstoreScanner); } scanner = postCreateCoprocScanner(scanner); if (scanner == null) { return result; // NULL scanner returned from coprocessor hooks means skip normal processing } StoreFile.Writer writer; long flushed = 0; try { // TODO: We can fail in the below block before we complete adding this flush to // list of store files. Add cleanup of anything put on filesystem if we fail. synchronized (flushLock) { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk writer = store.createWriterInTmp( snapshot.size(), store.getFamily().getCompression(), false, true); writer.setTimeRangeTracker(snapshotTimeRangeTracker); try { flushed = performFlush(scanner, writer, smallestReadPoint); } finally { finalizeWriter(writer, cacheFlushId, status); } } } finally { flushedSize.set(flushed); scanner.close(); } LOG.info("Flushed, sequenceid=" + cacheFlushId +", memsize=" + StringUtils.humanReadableInt(flushed) + ", hasBloomFilter=" + writer.hasGeneralBloom() + ", into tmp file " + writer.getPath()); result.add(writer.getPath()); return result; }
