@Test public void testReplayingCompactionWithFileAlreadyDeleted() throws IOException { // tests replaying compaction marker, but the compaction output file has already been compacted // from primary and also deleted from the archive directory secondaryRegion.replayWALCompactionMarker(CompactionDescriptor.newBuilder() .setTableName(ByteString.copyFrom(primaryRegion.getTableDesc().getTableName().getName())) .setEncodedRegionName( ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes())) .setFamilyName(ByteString.copyFrom(families[0])) .addCompactionInput("/foo") .addCompactionOutput("/bar") .setStoreHomeDir("/store_home_dir") .setRegionName(ByteString.copyFrom(primaryRegion.getRegionInfo().getRegionName())) .build() , true, true, Long.MAX_VALUE); }
@SuppressWarnings("deprecation") public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName, byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) { // compaction descriptor contains relative paths. // input / output paths are relative to the store dir // store dir is relative to region dir CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder() .setTableName(ByteStringer.wrap(info.getTableName())) .setEncodedRegionName(ByteStringer.wrap( regionName == null ? info.getEncodedNameAsBytes() : regionName)) .setFamilyName(ByteStringer.wrap(family)) .setStoreHomeDir(storeDir.getName()); //make relative for (Path inputPath : inputPaths) { builder.addCompactionInput(inputPath.getName()); //relative path } for (Path outputPath : outputPaths) { builder.addCompactionOutput(outputPath.getName()); } builder.setRegionName(ByteStringer.wrap(info.getRegionName())); return builder.build(); }
public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) { // compaction descriptor contains relative paths. // input / output paths are relative to the store dir // store dir is relative to region dir CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder() .setTableName(ByteStringer.wrap(info.getTableName())) .setEncodedRegionName(ByteStringer.wrap(info.getEncodedNameAsBytes())) .setFamilyName(ByteStringer.wrap(family)) .setStoreHomeDir(storeDir.getName()); //make relative for (Path inputPath : inputPaths) { builder.addCompactionInput(inputPath.getName()); //relative path } for (Path outputPath : outputPaths) { builder.addCompactionOutput(outputPath.getName()); } builder.setRegionName(ByteStringer.wrap(info.getRegionName())); return builder.build(); }
public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) { // compaction descriptor contains relative paths. // input / output paths are relative to the store dir // store dir is relative to region dir CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder() .setTableName(HBaseZeroCopyByteString.wrap(info.getTableName())) .setEncodedRegionName(HBaseZeroCopyByteString.wrap(info.getEncodedNameAsBytes())) .setFamilyName(HBaseZeroCopyByteString.wrap(family)) .setStoreHomeDir(storeDir.getName()); //make relative for (Path inputPath : inputPaths) { builder.addCompactionInput(inputPath.getName()); //relative path } for (Path outputPath : outputPaths) { builder.addCompactionOutput(outputPath.getName()); } return builder.build(); }
public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) { // compaction descriptor contains relative paths. // input / output paths are relative to the store dir // store dir is relative to region dir CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder() .setTableName(ZeroCopyLiteralByteString.wrap(info.getTableName())) .setEncodedRegionName(ZeroCopyLiteralByteString.wrap(info.getEncodedNameAsBytes())) .setFamilyName(ZeroCopyLiteralByteString.wrap(family)) .setStoreHomeDir(storeDir.getName()); //make relative for (Path inputPath : inputPaths) { builder.addCompactionInput(inputPath.getName()); //relative path } for (Path outputPath : outputPaths) { builder.addCompactionOutput(outputPath.getName()); } return builder.build(); }
/** * Create a compaction WALEdit * @param c * @return A WALEdit that has <code>c</code> serialized as its value */ public static WALEdit createCompaction(final HRegionInfo hri, final CompactionDescriptor c) { byte [] pbbytes = c.toByteArray(); KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION, EnvironmentEdgeManager.currentTime(), pbbytes); return new WALEdit().add(kv); //replication scope null so that this won't be replicated }
/** * Deserialized and returns a CompactionDescriptor is the KeyValue contains one. * @param kv the key value * @return deserialized CompactionDescriptor or null. */ public static CompactionDescriptor getCompaction(Cell kv) throws IOException { if (CellUtil.matchingColumn(kv, METAFAMILY, COMPACTION)) { return CompactionDescriptor.parseFrom(kv.getValue()); } return null; }
/** * Write the marker that a compaction has succeeded and is about to be committed. * This provides info to the HMaster to allow it to recover the compaction if * this regionserver dies in the middle (This part is not yet implemented). It also prevents * the compaction from finishing if this regionserver has already lost its lease on the log. * @param mvcc Used by WAL to get sequence Id for the waledit. */ public static long writeCompactionMarker(WAL wal, HTableDescriptor htd, HRegionInfo hri, final CompactionDescriptor c, MultiVersionConcurrencyControl mvcc) throws IOException { long trx = writeMarker(wal, htd, hri, WALEdit.createCompaction(hri, c), mvcc, true); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } return trx; }
/** * Create a compacion WALEdit * @param c * @return A WALEdit that has <code>c</code> serialized as its value */ public static WALEdit createCompaction(final HRegionInfo hri, final CompactionDescriptor c) { byte [] pbbytes = c.toByteArray(); KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION, EnvironmentEdgeManager.currentTime(), pbbytes); return new WALEdit().add(kv); //replication scope null so that this won't be replicated }
/** * Write the marker that a compaction has succeeded and is about to be committed. * This provides info to the HMaster to allow it to recover the compaction if * this regionserver dies in the middle (This part is not yet implemented). It also prevents * the compaction from finishing if this regionserver has already lost its lease on the log. * @param sequenceId Used by WAL to get sequence Id for the waledit. */ public static void writeCompactionMarker(WAL log, HTableDescriptor htd, HRegionInfo info, final CompactionDescriptor c, AtomicLong sequenceId) throws IOException { TableName tn = TableName.valueOf(c.getTableName().toByteArray()); // we use HLogKey here instead of WALKey directly to support legacy coprocessors. WALKey key = new HLogKey(info.getEncodedNameAsBytes(), tn); log.append(htd, info, key, WALEdit.createCompaction(info, c), sequenceId, false, null); log.sync(); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2331. */ void completeCompactionMarker(CompactionDescriptor compaction) throws IOException { Store store = this.getStore(compaction.getFamilyName().toByteArray()); if (store == null) { LOG.warn("Found Compaction WAL edit for deleted family:" + Bytes.toString(compaction.getFamilyName().toByteArray())); return; } store.completeCompactionMarker(compaction); }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2231. * * @param compaction */ @Override public void completeCompactionMarker(CompactionDescriptor compaction) throws IOException { LOG.debug("Completing compaction from the WAL marker"); List<String> compactionInputs = compaction.getCompactionInputList(); // The Compaction Marker is written after the compaction is completed, // and the files moved into the region/family folder. // // If we crash after the entry is written, we may not have removed the // input files, but the output file is present. // (The unremoved input files will be removed by this function) // // If we scan the directory and the file is not present, it can mean that: // - The file was manually removed by the user // - The file was removed as consequence of subsequent compaction // so, we can't do anything with the "compaction output list" because those // files have already been loaded when opening the region (by virtue of // being in the store's folder) or they may be missing due to a compaction. String familyName = this.getColumnFamilyName(); List<Path> inputPaths = new ArrayList<Path>(compactionInputs.size()); for (String compactionInput : compactionInputs) { Path inputPath = fs.getStoreFilePath(familyName, compactionInput); inputPaths.add(inputPath); } //some of the input files might already be deleted List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size()); for (StoreFile sf : this.getStorefiles()) { if (inputPaths.contains(sf.getQualifiedPath())) { inputStoreFiles.add(sf); } } this.replaceStoreFiles(inputStoreFiles, Collections.EMPTY_LIST); this.completeCompaction(inputStoreFiles); }
/** * Deserialized and returns a CompactionDescriptor is the KeyValue contains one. * @param kv the key value * @return deserialized CompactionDescriptor or null. */ public static CompactionDescriptor getCompaction(KeyValue kv) throws IOException { if (kv.matchingRow(METAROW) && kv.matchingColumn(METAFAMILY, COMPACTION)) { return CompactionDescriptor.parseFrom(kv.getValue()); } return null; }
/** * Write the marker that a compaction has succeeded and is about to be committed. * This provides info to the HMaster to allow it to recover the compaction if * this regionserver dies in the middle (This part is not yet implemented). It also prevents * the compaction from finishing if this regionserver has already lost its lease on the log. * @param sequenceId Used by HLog to get sequence Id for the waledit. */ public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info, final CompactionDescriptor c, AtomicLong sequenceId) throws IOException { WALEdit e = WALEdit.createCompaction(c); long now = EnvironmentEdgeManager.currentTimeMillis(); TableName tn = TableName.valueOf(c.getTableName().toByteArray()); long txid = log.appendNoSync(info, tn, e, new ArrayList<UUID>(), now, htd, sequenceId, false, HConstants.NO_NONCE, HConstants.NO_NONCE); log.sync(txid); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2331. * @param compaction */ void completeCompactionMarker(CompactionDescriptor compaction) throws IOException { Store store = this.getStore(compaction.getFamilyName().toByteArray()); if (store == null) { LOG.warn("Found Compaction WAL edit for deleted family:" + Bytes.toString(compaction.getFamilyName().toByteArray())); return; } store.completeCompactionMarker(compaction); }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2231. * @param compaction */ @Override public void completeCompactionMarker(CompactionDescriptor compaction) throws IOException { LOG.debug("Completing compaction from the WAL marker"); List<String> compactionInputs = compaction.getCompactionInputList(); // The Compaction Marker is written after the compaction is completed, // and the files moved into the region/family folder. // // If we crash after the entry is written, we may not have removed the // input files, but the output file is present. // (The unremoved input files will be removed by this function) // // If we scan the directory and the file is not present, it can mean that: // - The file was manually removed by the user // - The file was removed as consequence of subsequent compaction // so, we can't do anything with the "compaction output list" because those // files have already been loaded when opening the region (by virtue of // being in the store's folder) or they may be missing due to a compaction. String familyName = this.getColumnFamilyName(); List<Path> inputPaths = new ArrayList<Path>(compactionInputs.size()); for (String compactionInput : compactionInputs) { Path inputPath = fs.getStoreFilePath(familyName, compactionInput); inputPaths.add(inputPath); } //some of the input files might already be deleted List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size()); for (StoreFile sf : this.getStorefiles()) { if (inputPaths.contains(sf.getQualifiedPath())) { inputStoreFiles.add(sf); } } this.replaceStoreFiles(inputStoreFiles, Collections.EMPTY_LIST); this.completeCompaction(inputStoreFiles); }
/** * Deserialized and returns a CompactionDescriptor is the KeyValue contains one. * @param kv the key value * @return deserialized CompactionDescriptor or null. */ public static CompactionDescriptor getCompaction(KeyValue kv) throws IOException { if (CellUtil.matchingRow(kv, METAROW) && CellUtil.matchingColumn(kv, METAFAMILY, COMPACTION)) { return CompactionDescriptor.parseFrom(kv.getValue()); } return null; }
/** * Write the marker that a compaction has succeeded and is about to be committed. * This provides info to the HMaster to allow it to recover the compaction if * this regionserver dies in the middle (This part is not yet implemented). It also prevents * the compaction from finishing if this regionserver has already lost its lease on the log. * @param sequenceId Used by HLog to get sequence Id for the waledit. */ public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info, final CompactionDescriptor c, AtomicLong sequenceId) throws IOException { TableName tn = TableName.valueOf(c.getTableName().toByteArray()); HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn); log.appendNoSync(htd, info, key, WALEdit.createCompaction(c), sequenceId, false); log.sync(); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } }
/** * Write the marker that a compaction has succeeded and is about to be committed. * This provides info to the HMaster to allow it to recover the compaction if * this regionserver dies in the middle (This part is not yet implemented). It also prevents * the compaction from finishing if this regionserver has already lost its lease on the log. */ public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info, final CompactionDescriptor c) throws IOException { WALEdit e = WALEdit.createCompaction(c); log.append(info, TableName.valueOf(c.getTableName().toByteArray()), e, EnvironmentEdgeManager.currentTimeMillis(), htd, false); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2331. * @param fs * @param compaction */ void completeCompactionMarker(CompactionDescriptor compaction) throws IOException { Store store = this.getStore(compaction.getFamilyName().toByteArray()); if (store == null) { LOG.warn("Found Compaction WAL edit for deleted family:" + Bytes.toString(compaction.getFamilyName().toByteArray())); return; } store.completeCompactionMarker(compaction); }
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction that was * not finished. We could find one recovering a WAL after a regionserver crash. See HBASE-2331. */ void replayWALCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles, boolean removeFiles, long replaySeqId) throws IOException { try { checkTargetRegion(compaction.getEncodedRegionName().toByteArray(), "Compaction marker from WAL ", compaction); } catch (WrongRegionException wre) { if (RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) { // skip the compaction marker since it is not for this region return; } throw wre; } synchronized (writestate) { if (replaySeqId < lastReplayedOpenRegionSeqId) { LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction) + " because its sequence id " + replaySeqId + " is smaller than this regions " + "lastReplayedOpenRegionSeqId of " + lastReplayedOpenRegionSeqId); return; } if (replaySeqId < lastReplayedCompactionSeqId) { LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction) + " because its sequence id " + replaySeqId + " is smaller than this regions " + "lastReplayedCompactionSeqId of " + lastReplayedCompactionSeqId); return; } else { lastReplayedCompactionSeqId = replaySeqId; } if (LOG.isDebugEnabled()) { LOG.debug( getRegionInfo().getEncodedName() + " : " + "Replaying compaction marker " + TextFormat .shortDebugString(compaction) + " with seqId=" + replaySeqId + " and lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId); } startRegionOperation(Operation.REPLAY_EVENT); try { Store store = this.getStore(compaction.getFamilyName().toByteArray()); if (store == null) { LOG.warn(getRegionInfo().getEncodedName() + " : " + "Found Compaction WAL edit for deleted family:" + Bytes .toString(compaction.getFamilyName().toByteArray())); return; } store.replayCompactionMarker(compaction, pickCompactionFiles, removeFiles); logRegionFiles(); } catch (FileNotFoundException ex) { LOG.warn(getRegionInfo().getEncodedName() + " : " + "At least one of the store files in compaction: " + TextFormat .shortDebugString(compaction) + " doesn't exist any more. Skip loading the file(s)", ex); } finally { closeRegionOperation(Operation.REPLAY_EVENT); } } }
/** * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse. * @param region * @param mutations * @param replaySeqId * @return an array of OperationStatus which internally contains the OperationStatusCode and the * exceptionMessage if any * @throws IOException */ private OperationStatus [] doReplayBatchOp(final Region region, final List<WALSplitter.MutationReplay> mutations, long replaySeqId) throws IOException { long before = EnvironmentEdgeManager.currentTime(); boolean batchContainsPuts = false, batchContainsDelete = false; try { for (Iterator<WALSplitter.MutationReplay> it = mutations.iterator(); it.hasNext();) { WALSplitter.MutationReplay m = it.next(); if (m.type == MutationType.PUT) { batchContainsPuts = true; } else { batchContainsDelete = true; } NavigableMap<byte[], List<Cell>> map = m.mutation.getFamilyCellMap(); List<Cell> metaCells = map.get(WALEdit.METAFAMILY); if (metaCells != null && !metaCells.isEmpty()) { for (Cell metaCell : metaCells) { CompactionDescriptor compactionDesc = WALEdit.getCompaction(metaCell); boolean isDefaultReplica = RegionReplicaUtil.isDefaultReplica(region.getRegionInfo()); HRegion hRegion = (HRegion)region; if (compactionDesc != null) { // replay the compaction. Remove the files from stores only if we are the primary // region replica (thus own the files) hRegion.replayWALCompactionMarker(compactionDesc, !isDefaultReplica, isDefaultReplica, replaySeqId); continue; } FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(metaCell); if (flushDesc != null && !isDefaultReplica) { hRegion.replayWALFlushMarker(flushDesc, replaySeqId); continue; } RegionEventDescriptor regionEvent = WALEdit.getRegionEventDescriptor(metaCell); if (regionEvent != null && !isDefaultReplica) { hRegion.replayWALRegionEventMarker(regionEvent); continue; } BulkLoadDescriptor bulkLoadEvent = WALEdit.getBulkLoadDescriptor(metaCell); if (bulkLoadEvent != null) { hRegion.replayWALBulkLoadEventMarker(bulkLoadEvent); continue; } } it.remove(); } } requestCount.add(mutations.size()); if (!region.getRegionInfo().isMetaTable()) { regionServer.cacheFlusher.reclaimMemStoreMemory(); } return region.batchReplay(mutations.toArray( new WALSplitter.MutationReplay[mutations.size()]), replaySeqId); } finally { if (regionServer.metricsRegionServer != null) { long after = EnvironmentEdgeManager.currentTime(); if (batchContainsPuts) { regionServer.metricsRegionServer.updatePut(after - before); } if (batchContainsDelete) { regionServer.metricsRegionServer.updateDelete(after - before); } } } }
public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) { return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir); }
/** * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse. * * @param region * @param mutations * @param replaySeqId * @return an array of OperationStatus which internally contains the OperationStatusCode and the * exceptionMessage if any * @throws IOException */ private OperationStatus[] doReplayBatchOp(final HRegion region, final List<WALSplitter.MutationReplay> mutations, long replaySeqId) throws IOException { long before = EnvironmentEdgeManager.currentTime(); boolean batchContainsPuts = false, batchContainsDelete = false; try { for (Iterator<WALSplitter.MutationReplay> it = mutations.iterator(); it.hasNext(); ) { WALSplitter.MutationReplay m = it.next(); if (m.type == MutationType.PUT) { batchContainsPuts = true; } else { batchContainsDelete = true; } NavigableMap<byte[], List<Cell>> map = m.mutation.getFamilyCellMap(); List<Cell> metaCells = map.get(WALEdit.METAFAMILY); if (metaCells != null && !metaCells.isEmpty()) { for (Cell metaCell : metaCells) { CompactionDescriptor compactionDesc = WALEdit.getCompaction(metaCell); if (compactionDesc != null) { region.completeCompactionMarker(compactionDesc); } } it.remove(); } } requestCount.add(mutations.size()); if (!region.getRegionInfo().isMetaTable()) { regionServer.cacheFlusher.reclaimMemStoreMemory(); } return region.batchReplay(mutations.toArray( new WALSplitter.MutationReplay[mutations.size()]), replaySeqId); } finally { if (regionServer.metricsRegionServer != null) { long after = EnvironmentEdgeManager.currentTime(); if (batchContainsPuts) { regionServer.metricsRegionServer.updatePut(after - before); } if (batchContainsDelete) { regionServer.metricsRegionServer.updateDelete(after - before); } } } }
@Test public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { String method = "testSkipRecoveredEditsReplayTheLastFileIgnored"; TableName tableName = TableName.valueOf(method); byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); final WALFactory wals = new WALFactory(CONF, null, method); try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); assertEquals(0, region.getStoreFileList( region.getStores().keySet().toArray(new byte[0][])).size()); Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); long maxSeqId = 1050; long minSeqId = 1000; for (long i = minSeqId; i <= maxSeqId; i += 10) { Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); fs.create(recoveredEdits); WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); long time = System.nanoTime(); WALEdit edit = null; if (i == maxSeqId) { edit = WALEdit.createCompaction(region.getRegionInfo(), CompactionDescriptor.newBuilder() .setTableName(ByteString.copyFrom(tableName.getName())) .setFamilyName(ByteString.copyFrom(regionName)) .setEncodedRegionName(ByteString.copyFrom(regionName)) .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())) .build()); } else { edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); } writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); } long recoverSeqId = 1030; Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR); MonitoredTask status = TaskMonitor.get().createStatus(method); for (Store store : region.getStores().values()) { maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1); } long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status); assertEquals(maxSeqId, seqId); // assert that the files are flushed assertEquals(1, region.getStoreFileList( region.getStores().keySet().toArray(new byte[0][])).size()); } finally { HRegion.closeHRegion(this.region); this.region = null; wals.close(); } }
/** * Create a compacion WALEdit * @param c * @return A WALEdit that has <code>c</code> serialized as its value */ public static WALEdit createCompaction(final CompactionDescriptor c) { byte [] pbbytes = c.toByteArray(); KeyValue kv = new KeyValue(METAROW, METAFAMILY, COMPACTION, System.currentTimeMillis(), pbbytes); return new WALEdit().add(kv); //replication scope null so that this won't be replicated }
@Test public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { String method = "testSkipRecoveredEditsReplayTheLastFileIgnored"; TableName tableName = TableName.valueOf(method); byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); assertEquals(0, region.getStoreFileList( region.getStores().keySet().toArray(new byte[0][])).size()); Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir); long maxSeqId = 1050; long minSeqId = 1000; for (long i = minSeqId; i <= maxSeqId; i += 10) { Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); fs.create(recoveredEdits); HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF); long time = System.nanoTime(); WALEdit edit = null; if (i == maxSeqId) { edit = WALEdit.createCompaction(CompactionDescriptor.newBuilder() .setTableName(ByteString.copyFrom(tableName.getName())) .setFamilyName(ByteString.copyFrom(regionName)) .setEncodedRegionName(ByteString.copyFrom(regionName)) .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) .build()); } else { edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); } writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); } long recoverSeqId = 1030; Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR); MonitoredTask status = TaskMonitor.get().createStatus(method); for (Store store : region.getStores().values()) { maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1); } long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status); assertEquals(maxSeqId, seqId); // assert that the files are flushed assertEquals(1, region.getStoreFileList( region.getStores().keySet().toArray(new byte[0][])).size()); } finally { HRegion.closeHRegion(this.region); this.region = null; } }
/** * Call to complete a compaction. Its for the case where we find in the WAL a * compaction that was not finished. We could find one recovering a WAL after * a regionserver crash. See HBASE-2331. * * @param compaction the descriptor for compaction * @param pickCompactionFiles whether or not pick up the new compaction output * files and add it to the store * @param removeFiles whether to remove/archive files from filesystem */ void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles, boolean removeFiles) throws IOException;
/** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2331. * @param compaction */ void completeCompactionMarker(CompactionDescriptor compaction) throws IOException;
