/** * This method should only be called <b>once</b>, as it starts a thread to keep the cache * up-to-date. * <p> * {@inheritDoc} */ @Override public void setConf(Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong( WAL_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_WAL_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getWALNames(fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create snapshot log cleaner", e); } }
public void setConf(final Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create cleaner util", e); } }
/** * This method should only be called <b>once</b>, as it starts a thread to keep the cache * up-to-date. * <p> * {@inheritDoc} */ @Override public void setConf(Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong( HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create snapshot log cleaner", e); } }
@Override public void setConf(Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create cleaner util", e); } }
/** * Corrupt the specified snapshot by deleting some files. * * @param util {@link HBaseTestingUtility} * @param snapshotName name of the snapshot to corrupt * @return array of the corrupted HFiles * @throws IOException on unexecpted error reading the FS */ public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) throws IOException { final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); final FileSystem fs = mfs.getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); final String table = snapshotDesc.getTable(); final ArrayList corruptedFiles = new ArrayList(); SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs)); corruptedFiles.add(hfile); } } }); assertTrue(corruptedFiles.size() > 0); return corruptedFiles; }
@Override public void setConf(final Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { @Override public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create cleaner util", e); } }
@Override public void setConf(final Configuration conf) { super.setConf(conf); try { long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HFILE_CACHE_REFRESH_PERIOD); final FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path rootDir = FSUtils.getRootDir(conf); cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); } }); } catch (IOException e) { LOG.error("Failed to create cleaner util", e); } }
/** * Verify that the region (regioninfo, hfiles) are valid * @param fs the FileSystem instance * @param snapshotDir snapshot directory to check * @param region the region to check */ private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region) throws IOException { // make sure we have region in the snapshot Path regionDir = new Path(snapshotDir, region.getEncodedName()); // make sure we have the region info in the snapshot Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE); // make sure the file exists if (!fs.exists(regionInfo)) { throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot); } FSDataInputStream in = fs.open(regionInfo); HRegionInfo found = new HRegionInfo(); try { found.readFields(in); if (!region.equals(found)) { throw new CorruptedSnapshotException("Found region info (" + found + ") doesn't match expected region:" + region, snapshot); } } finally { in.close(); } // make sure we have the expected recovered edits files TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot); // make sure we have all the expected store files SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() { public void storeFile(final String regionNameSuffix, final String family, final String hfileName) throws IOException { verifyStoreFile(snapshotDir, region, family, hfileName); } }); }
/** * Add the specified recovered.edits file to the stats * @param region region encoded name * @param logfile log file name * @return the recovered.edits information */ FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException { Path rootDir = FSUtils.getRootDir(conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile); long size = fs.getFileStatus(path).getLen(); logSize += size; logsCount++; return new FileInfo(true, size); }
private void init() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); //load table descriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir); Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir); if (snapshotRegionNames == null) { throw new IllegalArgumentException("Snapshot seems empty"); } regions = new ArrayList<HRegionInfo>(snapshotRegionNames.size()); for (String regionName : snapshotRegionNames) { // load region descriptor Path regionDir = new Path(snapshotDir, regionName); HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { regions.add(hri); } } // sort for regions according to startKey. Collections.sort(regions); initScanMetrics(scan); RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param snapshotDir snapshot directory to check * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(Path snapshotDir) throws IOException { List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(), tableName); Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir); if (snapshotRegions == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (snapshotRegions.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + snapshotRegions.size() + "."; LOG.error(errorMsg); } for (HRegionInfo region : regions) { if (!snapshotRegions.contains(region.getEncodedName())) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); } verifyRegion(fs, snapshotDir, region); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } }
/** * Verify that the region (regioninfo, hfiles) are valid * @param fs the FileSystem instance * @param snapshotDir snapshot directory to check * @param region the region to check */ private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region) throws IOException { // make sure we have region in the snapshot Path regionDir = new Path(snapshotDir, region.getEncodedName()); // make sure we have the region info in the snapshot Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); // make sure the file exists if (!fs.exists(regionInfo)) { throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot); } HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); if (!region.equals(found)) { throw new CorruptedSnapshotException("Found region info (" + found + ") doesn't match expected region:" + region, snapshot); } // make sure we have the expected recovered edits files TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot); // make sure we have all the expected store files SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() { public void storeFile(final String regionNameSuffix, final String family, final String hfileName) throws IOException { verifyStoreFile(snapshotDir, region, family, hfileName); } }); }
/** * Corrupt the specified snapshot by deleting some files. * * @param util {@link HBaseTestingUtility} * @param snapshotName name of the snapshot to corrupt * @return array of the corrupted HFiles * @throws IOException on unexecpted error reading the FS */ public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) throws IOException { final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); final FileSystem fs = mfs.getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); final TableName table = TableName.valueOf(snapshotDesc.getTable()); final ArrayList corruptedFiles = new ArrayList(); SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { @Override public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs)); corruptedFiles.add(hfile); } } }); assertTrue(corruptedFiles.size() > 0); return corruptedFiles; }
/** * Restore or Clone the specified snapshot * @param reqSnapshot * @param nonceKey unique identifier to prevent duplicated RPC * @throws IOException */ public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey, final boolean restoreAcl) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); // check if the snapshot exists if (!fs.exists(snapshotDir)) { LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist."); throw new SnapshotDoesNotExistException( ProtobufUtil.createSnapshotDesc(reqSnapshot)); } // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot // information. SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot); TableDescriptor snapshotTableDesc = manifest.getTableDescriptor(); TableName tableName = TableName.valueOf(reqSnapshot.getTable()); // stop tracking "abandoned" handlers cleanupSentinels(); // Verify snapshot validity SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest); // Execute the restore/clone operation long procId; if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) { procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl); } else { procId = cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl); } return procId; }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param manifest snapshot manifest to inspect * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(final SnapshotManifest manifest) throws IOException { List<HRegionInfo> regions; if (TableName.META_TABLE_NAME.equals(tableName)) { regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper()); } else { regions = MetaTableAccessor.getTableRegions(services.getZooKeeper(), services.getConnection(), tableName); } // Remove the non-default regions RegionReplicaUtil.removeNonDefaultRegions(regions); Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); if (regionManifests == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (regionManifests.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + regionManifests.size() + "."; LOG.error(errorMsg); } // Verify HRegionInfo for (HRegionInfo region : regions) { SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName()); if (regionManifest == null) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); continue; } verifyRegionInfo(region, regionManifest); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } // Verify Snapshot HFiles SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest); }
