/** * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, * instead of an IOException */ @Test public void testCorruptedRegionManifest() throws IOException { SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( SNAPSHOT_NAME_STR, TABLE_NAME_STR); builder.addRegionV2(); builder.corruptOneRegionManifest(); long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); try { cache.getSnapshotsInProgress(null); } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } }
/** * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, * instead of an IOException */ @Test public void testCorruptedDataManifest() throws IOException { SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( SNAPSHOT_NAME_STR, TABLE_NAME_STR); builder.addRegionV2(); // consolidate to generate a data.manifest file builder.consolidate(); builder.corruptDataManifest(); long period = Long.MAX_VALUE; SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles()); try { cache.getSnapshotsInProgress(null); } catch (CorruptedSnapshotException cse) { LOG.info("Expected exception " + cse); } finally { fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); } }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link org.apache.hadoop.hbase.ServerName} * of the servers that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
/** * Check that the snapshot description written in the filesystem matches the current snapshot * @param snapshotDir snapshot directory to check */ private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException { SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); if (!this.snapshot.equals(found)) { throw new CorruptedSnapshotException("Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot); } }
/** * Check that the table descriptor for the snapshot is a valid table descriptor * @param manifest snapshot manifest to inspect */ private void verifyTableInfo(final SnapshotManifest manifest) throws IOException { HTableDescriptor htd = manifest.getTableDescriptor(); if (htd == null) { throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot); } if (!htd.getNameAsString().equals(snapshot.getTable())) { throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot); } }
/** * Verify that the regionInfo is valid * @param region the region to check * @param manifest snapshot manifest to inspect */ private void verifyRegionInfo(final HRegionInfo region, final SnapshotRegionManifest manifest) throws IOException { HRegionInfo manifestRegionInfo = HRegionInfo.convert(manifest.getRegionInfo()); if (!region.equals(manifestRegionInfo)) { String msg = "Manifest region info " + manifestRegionInfo + "doesn't match expected region:" + region; throw new CorruptedSnapshotException(msg, snapshot); } }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(snapshotDir); // check that each region is valid verifyRegions(snapshotDir); }
/** * Verify that the region (regioninfo, hfiles) are valid * @param fs the FileSystem instance * @param snapshotDir snapshot directory to check * @param region the region to check */ private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region) throws IOException { // make sure we have region in the snapshot Path regionDir = new Path(snapshotDir, region.getEncodedName()); // make sure we have the region info in the snapshot Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE); // make sure the file exists if (!fs.exists(regionInfo)) { throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot); } FSDataInputStream in = fs.open(regionInfo); HRegionInfo found = new HRegionInfo(); try { found.readFields(in); if (!region.equals(found)) { throw new CorruptedSnapshotException("Found region info (" + found + ") doesn't match expected region:" + region, snapshot); } } finally { in.close(); } // make sure we have the expected recovered edits files TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot); // make sure we have all the expected store files SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() { public void storeFile(final String regionNameSuffix, final String family, final String hfileName) throws IOException { verifyStoreFile(snapshotDir, region, family, hfileName); } }); }
private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo, final String family, final String fileName) throws IOException { Path refPath = null; if (StoreFile.isReference(fileName)) { // If is a reference file check if the parent file is present in the snapshot Path snapshotHFilePath = new Path(new Path( new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName); refPath = StoreFile.getReferredToFile(snapshotHFilePath); if (!fs.exists(refPath)) { throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot); } } Path linkPath; if (refPath != null && HFileLink.isHFileLink(refPath)) { linkPath = new Path(family, refPath.getName()); } else if (HFileLink.isHFileLink(fileName)) { linkPath = new Path(family, fileName); } else { linkPath = new Path(family, HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), fileName)); } // check if the linked file exists (in the archive, or in the table dir) HFileLink link = new HFileLink(services.getConfiguration(), linkPath); if (!link.exists(fs)) { throw new CorruptedSnapshotException("Can't find hfile: " + fileName + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath() + ") directory for the primary table.", snapshot); } }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param snapshotDir snapshot directory to check * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(Path snapshotDir) throws IOException { List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(), tableName); Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir); if (snapshotRegions == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (snapshotRegions.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + snapshotRegions.size() + "."; LOG.error(errorMsg); } for (HRegionInfo region : regions) { if (!snapshotRegions.contains(region.getEncodedName())) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); } verifyRegion(fs, snapshotDir, region); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } }
/** * Verify that the region (regioninfo, hfiles) are valid * @param fs the FileSystem instance * @param snapshotDir snapshot directory to check * @param region the region to check */ private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region) throws IOException { // make sure we have region in the snapshot Path regionDir = new Path(snapshotDir, region.getEncodedName()); // make sure we have the region info in the snapshot Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); // make sure the file exists if (!fs.exists(regionInfo)) { throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot); } HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); if (!region.equals(found)) { throw new CorruptedSnapshotException("Found region info (" + found + ") doesn't match expected region:" + region, snapshot); } // make sure we have the expected recovered edits files TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot); // make sure we have all the expected store files SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() { public void storeFile(final String regionNameSuffix, final String family, final String hfileName) throws IOException { verifyStoreFile(snapshotDir, region, family, hfileName); } }); }
private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo, final String family, final String fileName) throws IOException { Path refPath = null; if (StoreFileInfo.isReference(fileName)) { // If is a reference file check if the parent file is present in the snapshot Path snapshotHFilePath = new Path(new Path( new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName); refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath); if (!fs.exists(refPath)) { throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot); } } Path linkPath; if (refPath != null && HFileLink.isHFileLink(refPath)) { linkPath = new Path(family, refPath.getName()); } else if (HFileLink.isHFileLink(fileName)) { linkPath = new Path(family, fileName); } else { linkPath = new Path(family, HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), fileName)); } // check if the linked file exists (in the archive, or in the table dir) HFileLink link = new HFileLink(services.getConfiguration(), linkPath); if (!link.exists(fs)) { throw new CorruptedSnapshotException("Can't find hfile: " + fileName + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath() + ") directory for the primary table.", snapshot); } }
@VisibleForTesting List<String> getSnapshotsInProgress( final SnapshotManager snapshotManager) throws IOException { List<String> snapshotInProgress = Lists.newArrayList(); // only add those files to the cache, but not to the known snapshots Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); // only add those files to the cache, but not to the known snapshots FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir); if (running != null) { for (FileStatus run : running) { ReentrantLock lock = null; if (snapshotManager != null) { lock = snapshotManager.getLocks().acquireLock(run.getPath().getName()); } try { snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); } catch (CorruptedSnapshotException e) { // See HBASE-16464 if (e.getCause() instanceof FileNotFoundException) { // If the snapshot is corrupt, we will delete it fs.delete(run.getPath(), true); LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause()); } else { throw e; } } finally { if (lock != null) { lock.unlock(); } } } } return snapshotInProgress; }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link org.apache.hadoop.hbase.ServerName} of the servers * that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
/** * Check that the snapshot description written in the filesystem matches the current snapshot * @param snapshotDir snapshot directory to check */ private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException { SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); if (!this.snapshot.equals(found)) { throw new CorruptedSnapshotException( "Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").", ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Check that the table descriptor for the snapshot is a valid table descriptor * @param manifest snapshot manifest to inspect */ private void verifyTableInfo(final SnapshotManifest manifest) throws IOException { TableDescriptor htd = manifest.getTableDescriptor(); if (htd == null) { throw new CorruptedSnapshotException("Missing Table Descriptor", ProtobufUtil.createSnapshotDesc(snapshot)); } if (!htd.getTableName().getNameAsString().equals(snapshot.getTable())) { throw new CorruptedSnapshotException( "Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got " + htd.getTableName().getNameAsString(), ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Verify that the regionInfo is valid * @param region the region to check * @param manifest snapshot manifest to inspect */ private void verifyRegionInfo(final RegionInfo region, final SnapshotRegionManifest manifest) throws IOException { RegionInfo manifestRegionInfo = ProtobufUtil.toRegionInfo(manifest.getRegionInfo()); if (RegionInfo.COMPARATOR.compare(region, manifestRegionInfo) != 0) { String msg = "Manifest region info " + manifestRegionInfo + "doesn't match expected region:" + region; throw new CorruptedSnapshotException(msg, ProtobufUtil.createSnapshotDesc(snapshot)); } }
@Override public synchronized Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) { try { return cache.getUnreferencedFiles(files, master.getSnapshotManager()); } catch (CorruptedSnapshotException cse) { LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse); } catch (IOException e) { LOG.error("Exception while checking if files were valid, keeping them just in case.", e); } return Collections.emptyList(); }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
