/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param storeFile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size, isCorrupted); }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size, isCorrupted); }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size); }
/** * Verify that the regionInfo is valid * @param region the region to check * @param manifest snapshot manifest to inspect */ private void verifyRegionInfo(final HRegionInfo region, final SnapshotRegionManifest manifest) throws IOException { HRegionInfo manifestRegionInfo = HRegionInfo.convert(manifest.getRegionInfo()); if (!region.equals(manifestRegionInfo)) { String msg = "Manifest region info " + manifestRegionInfo + "doesn't match expected region:" + region; throw new CorruptedSnapshotException(msg, snapshot); } }
public static List<HRegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) { List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size()); for (SnapshotRegionManifest regionManifest : regionManifests) { regionInfos.add(HRegionInfo.convert(regionManifest.getRegionInfo())); } return regionInfos; }
/** * Get all the Region Manifest from the snapshot. * This is an helper to get a map with the region encoded name */ public Map<String, SnapshotRegionManifest> getRegionManifestsMap() { if (regionManifests == null || regionManifests.size() == 0) return null; HashMap<String, SnapshotRegionManifest> regionsMap = new HashMap<String, SnapshotRegionManifest>(regionManifests.size()); for (SnapshotRegionManifest manifest: regionManifests) { String regionName = getRegionNameFromManifest(manifest); regionsMap.put(regionName, manifest); } return regionsMap; }
/** * Extract the region encoded name from the region manifest */ static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) { byte[] regionName = HRegionInfo.createRegionName( ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()), manifest.getRegionInfo().getStartKey().toByteArray(), manifest.getRegionInfo().getRegionId(), true); return HRegionInfo.encodeRegionName(regionName); }
/**© * Iterate over the snapshot store files * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param desc the {@link SnapshotDescription} of the snapshot to verify * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ static void visitTableStoreFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor) throws IOException { SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null || regionManifests.size() == 0) { LOG.debug("No manifest files present: " + snapshotDir); return; } for (SnapshotRegionManifest regionManifest: regionManifests) { visitRegionStoreFiles(regionManifest, visitor); } }
/** * Iterate over the snapshot store files in the specified region * * @param manifest snapshot manifest to inspect * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ static void visitRegionStoreFiles(final SnapshotRegionManifest manifest, final StoreFileVisitor visitor) throws IOException { HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo()); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { String familyName = familyFiles.getFamilyName().toStringUtf8(); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { visitor.storeFile(regionInfo, familyName, storeFile); } } }
public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException { SnapshotRegionManifest manifest = region.build(); FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest)); try { manifest.writeTo(stream); } finally { stream.close(); } }
public SnapshotRegionManifest.FamilyFiles.Builder familyOpen( final SnapshotRegionManifest.Builder region, final byte[] familyName) { SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder(); family.setFamilyName(ByteStringer.wrap(familyName)); return family; }
/** * Restore specified regions by restoring content to the snapshot state. */ private void restoreHdfsRegions(final ThreadPoolExecutor exec, final Map<String, SnapshotRegionManifest> regionManifests, final List<HRegionInfo> regions) throws IOException { if (regions == null || regions.size() == 0) return; ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { @Override public void editRegion(final HRegionInfo hri) throws IOException { restoreRegion(hri, regionManifests.get(hri.getEncodedName())); } }); }
private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences( final SnapshotRegionManifest manifest) { Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap = new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount()); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { familyMap.put(familyFiles.getFamilyName().toStringUtf8(), new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList())); } return familyMap; }
/** * Clone specified regions. For each region create a new region * and create a HFileLink for each hfile. */ private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec, final Map<String, SnapshotRegionManifest> regionManifests, final List<HRegionInfo> regions) throws IOException { if (regions == null || regions.size() == 0) return null; final Map<String, HRegionInfo> snapshotRegions = new HashMap<String, HRegionInfo>(regions.size()); // clone region info (change embedded tableName with the new one) HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()]; for (int i = 0; i < clonedRegionsInfo.length; ++i) { // clone the region info from the snapshot region info HRegionInfo snapshotRegionInfo = regions.get(i); clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo); // add the region name mapping between snapshot and cloned String snapshotRegionName = snapshotRegionInfo.getEncodedName(); String clonedRegionName = clonedRegionsInfo[i].getEncodedName(); regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName)); LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName); // Add mapping between cloned region name and snapshot region info snapshotRegions.put(clonedRegionName, snapshotRegionInfo); } // create the regions on disk ModifyRegionUtils.createRegions(exec, conf, rootDir, tableDir, tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() { @Override public void fillRegion(final HRegion region) throws IOException { HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName()); cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName())); } }); return clonedRegionsInfo; }
/** * Clone region directory content from the snapshot info. * * Each region is encoded with the table name, so the cloned region will have * a different region name. * * Instead of copying the hfiles a HFileLink is created. * * @param region {@link HRegion} cloned * @param snapshotRegionInfo */ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); } } }
private void init() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); // load table descriptor htd = manifest.getTableDescriptor(); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } regions = new ArrayList<HRegionInfo>(regionManifests.size()); for (SnapshotRegionManifest regionManifest : regionManifests) { // load region descriptor HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { regions.add(hri); } } // sort for regions according to startKey. Collections.sort(regions); initScanMetrics(scan); RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); }
/** * Clone region directory content from the snapshot info. * * Each region is encoded with the table name, so the cloned region will have * a different region name. * * Instead of copying the hfiles a HFileLink is created. * * @param region {@link HRegion} cloned * @param snapshotRegionInfo */ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile); } } }
public SnapshotRegionManifest.FamilyFiles.Builder familyOpen( final SnapshotRegionManifest.Builder region, final byte[] familyName) { SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder(); family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName)); return family; }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param manifest snapshot manifest to inspect * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(final SnapshotManifest manifest) throws IOException { List<HRegionInfo> regions; if (TableName.META_TABLE_NAME.equals(tableName)) { regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper()); } else { regions = MetaTableAccessor.getTableRegions(services.getZooKeeper(), services.getConnection(), tableName); } // Remove the non-default regions RegionReplicaUtil.removeNonDefaultRegions(regions); Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); if (regionManifests == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (regionManifests.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + regionManifests.size() + "."; LOG.error(errorMsg); } // Verify HRegionInfo for (HRegionInfo region : regions) { SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName()); if (regionManifest == null) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); continue; } verifyRegionInfo(region, regionManifest); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } // Verify Snapshot HFiles SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest); }