public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); return getSplits(scan, manifest, regionInfos, restoreDir, conf); }
/** * Get table descriptor * @param tableName is the table backed up * @return {@link TableDescriptor} saved in backup image of the table */ TableDescriptor getTableDesc(TableName tableName) throws IOException { Path tableInfoPath = this.getTableInfoPath(tableName); SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); TableDescriptor tableDescriptor = manifest.getTableDescriptor(); if (!tableDescriptor.getTableName().equals(tableName)) { LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " + tableInfoPath.toString()); LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getTableName().getNameAsString()); throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " + tableInfoPath.toString()); } return tableDescriptor; }
private void openWithoutRestoringSnapshot() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName); } regions = new ArrayList<>(regionManifests.size()); regionManifests.stream().map(r -> HRegionInfo.convert(r.getRegionInfo())) .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r)); htd = manifest.getTableDescriptor(); }
/** * Action before cloning from snapshot. * @param env MasterProcedureEnv * @throws IOException * @throws InterruptedException */ private void preCloneSnapshot(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { // Check and update namespace quota final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); ProcedureSyncWait.getMasterQuotaManager(env) .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); } final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableAction(tableDescriptor, null, getUser()); } }
public static List<HRegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) { List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size()); for (SnapshotRegionManifest regionManifest : regionManifests) { HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) { continue; } regionInfos.add(hri); } return regionInfos; }
public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final HTableDescriptor tableDescriptor, final Path rootDir, final ForeignExceptionDispatcher monitor, final MonitoredTask status) { this.fs = fs; this.conf = conf; this.snapshotManifest = manifest; this.snapshotDesc = manifest.getSnapshotDescription(); this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); this.tableDesc = tableDescriptor; this.rootDir = rootDir; this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName()); this.monitor = monitor; this.status = status; }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link org.apache.hadoop.hbase.ServerName} * of the servers that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
/** * Check that the table descriptor for the snapshot is a valid table descriptor * @param manifest snapshot manifest to inspect */ private void verifyTableInfo(final SnapshotManifest manifest) throws IOException { HTableDescriptor htd = manifest.getTableDescriptor(); if (htd == null) { throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot); } if (!htd.getNameAsString().equals(snapshot.getTable())) { throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot); } }
private void checkAndUpdateNamespaceQuota(SnapshotManifest manifest, TableName tableName) throws IOException { if (this.master.getMasterQuotaManager().isQuotaEnabled()) { this.master.getMasterQuotaManager().checkNamespaceTableAndRegionQuota(tableName, manifest.getRegionManifestsMap().size()); } }
/** * Return the list of splits extracted from the scans/snapshots pushed to conf by * {@link * #setInput(org.apache.hadoop.conf.Configuration, java.util.Map, org.apache.hadoop.fs.Path)} * * @param conf Configuration to determine splits from * @return Return the list of splits extracted from the scans/snapshots pushed to conf * @throws IOException */ public List<TableSnapshotInputFormatImpl.InputSplit> getSplits(Configuration conf) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); List<TableSnapshotInputFormatImpl.InputSplit> rtn = Lists.newArrayList(); Map<String, Collection<Scan>> snapshotsToScans = getSnapshotsToScans(conf); Map<String, Path> snapshotsToRestoreDirs = getSnapshotDirs(conf); for (Map.Entry<String, Collection<Scan>> entry : snapshotsToScans.entrySet()) { String snapshotName = entry.getKey(); Path restoreDir = snapshotsToRestoreDirs.get(snapshotName); SnapshotManifest manifest = TableSnapshotInputFormatImpl.getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = TableSnapshotInputFormatImpl.getRegionInfosFromManifest(manifest); for (Scan scan : entry.getValue()) { List<TableSnapshotInputFormatImpl.InputSplit> splits = TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); rtn.addAll(splits); } } return rtn; }
public static List<HRegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) { List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size()); for (SnapshotRegionManifest regionManifest : regionManifests) { regionInfos.add(HRegionInfo.convert(regionManifest.getRegionInfo())); } return regionInfos; }
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf) throws IOException { // load table descriptor HTableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); List<InputSplit> splits = new ArrayList<InputSplit>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { // compute HDFS locations from snapshot files (which will get the locations for // referred hfiles) List<String> hosts = getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); int len = Math.min(3, hosts.size()); hosts = hosts.subList(0, len); splits.add(new InputSplit(htd, hri, hosts, scan, restoreDir)); } } return splits; }
private void init() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); // load table descriptor htd = manifest.getTableDescriptor(); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } regions = new ArrayList<HRegionInfo>(regionManifests.size()); for (SnapshotRegionManifest regionManifest : regionManifests) { // load region descriptor HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { regions.add(hri); } } // sort for regions according to startKey. Collections.sort(regions); initScanMetrics(scan); RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); }
/** * Extracts the names of the store files referenced by this snapshot which satisfy the given * predicate (the predicate returns {@code true}). */ Set<StoreFileReference> getStoreFilesFromSnapshot( SnapshotManifest manifest, Predicate<String> filter) { Set<StoreFileReference> references = new HashSet<>(); // For each region referenced by the snapshot for (SnapshotRegionManifest rm : manifest.getRegionManifests()) { StoreFileReference regionReference = new StoreFileReference( HRegionInfo.convert(rm.getRegionInfo()).getEncodedName()); // For each column family in this region for (FamilyFiles ff : rm.getFamilyFilesList()) { final String familyName = ff.getFamilyName().toStringUtf8(); // And each store file in that family for (StoreFile sf : ff.getStoreFilesList()) { String storeFileName = sf.getName(); // A snapshot only "inherits" a files size if it uniquely refers to it (no table // and no other snapshot references it). if (filter.test(storeFileName)) { regionReference.addFamilyStoreFile(familyName, storeFileName); } } } // Only add this Region reference if we retained any files. if (!regionReference.getFamilyToFilesMapping().isEmpty()) { references.add(regionReference); } } return references; }
/** * Action before any real action of restoring from snapshot. * @param env MasterProcedureEnv * @throws IOException */ private void prepareRestore(final MasterProcedureEnv env) throws IOException { final TableName tableName = getTableName(); // Checks whether the table exists if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) { throw new TableNotFoundException(tableName); } // Check whether table is disabled. env.getMasterServices().checkTableModifiable(tableName); // Check that we have at least 1 CF if (modifiedTableDescriptor.getColumnFamilyCount() == 0) { throw new DoNotRetryIOException("Table " + getTableName().toString() + " should have at least one column family."); } if (!getTableName().isSystemTable()) { // Table already exist. Check and update the region quota for this table namespace. final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); int snapshotRegionCount = manifest.getRegionManifestsMap().size(); int tableRegionCount = ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName); if (snapshotRegionCount > 0 && tableRegionCount != snapshotRegionCount) { ProcedureSyncWait.getMasterQuotaManager(env).checkAndUpdateNamespaceRegionQuota( tableName, snapshotRegionCount); } } }
/** * Execute the on-disk Restore * @param env MasterProcedureEnv * @throws IOException **/ private void restoreSnapshot(final MasterProcedureEnv env) throws IOException { MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem(); FileSystem fs = fileSystemManager.getFileSystem(); Path rootDir = fileSystemManager.getRootDir(); final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)); try { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot); RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( env.getMasterServices().getConfiguration(), fs, manifest, modifiedTableDescriptor, rootDir, monitorException, getMonitorStatus()); RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); regionsToRestore = metaChanges.getRegionsToRestore(); regionsToRemove = metaChanges.getRegionsToRemove(); regionsToAdd = metaChanges.getRegionsToAdd(); parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap(); } catch (IOException e) { String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " failed in on-disk restore. Try re-running the restore command."; LOG.error(msg, e); monitorException.receive( new ForeignException(env.getMasterServices().getServerName().toString(), e)); throw new IOException(msg, e); } }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link org.apache.hadoop.hbase.ServerName} of the servers * that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
/** * Check that the table descriptor for the snapshot is a valid table descriptor * @param manifest snapshot manifest to inspect */ private void verifyTableInfo(final SnapshotManifest manifest) throws IOException { TableDescriptor htd = manifest.getTableDescriptor(); if (htd == null) { throw new CorruptedSnapshotException("Missing Table Descriptor", ProtobufUtil.createSnapshotDesc(snapshot)); } if (!htd.getTableName().getNameAsString().equals(snapshot.getTable())) { throw new CorruptedSnapshotException( "Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got " + htd.getTableName().getNameAsString(), ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Restore or Clone the specified snapshot * @param reqSnapshot * @param nonceKey unique identifier to prevent duplicated RPC * @throws IOException */ public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey, final boolean restoreAcl) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); // check if the snapshot exists if (!fs.exists(snapshotDir)) { LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist."); throw new SnapshotDoesNotExistException( ProtobufUtil.createSnapshotDesc(reqSnapshot)); } // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot // information. SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot); TableDescriptor snapshotTableDesc = manifest.getTableDescriptor(); TableName tableName = TableName.valueOf(reqSnapshot.getTable()); // stop tracking "abandoned" handlers cleanupSentinels(); // Verify snapshot validity SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest); // Execute the restore/clone operation long procId; if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) { procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl); } else { procId = cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl); } return procId; }
/** * Verify that the snapshot in the directory is a valid snapshot * @param snapshotDir snapshot directory to check * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot * @throws CorruptedSnapshotException if the snapshot is invalid * @throws IOException if there is an unexpected connection issue to the filesystem */ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers) throws CorruptedSnapshotException, IOException { SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs, snapshotDir, snapshot); // verify snapshot info matches verifySnapshotDescription(snapshotDir); // check that tableinfo is a valid table description verifyTableInfo(manifest); // check that each region is valid verifyRegions(manifest); }
/** * Restore the on-disk table to a specified snapshot state. * @return the set of regions touched by the restore operation */ public RestoreMetaChanges restoreHdfsRegions() throws IOException { ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot"); try { return restoreHdfsRegions(exec); } finally { exec.shutdown(); } }
/** * Copy the snapshot files for a snapshot scanner, discards meta changes. * @param conf * @param fs * @param rootDir * @param restoreDir * @param snapshotName * @throws IOException */ public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir, Path restoreDir, String snapshotName) throws IOException { // ensure that restore dir is not under root dir if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) { throw new IllegalArgumentException("Filesystems for restore directory and HBase root directory " + "should be the same"); } if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) { throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir); } Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); MonitoredTask status = TaskMonitor.get().createStatus( "Restoring snapshot '" + snapshotName + "' to directory " + restoreDir); ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(); RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, manifest.getTableDescriptor(), restoreDir, monitor, status); helper.restoreHdfsRegions(); // TODO: parallelize. if (LOG.isDebugEnabled()) { LOG.debug("Restored table dir:" + restoreDir); FSUtils.logFileSystemState(fs, restoreDir, LOG); } }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param manifest snapshot manifest to inspect * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(final SnapshotManifest manifest) throws IOException { List<HRegionInfo> regions; if (TableName.META_TABLE_NAME.equals(tableName)) { regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper()); } else { regions = MetaTableAccessor.getTableRegions(services.getZooKeeper(), services.getConnection(), tableName); } // Remove the non-default regions RegionReplicaUtil.removeNonDefaultRegions(regions); Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); if (regionManifests == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (regionManifests.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + regionManifests.size() + "."; LOG.error(errorMsg); } // Verify HRegionInfo for (HRegionInfo region : regions) { SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName()); if (regionManifest == null) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); continue; } verifyRegionInfo(region, regionManifest); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } // Verify Snapshot HFiles SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest); }
/** * The restore table is executed in place. * - The on-disk data will be restored - reference files are put in place without moving data * - [if something fail here: you need to delete the table and re-run the restore] * - hbase:meta will be updated * - [if something fail here: you need to run hbck to fix hbase:meta entries] * The passed in list gets changed in this method */ @Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem(); Connection conn = masterServices.getConnection(); FileSystem fs = fileSystemManager.getFileSystem(); Path rootDir = fileSystemManager.getRootDir(); TableName tableName = hTableDescriptor.getTableName(); try { // 1. Update descriptor this.masterServices.getTableDescriptors().add(hTableDescriptor); // 2. Execute the on-disk Restore LOG.debug("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open(masterServices.getConfiguration(), fs, snapshotDir, snapshot); RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( masterServices.getConfiguration(), fs, manifest, this.hTableDescriptor, rootDir, monitor, status); RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); // 3. Forces all the RegionStates to be offline // // The AssignmentManager keeps all the region states around // with no possibility to remove them, until the master is restarted. // This means that a region marked as SPLIT before the restore will never be assigned again. // To avoid having all states around all the regions are switched to the OFFLINE state, // which is the same state that the regions will be after a delete table. forceRegionsOffline(metaChanges); // 4. Applies changes to hbase:meta status.setStatus("Preparing to restore each region"); // 4.1 Removes the current set of regions from META // // By removing also the regions to restore (the ones present both in the snapshot // and in the current state) we ensure that no extra fields are present in META // e.g. with a simple add addRegionToMeta() the splitA and splitB attributes // not overwritten/removed, so you end up with old informations // that are not correct after the restore. List<HRegionInfo> hrisToRemove = new LinkedList<HRegionInfo>(); if (metaChanges.hasRegionsToRemove()) hrisToRemove.addAll(metaChanges.getRegionsToRemove()); MetaTableAccessor.deleteRegions(conn, hrisToRemove); // 4.2 Add the new set of regions to META // // At this point the old regions are no longer present in META. // and the set of regions present in the snapshot will be written to META. // All the information in hbase:meta are coming from the .regioninfo of each region present // in the snapshot folder. hris.clear(); if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd()); MetaTableAccessor.addRegionsToMeta(conn, hris, hTableDescriptor.getRegionReplication()); if (metaChanges.hasRegionsToRestore()) { MetaTableAccessor.overwriteRegions(conn, metaChanges.getRegionsToRestore(), hTableDescriptor.getRegionReplication()); } metaChanges.updateMetaParentRegions(this.server.getConnection(), hris); // At this point the restore is complete. Next step is enabling the table. LOG.info("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + tableName + " completed!"); } catch (IOException e) { String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " failed. Try re-running the restore command."; LOG.error(msg, e); monitor.receive(new ForeignException(masterServices.getServerName().toString(), e)); throw new RestoreSnapshotException(msg, e); } }
@Override public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException, KeeperException { try { // 1. get all the regions hosting this table. // extract each pair to separate lists Set<HRegionInfo> regions = new HashSet<HRegionInfo>(); for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) { // Don't include non-default regions HRegionInfo hri = p.getFirst(); if (RegionReplicaUtil.isDefaultReplica(hri)) { regions.add(hri); } } // 2. for each region, write all the info to disk String msg = "Starting to write region info and WALs for regions for offline snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot); LOG.info(msg); status.setStatus(msg); ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot"); try { ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { @Override public void editRegion(final HRegionInfo regionInfo) throws IOException { snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo); } }); } finally { exec.shutdown(); } } catch (Exception e) { // make sure we capture the exception to propagate back to the client later String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); ForeignException ee = new ForeignException(reason, e); monitor.receive(ee); status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage()); } finally { LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot) + " as finished."); } }
public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, Path rootDir, FileSystem fs) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); }
/** * Restore the specified snapshot * @param reqSnapshot * @throws IOException */ public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); // check if the snapshot exists if (!fs.exists(snapshotDir)) { LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist."); throw new SnapshotDoesNotExistException(reqSnapshot); } // read snapshot information SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, fsSnapshot); HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor(); TableName tableName = TableName.valueOf(reqSnapshot.getTable()); // stop tracking "abandoned" handlers cleanupSentinels(); // Verify snapshot validity SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest); // Execute the restore/clone operation if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) { if (master.getAssignmentManager().getTableStateManager().isTableState( TableName.valueOf(fsSnapshot.getTable()), ZooKeeperProtos.Table.State.ENABLED)) { throw new UnsupportedOperationException("Table '" + TableName.valueOf(fsSnapshot.getTable()) + "' must be disabled in order to " + "perform a restore operation" + "."); } // call coproc pre hook if (cpHost != null) { cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc); } restoreSnapshot(fsSnapshot, snapshotTableDesc); LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName); if (cpHost != null) { cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc); } } else { HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName); if (cpHost != null) { cpHost.preCloneSnapshot(reqSnapshot, htd); } cloneSnapshot(fsSnapshot, htd); LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName); if (cpHost != null) { cpHost.postCloneSnapshot(reqSnapshot, htd); } } }
/** * The restore table is executed in place. * - The on-disk data will be restored - reference files are put in place without moving data * - [if something fail here: you need to delete the table and re-run the restore] * - hbase:meta will be updated * - [if something fail here: you need to run hbck to fix hbase:meta entries] * The passed in list gets changed in this method */ @Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem(); Connection conn = masterServices.getConnection(); FileSystem fs = fileSystemManager.getFileSystem(); Path rootDir = fileSystemManager.getRootDir(); TableName tableName = hTableDescriptor.getTableName(); try { // 1. Update descriptor this.masterServices.getTableDescriptors().add(hTableDescriptor); // 2. Execute the on-disk Restore LOG.debug("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open(masterServices.getConfiguration(), fs, snapshotDir, snapshot); RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( masterServices.getConfiguration(), fs, manifest, this.hTableDescriptor, rootDir, monitor, status); RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); // 3. Forces all the RegionStates to be offline // // The AssignmentManager keeps all the region states around // with no possibility to remove them, until the master is restarted. // This means that a region marked as SPLIT before the restore will never be assigned again. // To avoid having all states around all the regions are switched to the OFFLINE state, // which is the same state that the regions will be after a delete table. forceRegionsOffline(metaChanges); // 4. Applies changes to hbase:meta status.setStatus("Preparing to restore each region"); // 4.1 Removes the current set of regions from META // // By removing also the regions to restore (the ones present both in the snapshot // and in the current state) we ensure that no extra fields are present in META // e.g. with a simple add addRegionToMeta() the splitA and splitB attributes // not overwritten/removed, so you end up with old informations // that are not correct after the restore. List<HRegionInfo> hrisToRemove = new LinkedList<HRegionInfo>(); if (metaChanges.hasRegionsToRemove()) hrisToRemove.addAll(metaChanges.getRegionsToRemove()); MetaTableAccessor.deleteRegions(conn, hrisToRemove); // 4.2 Add the new set of regions to META // // At this point the old regions are no longer present in META. // and the set of regions present in the snapshot will be written to META. // All the information in hbase:meta are coming from the .regioninfo of each region present // in the snapshot folder. hris.clear(); if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd()); MetaTableAccessor.addRegionsToMeta(conn, hris); if (metaChanges.hasRegionsToRestore()) { MetaTableAccessor.overwriteRegions(conn, metaChanges.getRegionsToRestore()); } metaChanges.updateMetaParentRegions(this.server.getConnection(), hris); // At this point the restore is complete. Next step is enabling the table. LOG.info("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + tableName + " completed!"); } catch (IOException e) { String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " failed. Try re-running the restore command."; LOG.error(msg, e); monitor.receive(new ForeignException(masterServices.getServerName().toString(), e)); throw new RestoreSnapshotException(msg, e); } }
@Override public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException, KeeperException { try { timeoutInjector.start(); // 1. get all the regions hosting this table. // extract each pair to separate lists Set<HRegionInfo> regions = new HashSet<HRegionInfo>(); for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) { // Don't include non-default regions HRegionInfo hri = p.getFirst(); if (RegionReplicaUtil.isDefaultReplica(hri)) { regions.add(hri); } } // 2. for each region, write all the info to disk String msg = "Starting to write region info and WALs for regions for offline snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot); LOG.info(msg); status.setStatus(msg); ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot"); try { ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { @Override public void editRegion(final HRegionInfo regionInfo) throws IOException { snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo); } }); } finally { exec.shutdown(); } } catch (Exception e) { // make sure we capture the exception to propagate back to the client later String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); ForeignException ee = new ForeignException(reason, e); monitor.receive(ee); status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage()); } finally { LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot) + " as finished."); // 3. mark the timer as finished - even if we got an exception, we don't need to time the // operation any further timeoutInjector.complete(); } }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } // load table descriptor HTableDescriptor htd = manifest.getTableDescriptor(); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = null; if (conf.get(TableInputFormat.SCAN) != null) { scan = TableMapReduceUtil.convertStringToScan(conf.get(TableInputFormat.SCAN)); } else if (conf.get(org.apache.hadoop.hbase.mapred.TableInputFormat.COLUMN_LIST) != null) { String[] columns = conf.get(org.apache.hadoop.hbase.mapred.TableInputFormat.COLUMN_LIST).split(" "); scan = new Scan(); for (String col : columns) { scan.addFamily(Bytes.toBytes(col)); } } else { throw new IllegalArgumentException("Unable to create scan"); } // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); List<InputSplit> splits = new ArrayList<InputSplit>(); for (SnapshotRegionManifest regionManifest : regionManifests) { // load region descriptor HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { // compute HDFS locations from snapshot files (which will get the locations for // referred hfiles) List<String> hosts = getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); int len = Math.min(3, hosts.size()); hosts = hosts.subList(0, len); splits.add(new InputSplit(htd, hri, hosts)); } } return splits; }
private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName, Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException { if (newTableName == null) { newTableName = tableName; } FileSystem fileSys = tableBackupPath.getFileSystem(this.conf); // get table descriptor first TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId); if (tableDescriptor != null) { LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId); } if (tableDescriptor == null) { Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); if (fileSys.exists(tableSnapshotPath)) { // snapshot path exist means the backup path is in HDFS // check whether snapshot dir already recorded for target table if (snapshotMap.get(tableName) != null) { SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); tableDescriptor = manifest.getTableDescriptor(); } else { tableDescriptor = getTableDesc(tableName); snapshotMap.put(tableName, getTableInfoPath(tableName)); } if (tableDescriptor == null) { LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost"); } } else { throw new IOException("Table snapshot directory: " + tableSnapshotPath + " does not exist."); } } Path tableArchivePath = getTableArchivePath(tableName); if (tableArchivePath == null) { if (tableDescriptor != null) { // find table descriptor but no archive dir means the table is empty, create table and exit if (LOG.isDebugEnabled()) { LOG.debug("find table descriptor but no archive dir for table " + tableName + ", will only create table"); } tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor); checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor, truncateIfExists); return; } else { throw new IllegalStateException("Cannot restore hbase table because directory '" + " tableArchivePath is null."); } } if (tableDescriptor == null) { tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build(); } else { tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor); } // record all region dirs: // load all files in dir try { ArrayList<Path> regionPathList = getRegionList(tableName); // should only try to create the table with all region informations, so we could pre-split // the regions in fine grain checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList, tableDescriptor, truncateIfExists); RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf); Path[] paths = new Path[regionPathList.size()]; regionPathList.toArray(paths); restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true); } catch (Exception e) { LOG.error(e.toString(), e); throw new IllegalStateException("Cannot restore hbase table", e); } }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param manifest snapshot manifest to inspect * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(final SnapshotManifest manifest) throws IOException { List<RegionInfo> regions; if (TableName.META_TABLE_NAME.equals(tableName)) { regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper()); } else { regions = MetaTableAccessor.getTableRegions(services.getConnection(), tableName); } // Remove the non-default regions RegionReplicaUtil.removeNonDefaultRegions(regions); Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap(); if (regionManifests == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; boolean hasMobStore = false; // the mob region is a dummy region, it's not a real region in HBase. // the mob region has a special name, it could be found by the region name. if (regionManifests.get(MobUtils.getMobRegionInfo(tableName).getEncodedName()) != null) { hasMobStore = true; } int realRegionCount = hasMobStore ? regionManifests.size() - 1 : regionManifests.size(); if (realRegionCount != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + realRegionCount + "."; LOG.error(errorMsg); } // Verify RegionInfo for (RegionInfo region : regions) { SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName()); if (regionManifest == null) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); continue; } verifyRegionInfo(region, regionManifest); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } // Verify Snapshot HFiles SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(), fs, manifest); }
@Override public void snapshotRegions(List<Pair<RegionInfo, ServerName>> regionsAndLocations) throws IOException, KeeperException { try { // 1. get all the regions hosting this table. // extract each pair to separate lists Set<RegionInfo> regions = new HashSet<>(); for (Pair<RegionInfo, ServerName> p : regionsAndLocations) { // Don't include non-default regions RegionInfo hri = p.getFirst(); if (RegionReplicaUtil.isDefaultReplica(hri)) { regions.add(hri); } } // handle the mob files if any. boolean mobEnabled = MobUtils.hasMobColumns(htd); if (mobEnabled) { // snapshot the mob files as a offline region. RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(htd.getTableName()); regions.add(mobRegionInfo); } // 2. for each region, write all the info to disk String msg = "Starting to write region info and WALs for regions for offline snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot); LOG.info(msg); status.setStatus(msg); ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot"); try { ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() { @Override public void editRegion(final RegionInfo regionInfo) throws IOException { snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo); } }); } finally { exec.shutdown(); } } catch (Exception e) { // make sure we capture the exception to propagate back to the client later String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); ForeignException ee = new ForeignException(reason, e); monitor.receive(ee); status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage()); } finally { LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot) + " as finished."); } }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); RegionSplitter.SplitAlgorithm splitAlgo = getSplitAlgo(conf); int numSplits = conf.getInt(NUM_SPLITS_PER_REGION, 1); return getSplits(scan, manifest, regionInfos, restoreDir, conf, splitAlgo, numSplits); }
