/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot); } }
/** * Take a snapshot without waiting for the server to complete that snapshot (asynchronous) * <p> * Only a single snapshot should be taken at a time, or results may be undefined. * @param snapshot snapshot to take * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if the snapshot did not succeed or we lose contact with the master. * @throws SnapshotCreationException if snapshot creation failed * @throws IllegalArgumentException if the snapshot request is formatted incorrectly */ @Override public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, SnapshotCreationException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot on the master return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) { @Override public SnapshotResponse call(int callTimeout) throws ServiceException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); return master.snapshot(controller, request); } }); }
/** * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous) * <p> * Only a single snapshot should be restored at a time, or results may be undefined. * @param snapshot snapshot to restore * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if a remote or network exception occurs * @throws RestoreSnapshotException if snapshot failed to be restored * @throws IllegalArgumentException if the restore request is formatted incorrectly */ private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot) throws IOException, RestoreSnapshotException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot restore on the master return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) { @Override public RestoreSnapshotResponse call(int callTimeout) throws ServiceException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); return master.restoreSnapshot(controller, request); } }); }
/** * Checks if the specified snapshot is done. * @return true if the snapshot is in file system ready to use, * false if the snapshot is in the process of completing * @throws ServiceException wrapping UnknownSnapshotException if invalid snapshot, or * a wrapped HBaseSnapshotException with progress failure reason. */ @Override public IsSnapshotDoneResponse isSnapshotDone(RpcController controller, IsSnapshotDoneRequest request) throws ServiceException { LOG.debug("Checking to see if snapshot from request:" + ClientSnapshotDescriptionUtils.toString(request.getSnapshot()) + " is done"); try { master.checkInitialized(); IsSnapshotDoneResponse.Builder builder = IsSnapshotDoneResponse.newBuilder(); boolean done = master.snapshotManager.isSnapshotDone(request.getSnapshot()); builder.setDone(done); return builder.build(); } catch (IOException e) { throw new ServiceException(e); } }
/** * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous) * <p> * Only a single snapshot should be restored at a time, or results may be undefined. * @param snapshot snapshot to restore * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if a remote or network exception occurs * @throws RestoreSnapshotException if snapshot failed to be restored * @throws IllegalArgumentException if the restore request is formatted incorrectly */ private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot) throws IOException, RestoreSnapshotException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot restore on the master return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) { @Override public RestoreSnapshotResponse call(int callTimeout) throws ServiceException { return master.restoreSnapshot(null, request); } }); }
/** * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous) * <p> * Only a single snapshot should be restored at a time, or results may be undefined. * @param snapshot snapshot to restore * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if a remote or network exception occurs * @throws RestoreSnapshotException if snapshot failed to be restored * @throws IllegalArgumentException if the restore request is formatted incorrectly */ private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot) throws IOException, RestoreSnapshotException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot restore on the master return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) { @Override public RestoreSnapshotResponse call() throws ServiceException { return master.restoreSnapshot(null, request); } }); }
/** * Take a snapshot using the specified handler. * On failure the snapshot temporary working directory is removed. * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the * snapshot request if the table is busy with another snapshot/restore operation. * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable(SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException("Could not build snapshot handler", e, ProtobufUtil.createSnapshotDesc(snapshot)); } }
private CompletableFuture<Void> internalRestoreSnapshot(String snapshotName, TableName tableName) { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build(); try { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); } catch (IllegalArgumentException e) { return failedFuture(e); } return waitProcedureResult(this .<Long> newMasterCaller() .action( (controller, stub) -> this.<RestoreSnapshotRequest, RestoreSnapshotResponse, Long> call( controller, stub, RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce()).build(), (s, c, req, done) -> s.restoreSnapshot(c, req, done), (resp) -> resp.getProcId())).call()); }
/** * Clone the specified snapshot into a new table. * The operation will fail if the destination table has a snapshot or restore in progress. * * @param snapshot Snapshot Descriptor * @param hTableDescriptor Table Descriptor of the table to create */ synchronized void cloneSnapshot(final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException { TableName tableName = hTableDescriptor.getTableName(); // make sure we aren't running a snapshot on the same table if (isTakingSnapshot(tableName)) { throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName); } // make sure we aren't running a restore on the same table if (isRestoringTable(tableName)) { throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName); } try { CloneSnapshotHandler handler = new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare(); this.executorService.submit(handler); this.restoreHandlers.put(tableName, handler); } catch (Exception e) { String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + tableName; LOG.error(msg, e); throw new RestoreSnapshotException(msg, e); } }
/** * Restore the specified snapshot. * The restore will fail if the destination table has a snapshot or restore in progress. * * @param snapshot Snapshot Descriptor * @param hTableDescriptor Table Descriptor */ private synchronized void restoreSnapshot(final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException { TableName tableName = hTableDescriptor.getTableName(); // make sure we aren't running a snapshot on the same table if (isTakingSnapshot(tableName)) { throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName); } // make sure we aren't running a restore on the same table if (isRestoringTable(tableName)) { throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName); } try { RestoreSnapshotHandler handler = new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare(); this.executorService.submit(handler); restoreHandlers.put(tableName, handler); } catch (Exception e) { String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString( snapshot) + " on table=" + tableName; LOG.error(msg, e); throw new RestoreSnapshotException(msg, e); } }
/** * Returns the status of a restore operation. * If the in-progress restore is failed throws the exception that caused the failure. * * @param snapshot * @return false if in progress, true if restore is completed or not requested. * @throws IOException if there was a failure during the restore */ public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException { // check to see if the sentinel exists, // and if the task is complete removes it from the in-progress restore map. SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot); // stop tracking "abandoned" handlers cleanupSentinels(); if (sentinel == null) { // there is no sentinel so restore is not in progress. return true; } LOG.debug("Verify snapshot=" + snapshot.getName() + " against=" + sentinel.getSnapshot().getName() + " table=" + TableName.valueOf(snapshot.getTable())); // If the restore is failed, rethrow the exception sentinel.rethrowExceptionIfFailed(); // check to see if we are done if (sentinel.isFinished()) { LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " has completed. Notifying the client."); return true; } if (LOG.isDebugEnabled()) { LOG.debug("Sentinel is not yet finished with restoring snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)); } return false; }
@Override public void cancel(String why) { if (this.stopped) return; this.stopped = true; String msg = "Stopping restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " because: " + why; LOG.info(msg); CancellationException ce = new CancellationException(why); this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce)); }
@Override public void cancel(String why) { if (finished) return; this.finished = true; LOG.info("Stop taking snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " because: " + why); CancellationException ce = new CancellationException(why); monitor.receive(new ForeignException(master.getServerName().toString(), ce)); }
private void flushSnapshot() throws ForeignException { if (regions.isEmpty()) { // No regions on this RS, we are basically done. return; } monitor.rethrowException(); // assert that the taskManager is empty. if (taskManager.hasTasks()) { throw new IllegalStateException("Attempting to take snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " but we currently have outstanding tasks"); } // Add all hfiles already existing in region. for (Region region : regions) { // submit one task per region for parallelize by region. taskManager.submitTask(new RegionSnapshotTask(region)); monitor.rethrowException(); } // wait for everything to complete. LOG.debug("Flush Snapshot Tasks submitted for " + regions.size() + " regions"); try { taskManager.waitForOutstandingTasks(); } catch (InterruptedException e) { LOG.error("got interrupted exception for " + getMemberName()); throw new ForeignException(getMemberName(), e); } }
private void flushSnapshot() throws ForeignException { if (regions.isEmpty()) { // No regions on this RS, we are basically done. return; } monitor.rethrowException(); // assert that the taskManager is empty. if (taskManager.hasTasks()) { throw new IllegalStateException("Attempting to take snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " but we currently have outstanding tasks"); } // Add all hfiles already existing in region. for (HRegion region : regions) { // submit one task per region for parallelize by region. taskManager.submitTask(new RegionSnapshotTask(region)); monitor.rethrowException(); } // wait for everything to complete. LOG.debug("Flush Snapshot Tasks submitted for " + regions.size() + " regions"); try { taskManager.waitForOutstandingTasks(); } catch (InterruptedException e) { throw new ForeignException(getMemberName(), e); } }
/** * Take a snapshot without waiting for the server to complete that snapshot (asynchronous) * <p> * Only a single snapshot should be taken at a time, or results may be undefined. * @param snapshot snapshot to take * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if the snapshot did not succeed or we lose contact with the master. * @throws SnapshotCreationException if snapshot creation failed * @throws IllegalArgumentException if the snapshot request is formatted incorrectly */ @Override public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, SnapshotCreationException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot on the master return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) { @Override public SnapshotResponse call(int callTimeout) throws ServiceException { return master.snapshot(null, request); } }); }
/** * Checks if the specified snapshot is done. * @return true if the snapshot is in file system ready to use, * false if the snapshot is in the process of completing * @throws ServiceException wrapping UnknownSnapshotException if invalid snapshot, or * a wrapped HBaseSnapshotException with progress failure reason. */ @Override public IsSnapshotDoneResponse isSnapshotDone(RpcController controller, IsSnapshotDoneRequest request) throws ServiceException { LOG.debug("Checking to see if snapshot from request:" + ClientSnapshotDescriptionUtils.toString(request.getSnapshot()) + " is done"); try { IsSnapshotDoneResponse.Builder builder = IsSnapshotDoneResponse.newBuilder(); boolean done = snapshotManager.isSnapshotDone(request.getSnapshot()); builder.setDone(done); return builder.build(); } catch (IOException e) { throw new ServiceException(e); } }
/** * Check that all the regions in the snapshot are valid, and accounted for. * @param snapshotDir snapshot directory to check * @throws IOException if we can't reach hbase:meta or read the files from the FS */ private void verifyRegions(Path snapshotDir) throws IOException { List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(), tableName); Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir); if (snapshotRegions == null) { String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty"; LOG.error(msg); throw new CorruptedSnapshotException(msg); } String errorMsg = ""; if (snapshotRegions.size() != regions.size()) { errorMsg = "Regions moved during the snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" + regions.size() + " snapshotted=" + snapshotRegions.size() + "."; LOG.error(errorMsg); } for (HRegionInfo region : regions) { if (!snapshotRegions.contains(region.getEncodedName())) { // could happen due to a move or split race. String mesg = " No snapshot region directory found for region:" + region; if (errorMsg.isEmpty()) errorMsg = mesg; LOG.error(mesg); } verifyRegion(fs, snapshotDir, region); } if (!errorMsg.isEmpty()) { throw new CorruptedSnapshotException(errorMsg); } }
/** * Take a snapshot without waiting for the server to complete that snapshot (asynchronous) * <p> * Only a single snapshot should be taken at a time, or results may be undefined. * @param snapshot snapshot to take * @return response from the server indicating the max time to wait for the snapshot * @throws IOException if the snapshot did not succeed or we lose contact with the master. * @throws SnapshotCreationException if snapshot creation failed * @throws IllegalArgumentException if the snapshot request is formatted incorrectly */ public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, SnapshotCreationException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); // run the snapshot on the master return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) { @Override public SnapshotResponse call() throws ServiceException { return master.snapshot(null, request); } }); }
/** * Execute the on-disk Restore * @param env MasterProcedureEnv * @throws IOException **/ private void restoreSnapshot(final MasterProcedureEnv env) throws IOException { MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem(); FileSystem fs = fileSystemManager.getFileSystem(); Path rootDir = fileSystemManager.getRootDir(); final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher(); LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)); try { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot); RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( env.getMasterServices().getConfiguration(), fs, manifest, modifiedTableDescriptor, rootDir, monitorException, getMonitorStatus()); RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); regionsToRestore = metaChanges.getRegionsToRestore(); regionsToRemove = metaChanges.getRegionsToRemove(); regionsToAdd = metaChanges.getRegionsToAdd(); parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap(); } catch (IOException e) { String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " failed in on-disk restore. Try re-running the restore command."; LOG.error(msg, e); monitorException.receive( new ForeignException(env.getMasterServices().getServerName().toString(), e)); throw new IOException(msg, e); } }
/** * Clone the specified snapshot into a new table. * The operation will fail if the destination table has a snapshot or restore in progress. * * @param snapshot Snapshot Descriptor * @param tableDescriptor Table Descriptor of the table to create * @param nonceKey unique identifier to prevent duplicated RPC * @return procId the ID of the clone snapshot procedure */ synchronized long cloneSnapshot(final SnapshotDescription snapshot, final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl) throws HBaseSnapshotException { TableName tableName = tableDescriptor.getTableName(); // make sure we aren't running a snapshot on the same table if (isTakingSnapshot(tableName)) { throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName); } // make sure we aren't running a restore on the same table if (isRestoringTable(tableName)) { throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName); } try { long procId = master.getMasterProcedureExecutor().submitProcedure( new CloneSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(), tableDescriptor, snapshot, restoreAcl), nonceKey); this.restoreTableToProcIdMap.put(tableName, procId); return procId; } catch (Exception e) { String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + tableName; LOG.error(msg, e); throw new RestoreSnapshotException(msg, e); } }
/** * Restore the specified snapshot. The restore will fail if the destination table has a snapshot * or restore in progress. * @param snapshot Snapshot Descriptor * @param tableDescriptor Table Descriptor * @param nonceKey unique identifier to prevent duplicated RPC * @param restoreAcl true to restore acl of snapshot * @return procId the ID of the restore snapshot procedure */ private synchronized long restoreSnapshot(final SnapshotDescription snapshot, final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl) throws HBaseSnapshotException { final TableName tableName = tableDescriptor.getTableName(); // make sure we aren't running a snapshot on the same table if (isTakingSnapshot(tableName)) { throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName); } // make sure we aren't running a restore on the same table if (isRestoringTable(tableName)) { throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName); } try { long procId = master.getMasterProcedureExecutor().submitProcedure( new RestoreSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(), tableDescriptor, snapshot, restoreAcl), nonceKey); this.restoreTableToProcIdMap.put(tableName, procId); return procId; } catch (Exception e) { String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString( snapshot) + " on table=" + tableName; LOG.error(msg, e); throw new RestoreSnapshotException(msg, e); } }