public RestoreSnapshotHandler(final MasterServices masterServices, final SnapshotDescription snapshot, final HTableDescriptor htd) throws IOException { super(EventType.C_M_RESTORE_SNAPSHOT, htd.getTableName(), masterServices, masterServices); // Snapshot information this.snapshot = snapshot; // Monitor this.monitor = new ForeignExceptionDispatcher(); // Check table exists. getTableDescriptor(); // This is the new schema we are going to write out as this modification. this.hTableDescriptor = htd; this.status = TaskMonitor.get().createStatus( "Restoring snapshot '" + snapshot.getName() + "' to table " + hTableDescriptor.getTableName()); }
/** * Creates a procedure. (FOR TESTING) * * {@link Procedure} state to be run by a {@link ProcedureCoordinator}. * @param coord coordinator to call back to for general errors (e.g. * {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}). * @param monitor error monitor to check for external errors * @param wakeFreq frequency to check for errors while waiting * @param timeout amount of time to allow the procedure to run before cancelling * @param procName name of the procedure instance * @param args argument data associated with the procedure instance * @param expectedMembers names of the expected members */ public Procedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, long wakeFreq, long timeout, String procName, byte[] args, List<String> expectedMembers) { this.coord = coord; this.acquiringMembers = new ArrayList<String>(expectedMembers); this.inBarrierMembers = new ArrayList<String>(acquiringMembers.size()); this.dataFromFinishedMembers = new HashMap<String, byte[]>(); this.procName = procName; this.args = args; this.monitor = monitor; this.wakeFrequency = wakeFreq; int count = expectedMembers.size(); this.acquiredBarrierLatch = new CountDownLatch(count); this.releasedBarrierLatch = new CountDownLatch(count); this.completedLatch = new CountDownLatch(1); this.timeoutInjector = new TimeoutExceptionInjector(monitor, timeout); }
/** * If in a running state, creates the specified subprocedure for handling a procedure. * @return Subprocedure to submit to the ProcedureMemeber. */ public Subprocedure buildSubprocedure(String name) { // don't run a procedure if the parent is stop(ping) if (rss.isStopping() || rss.isStopped()) { throw new IllegalStateException("Can't start procedure on RS: " + rss.getServerName() + ", because stopping/stopped!"); } LOG.info("Attempting to run a procedure."); ForeignExceptionDispatcher errorDispatcher = new ForeignExceptionDispatcher(); Configuration conf = rss.getConfiguration(); SimpleSubprocedurePool taskManager = new SimpleSubprocedurePool(rss.getServerName().toString(), conf); return new SimpleSubprocedure(rss, member, errorDispatcher, taskManager, name); }
/** * Currently we can only handle one procedure at a time. This makes sure we handle that and * reject submitting more. */ @Test public void testThreadPoolSize() throws Exception { ProcedureCoordinator coordinator = buildNewCoordinator(); Procedure proc = new Procedure(coordinator, monitor, WAKE_FREQUENCY, TIMEOUT, procName, procData, expected); Procedure procSpy = spy(proc); Procedure proc2 = new Procedure(coordinator, monitor, WAKE_FREQUENCY, TIMEOUT, procName +"2", procData, expected); Procedure procSpy2 = spy(proc2); when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) .thenReturn(procSpy, procSpy2); coordinator.startProcedure(procSpy.getErrorMonitor(), procName, procData, expected); // null here means second procedure failed to start. assertNull("Coordinator successfully ran two tasks at once with a single thread pool.", coordinator.startProcedure(proc2.getErrorMonitor(), "another op", procData, expected)); }
public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepareOperation, BarrierAnswer commitOperation, String... cohort) throws Exception { List<String> expected = Arrays.asList(cohort); when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class))) .thenReturn(spy); // use the passed controller responses doAnswer(prepareOperation).when(controller).sendGlobalBarrierAcquire(spy, procData, expected); doAnswer(commitOperation).when(controller) .sendGlobalBarrierReached(eq(spy), anyListOf(String.class)); // run the operation Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected); // and wait for it to finish task.waitForCompleted(); // make sure we mocked correctly prepareOperation.ensureRan(); // we never got an exception InOrder inorder = inOrder(spy, controller); inorder.verify(spy).sendGlobalBarrierStart(); inorder.verify(controller).sendGlobalBarrierAcquire(task, procData, expected); inorder.verify(spy).sendGlobalBarrierReached(); inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class)); }
@Test(timeout = 60000) public void testErrorPropagation() throws Exception { List<String> members = new ArrayList<String>(); members.add("member"); Procedure proc = new Procedure(coord, new ForeignExceptionDispatcher(), 100, Integer.MAX_VALUE, "op", null, members); final Procedure procspy = spy(proc); ForeignException cause = new ForeignException("SRC", "External Exception"); proc.receive(cause); // start the barrier procedure Thread t = new Thread() { public void run() { procspy.call(); } }; t.start(); t.join(); verify(procspy, never()).sendGlobalBarrierStart(); verify(procspy, never()).sendGlobalBarrierReached(); verify(procspy).sendGlobalBarrierComplete(); }
public RestoreSnapshotHandler(final MasterServices masterServices, final SnapshotDescription snapshot, final HTableDescriptor htd, final MasterMetrics metricsMaster) throws IOException { super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices); this.metricsMaster = metricsMaster; // Snapshot information this.snapshot = snapshot; // Monitor this.monitor = new ForeignExceptionDispatcher(); // Check table exists. getTableDescriptor(); // This is the new schema we are going to write out as this modification. this.hTableDescriptor = htd; this.status = TaskMonitor.get().createStatus( "Restoring snapshot '" + snapshot.getName() + "' to table " + hTableDescriptor.getNameAsString()); }
public CloneSnapshotHandler(final MasterServices masterServices, final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor, final MasterMetrics metricsMaster) throws NotAllMetaRegionsOnlineException, TableExistsException, IOException { super(masterServices, masterServices.getMasterFileSystem(), masterServices.getServerManager(), hTableDescriptor, masterServices.getConfiguration(), null, masterServices.getCatalogTracker(), masterServices.getAssignmentManager()); this.metricsMaster = metricsMaster; // Snapshot information this.snapshot = snapshot; // Monitor this.monitor = new ForeignExceptionDispatcher(); this.status = TaskMonitor.get().createStatus("Cloning snapshot '" + snapshot.getName() + "' to table " + hTableDescriptor.getNameAsString()); }
/** * Creates a procedure. (FOR TESTING) * * {@link Procedure} state to be run by a {@link ProcedureCoordinator}. * @param coord coordinator to call back to for general errors (e.g. * {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}). * @param monitor error monitor to check for external errors * @param wakeFreq frequency to check for errors while waiting * @param timeout amount of time to allow the procedure to run before cancelling * @param procName name of the procedure instance * @param args argument data associated with the procedure instance * @param expectedMembers names of the expected members */ public Procedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, long wakeFreq, long timeout, String procName, byte[] args, List<String> expectedMembers) { this.coord = coord; this.acquiringMembers = new ArrayList<String>(expectedMembers); this.inBarrierMembers = new ArrayList<String>(acquiringMembers.size()); this.procName = procName; this.args = args; this.monitor = monitor; this.wakeFrequency = wakeFreq; int count = expectedMembers.size(); this.acquiredBarrierLatch = new CountDownLatch(count); this.releasedBarrierLatch = new CountDownLatch(count); this.completedLatch = new CountDownLatch(1); this.timeoutInjector = new TimeoutExceptionInjector(monitor, timeout); }
/** * Reference all the files in the given region directory * @param snapshot snapshot for which to add references * @param monitor to check/send error * @param regionDir region directory to look for errors * @param fs {@link FileSystem} where the snapshot/region live * @param regionSnapshotDir directory in the snapshot to store region files */ public ReferenceRegionHFilesTask(final SnapshotDescription snapshot, ForeignExceptionDispatcher monitor, Path regionDir, final FileSystem fs, Path regionSnapshotDir) { super(snapshot, monitor); this.regiondir = regionDir; this.fs = fs; this.fileFilter = new PathFilter() { @Override public boolean accept(Path path) { try { return fs.isFile(path); } catch (IOException e) { LOG.error("Failed to reach fs to check file:" + path + ", marking as not file"); ReferenceRegionHFilesTask.this.snapshotFailure("Failed to reach fs to check file status", e); return false; } } }; this.snapshotDir = regionSnapshotDir; }
/** * Check that we don't get an exception if there is no recovered edits directory to copy * @throws Exception on failure */ @Test public void testNoEditsDir() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionDir); assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); task.call(); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
/** * Check that errors from running the task get propagated back to the error listener. */ @Test public void testErrorPropagation() throws Exception { ForeignExceptionDispatcher error = mock(ForeignExceptionDispatcher.class); SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot") .setTable("table").build(); final Exception thrown = new Exception("Failed!"); SnapshotTask fail = new SnapshotTask(snapshot, error) { @Override public Void call() { snapshotFailure("Injected failure", thrown); return null; } }; fail.call(); verify(error, Mockito.times(1)).receive(any(ForeignException.class)); }
public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final HTableDescriptor tableDescriptor, final Path rootDir, final ForeignExceptionDispatcher monitor, final MonitoredTask status) { this.fs = fs; this.conf = conf; this.snapshotManifest = manifest; this.snapshotDesc = manifest.getSnapshotDescription(); this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); this.tableDesc = tableDescriptor; this.rootDir = rootDir; this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName()); this.monitor = monitor; this.status = status; }
public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshotDescription, final Path snapshotDir, final HTableDescriptor tableDescriptor, final Path rootDir, final ForeignExceptionDispatcher monitor, final MonitoredTask status) { this.fs = fs; this.conf = conf; this.snapshotDesc = snapshotDescription; this.snapshotTable = TableName.valueOf(snapshotDescription.getTable()); this.snapshotDir = snapshotDir; this.tableDesc = tableDescriptor; this.rootDir = rootDir; this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName()); this.monitor = monitor; this.status = status; }
/** * Initialize the restore helper, based on the snapshot and table information provided. */ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, final String sourceTableName, final HTableDescriptor htdClone) throws IOException { CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class); HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); MonitoredTask status = Mockito.mock(MonitoredTask.class); SnapshotDescription sd = SnapshotDescription.newBuilder() .setName("snapshot") .setTable(sourceTableName) .build(); return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir, htdClone, rootDir, monitor, status); }
/** * Check that we don't get an exception if there is no recovered edits directory to copy * @throws Exception on failure */ @Test public void testNoEditsDir() throws Exception { SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("snapshot").build(); ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); FileSystem fs = UTIL.getTestFileSystem(); Path root = UTIL.getDataTestDir(); String regionName = "regionA"; Path regionDir = new Path(root, regionName); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, root); try { // doesn't really matter where the region's snapshot directory is, but this is pretty close Path snapshotRegionDir = new Path(workingDir, regionName); fs.mkdirs(snapshotRegionDir); Path regionEdits = HLogUtil.getRegionDirRecoveredEditsDir(regionDir); assertFalse("Edits dir exists already - it shouldn't", fs.exists(regionEdits)); CopyRecoveredEditsTask task = new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir); task.call(); } finally { // cleanup the working directory FSUtils.delete(fs, regionDir, true); FSUtils.delete(fs, workingDir, true); } }
public CloneSnapshotHandler(final MasterServices masterServices, final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) { super(masterServices, masterServices.getMasterFileSystem(), hTableDescriptor, masterServices.getConfiguration(), null, masterServices); // Snapshot information this.snapshot = snapshot; // Monitor this.monitor = new ForeignExceptionDispatcher(); this.status = TaskMonitor.get().createStatus("Cloning snapshot '" + snapshot.getName() + "' to table " + hTableDescriptor.getTableName()); }
public FlushTableSubprocedure(ProcedureMember member, ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, List<Region> regions, String table, FlushTableSubprocedurePool taskManager) { super(member, table, errorListener, wakeFrequency, timeout); this.table = table; this.regions = regions; this.taskManager = taskManager; }
/** * If in a running state, creates the specified subprocedure to flush table regions. * * Because this gets the local list of regions to flush and not the set the master had, * there is a possibility of a race where regions may be missed. * * @param table * @return Subprocedure to submit to the ProcedureMemeber. */ public Subprocedure buildSubprocedure(String table) { // don't run the subprocedure if the parent is stop(ping) if (rss.isStopping() || rss.isStopped()) { throw new IllegalStateException("Can't start flush region subprocedure on RS: " + rss.getServerName() + ", because stopping/stopped!"); } // check to see if this server is hosting any regions for the table List<Region> involvedRegions; try { involvedRegions = getRegionsToFlush(table); } catch (IOException e1) { throw new IllegalStateException("Failed to figure out if there is region to flush.", e1); } // We need to run the subprocedure even if we have no relevant regions. The coordinator // expects participation in the procedure and without sending message the master procedure // will hang and fail. LOG.debug("Launching subprocedure to flush regions for " + table); ForeignExceptionDispatcher exnDispatcher = new ForeignExceptionDispatcher(table); Configuration conf = rss.getConfiguration(); long timeoutMillis = conf.getLong(FLUSH_TIMEOUT_MILLIS_KEY, FLUSH_TIMEOUT_MILLIS_DEFAULT); long wakeMillis = conf.getLong(FLUSH_REQUEST_WAKE_MILLIS_KEY, FLUSH_REQUEST_WAKE_MILLIS_DEFAULT); FlushTableSubprocedurePool taskManager = new FlushTableSubprocedurePool(rss.getServerName().toString(), conf, rss); return new FlushTableSubprocedure(member, exnDispatcher, wakeMillis, timeoutMillis, involvedRegions, table, taskManager); }
public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final HTableDescriptor tableDescriptor, final Path rootDir, final ForeignExceptionDispatcher monitor, final MonitoredTask status) { this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true); }
public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final HTableDescriptor tableDescriptor, final Path rootDir, final ForeignExceptionDispatcher monitor, final MonitoredTask status, final boolean createBackRefs) { this.fs = fs; this.conf = conf; this.snapshotManifest = manifest; this.snapshotDesc = manifest.getSnapshotDescription(); this.snapshotTable = TableName.valueOf(snapshotDesc.getTable()); this.tableDesc = tableDescriptor; this.rootDir = rootDir; this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName()); this.monitor = monitor; this.status = status; this.createBackRefs = createBackRefs; }
/** * Copy the snapshot files for a snapshot scanner, discards meta changes. * @param conf * @param fs * @param rootDir * @param restoreDir * @param snapshotName * @throws IOException */ public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir, Path restoreDir, String snapshotName) throws IOException { // ensure that restore dir is not under root dir if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) { throw new IllegalArgumentException("Filesystems for restore directory and HBase root directory " + "should be the same"); } if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) { throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir); } Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); MonitoredTask status = TaskMonitor.get().createStatus( "Restoring snapshot '" + snapshotName + "' to directory " + restoreDir); ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(); // we send createBackRefs=false so that restored hfiles do not create back reference links // in the base hbase root dir. RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false); RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize. if (LOG.isDebugEnabled()) { LOG.debug("Restored table dir:" + restoreDir); FSUtils.logFileSystemState(fs, restoreDir, LOG); } return metaChanges; }
public FlushSnapshotSubprocedure(ProcedureMember member, ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout, List<Region> regions, SnapshotDescription snapshot, SnapshotSubprocedurePool taskManager) { super(member, snapshot.getName(), errorListener, wakeFrequency, timeout); this.snapshot = snapshot; if (this.snapshot.getType() == SnapshotDescription.Type.SKIPFLUSH) { snapshotSkipFlush = true; } this.regions = regions; this.taskManager = taskManager; }
