void regionServerReport(ServerName sn, HServerLoad hsl) throws YouAreDeadException, PleaseHoldException { checkIsDead(sn, "REPORT"); if (!this.onlineServers.containsKey(sn)) { // Already have this host+port combo and its just different start code? checkAlreadySameHostPort(sn); // Just let the server in. Presume master joining a running cluster. // recordNewServer is what happens at the end of reportServerStartup. // The only thing we are skipping is passing back to the regionserver // the ServerName to use. Here we presume a master has already done // that so we'll press on with whatever it gave us for ServerName. recordNewServer(sn, hsl); } else { this.onlineServers.put(sn, hsl); } }
/** * Test to see if we have a server of same host and port already. * @param serverName * @throws PleaseHoldException */ void checkAlreadySameHostPort(final ServerName serverName) throws PleaseHoldException { ServerName existingServer = ServerName.findServerWithSameHostnamePort(getOnlineServersList(), serverName); if (existingServer != null) { String message = "Server serverName=" + serverName + " rejected; we already have " + existingServer.toString() + " registered with same hostname and port"; LOG.info(message); if (existingServer.getStartcode() < serverName.getStartcode()) { LOG.info("Triggering server recovery; existingServer " + existingServer + " looks stale, new server:" + serverName); expireServer(existingServer); } if (services.isServerShutdownHandlerEnabled()) { // master has completed the initialization throw new PleaseHoldException(message); } } }
private void updateRegionTransition(final ServerName serverName, final TransitionCode state, final RegionInfo regionInfo, final long seqId) throws PleaseHoldException, UnexpectedStateException { checkFailoverCleanupCompleted(regionInfo); final RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); if (regionNode == null) { // the table/region is gone. maybe a delete, split, merge throw new UnexpectedStateException(String.format( "Server %s was trying to transition region %s to %s. but the region was removed.", serverName, regionInfo, state)); } if (LOG.isTraceEnabled()) { LOG.trace(String.format("Update region transition serverName=%s region=%s regionState=%s", serverName, regionNode, state)); } final ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); if (!reportTransition(regionNode, serverNode, state, seqId)) { LOG.warn(String.format( "No procedure for %s. server=%s to transition to %s", regionNode, serverName, state)); } }
/** * Test to see if we have a server of same host and port already. * @param serverName * @throws PleaseHoldException */ void checkAlreadySameHostPort(final ServerName serverName) throws PleaseHoldException { ServerName existingServer = ServerName.findServerWithSameHostnamePort(getOnlineServersList(), serverName); if (existingServer != null) { String message = "Server serverName=" + serverName + " rejected; we already have " + existingServer.toString() + " registered with same hostname and port"; LOG.info(message); if (existingServer.getStartcode() < serverName.getStartcode()) { LOG.info("Triggering server recovery; existingServer " + existingServer + " looks stale, new server:" + serverName); expireServer(existingServer); } throw new PleaseHoldException(message); } }
void regionServerReport(ServerName sn, ServerLoad sl) throws YouAreDeadException, PleaseHoldException { checkIsDead(sn, "REPORT"); if (!this.onlineServers.containsKey(sn)) { // Already have this host+port combo and its just different start code? checkAlreadySameHostPort(sn); // Just let the server in. Presume master joining a running cluster. // recordNewServer is what happens at the end of reportServerStartup. // The only thing we are skipping is passing back to the regionserver // the ServerName to use. Here we presume a master has already done // that so we'll press on with whatever it gave us for ServerName. recordNewServer(sn, sl); } else { this.onlineServers.put(sn, sl); } updateLastFlushedSequenceIds(sn, sl); }
private boolean causeIsPleaseHold(Throwable e) { if (e instanceof PleaseHoldException) return true; if (e instanceof TableNotEnabledException) return true; if (e instanceof RegionOfflineException) return true; if (e instanceof RetriesExhaustedException || e instanceof SocketTimeoutException) { if (e.getCause() instanceof RemoteException) { RemoteException re = (RemoteException) e.getCause(); if (PleaseHoldException.class.getName().equals(re.getClassName())) return true; } } return false; }
@Override public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException { try { SpliceLogUtils.info(LOG, "preCreateTable %s", Bytes.toString(desc.getTableName().getName())); if (Bytes.equals(desc.getTableName().getName(), INIT_TABLE)) { switch(manager.getState()){ case NOT_STARTED: boot(ctx.getEnvironment().getMasterServices().getServerName()); case BOOTING_ENGINE: case BOOTING_GENERAL_SERVICES: case BOOTING_SERVER: throw new PleaseHoldException("Please Hold - Starting"); case RUNNING: throw new InitializationCompleted("Success"); case STARTUP_FAILED: case SHUTTING_DOWN: case SHUTDOWN: throw new IllegalStateException("Startup failed"); } } } catch (Throwable t) { throw CoprocessorUtils.getIOException(t); } }
@Override public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c, ReportRegionStateTransitionRequest req) throws ServiceException { try { master.checkServiceStarted(); RegionStateTransition rt = req.getTransition(0); TableName tableName = ProtobufUtil.toTableName( rt.getRegionInfo(0).getTableName()); RegionStates regionStates = master.assignmentManager.getRegionStates(); if (!(TableName.META_TABLE_NAME.equals(tableName) && regionStates.getRegionState(HRegionInfo.FIRST_META_REGIONINFO) != null) && !master.assignmentManager.isFailoverCleanupDone()) { // Meta region is assigned before master finishes the // failover cleanup. So no need this check for it throw new PleaseHoldException("Master is rebuilding user regions"); } ServerName sn = ProtobufUtil.toServerName(req.getServer()); String error = master.assignmentManager.onRegionTransition(sn, rt); ReportRegionStateTransitionResponse.Builder rrtr = ReportRegionStateTransitionResponse.newBuilder(); if (error != null) { rrtr.setErrorMessage(error); } return rrtr.build(); } catch (IOException ioe) { throw new ServiceException(ioe); } }
@Test public void testMoveRegionWhenNotInitialized() { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HMaster m = cluster.getMaster(); try { m.setInitialized(false); // fake it, set back later HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO; m.move(meta.getEncodedNameAsBytes(), null); fail("Region should not be moved since master is not initialized"); } catch (IOException ioe) { assertTrue(ioe instanceof PleaseHoldException); } finally { m.setInitialized(true); } }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ @Test public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { Log.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null) connection.close(); } }
@Test public void testMoveRegionWhenNotInitialized() { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HMaster m = cluster.getMaster(); try { m.initialized = false; // fake it, set back later HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO; m.move(meta.getEncodedNameAsBytes(), null); fail("Region should not be moved since master is not initialized"); } catch (IOException ioe) { assertTrue(ioe instanceof PleaseHoldException); } finally { m.initialized = true; } }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ @Test public void testMasterMonitorCollableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); // Mock up our admin Interfaces HBaseAdmin admin = new HBaseAdmin(configuration); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { Log.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null)HConnectionManager.deleteConnection(configuration); } }
/** * Used to check if the failover cleanup is done. * if not we throw PleaseHoldException since we are rebuilding the RegionStates * @param hri region to check if it is already rebuild * @throws PleaseHoldException if the failover cleanup is not completed */ private void checkFailoverCleanupCompleted(final RegionInfo hri) throws PleaseHoldException { if (!isRunning()) { throw new PleaseHoldException("AssignmentManager not running"); } // TODO: can we avoid throwing an exception if hri is already loaded? // at the moment we bypass only meta boolean meta = isMetaRegion(hri); boolean cleanup = isFailoverCleanupDone(); if (!isMetaRegion(hri) && !isFailoverCleanupDone()) { String msg = "Master not fully online; hbase:meta=" + meta + ", failoverCleanup=" + cleanup; throw new PleaseHoldException(msg); } }
/** * Reports the given map of Regions and their size on the filesystem to the active Master. * * @param onlineRegionSizes A map of region info to size in bytes * @return false if FileSystemUtilizationChore should pause reporting to master. true otherwise */ public boolean reportRegionSizesForQuotas(final Map<RegionInfo, Long> onlineRegionSizes) { RegionServerStatusService.BlockingInterface rss = rssStub; if (rss == null) { // the current server could be stopping. LOG.trace("Skipping Region size report to HMaster as stub is null"); return true; } try { RegionSpaceUseReportRequest request = buildRegionSpaceUseReportRequest( Objects.requireNonNull(onlineRegionSizes)); rss.reportRegionSpaceUse(null, request); } catch (ServiceException se) { IOException ioe = ProtobufUtil.getRemoteException(se); if (ioe instanceof PleaseHoldException) { LOG.trace("Failed to report region sizes to Master because it is initializing." + " This will be retried.", ioe); // The Master is coming up. Will retry the report later. Avoid re-creating the stub. return true; } if (rssStub == rss) { rssStub = null; } createRegionServerStatusStub(true); if (ioe instanceof DoNotRetryIOException) { DoNotRetryIOException doNotRetryEx = (DoNotRetryIOException) ioe; if (doNotRetryEx.getCause() != null) { Throwable t = doNotRetryEx.getCause(); if (t instanceof UnsupportedOperationException) { LOG.debug("master doesn't support ReportRegionSpaceUse, pause before retrying"); return false; } } } LOG.debug("Failed to report region sizes to Master. This will be retried.", ioe); } return true; }
@Test public void testMoveRegionWhenNotInitialized() { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); HMaster m = cluster.getMaster(); try { m.setInitialized(false); // fake it, set back later RegionInfo meta = RegionInfoBuilder.FIRST_META_REGIONINFO; m.move(meta.getEncodedNameAsBytes(), null); fail("Region should not be moved since master is not initialized"); } catch (IOException ioe) { assertTrue(ioe instanceof PleaseHoldException); } finally { m.setInitialized(true); } }
/** * Creates a new HBase table. * * @param name * the name of the table * @param columnFamilies * a list of column family names to create * @param maxVersions * the max number of versions to maintain for the column families * @throws HBaseException */ public void createTable(String name, List<String> columnFamilies, int maxVersions) throws HBaseException { if (name == null || "".equals(name)) { throw new HBaseException("Table name must not be empty."); } HBaseAdmin admin = null; try { admin = new HBaseAdmin(CONFIG); HTableDescriptor desc = new HTableDescriptor(name.getBytes()); // add all column families if (columnFamilies != null) { for (String cf : columnFamilies) { HColumnDescriptor colFamilyDesc = new HColumnDescriptor(cf.getBytes()); colFamilyDesc.setMaxVersions(maxVersions); desc.addFamily(colFamilyDesc); } } int retries = 0; do { try { admin.createTable(desc); return; } catch (PleaseHoldException e) { TimeUnit.SECONDS.sleep(1); retries++; } } while(retries < 10); admin.createTable(desc); } catch (Exception ex) { throw new HBaseException("Exception occurred when creating HBase table.", ex); } finally { close(admin); } }
void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException { checkServiceStarted(); if (!isInitialized()) { throw new PleaseHoldException("Master is initializing"); } }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 * @throws IOException * @throws ZooKeeperConnectionException * @throws MasterNotRunningException * @throws ServiceException */ //TODO: Clean up, with Procedure V2 and nonce to prevent the same procedure to call mulitple // time, this test is invalid anymore. Just keep the test around for some time before // fully removing it. @Ignore @Test public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { LOG.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null) connection.close(); } }
void checkInitialized() throws PleaseHoldException { if (!this.initialized) { throw new PleaseHoldException("Master is initializing"); } }
void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException { checkServiceStarted(); if (!this.initialized) { throw new PleaseHoldException("Master is initializing"); } }
public ReportRegionStateTransitionResponse reportRegionStateTransition( final ReportRegionStateTransitionRequest req) throws PleaseHoldException { final ReportRegionStateTransitionResponse.Builder builder = ReportRegionStateTransitionResponse.newBuilder(); final ServerName serverName = ProtobufUtil.toServerName(req.getServer()); try { for (RegionStateTransition transition: req.getTransitionList()) { switch (transition.getTransitionCode()) { case OPENED: case FAILED_OPEN: case CLOSED: assert transition.getRegionInfoCount() == 1 : transition; final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); updateRegionTransition(serverName, transition.getTransitionCode(), hri, transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM); break; case READY_TO_SPLIT: case SPLIT: case SPLIT_REVERTED: assert transition.getRegionInfoCount() == 3 : transition; final RegionInfo parent = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); final RegionInfo splitA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); final RegionInfo splitB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); updateRegionSplitTransition(serverName, transition.getTransitionCode(), parent, splitA, splitB); break; case READY_TO_MERGE: case MERGED: case MERGE_REVERTED: assert transition.getRegionInfoCount() == 3 : transition; final RegionInfo merged = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); final RegionInfo mergeA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); final RegionInfo mergeB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); updateRegionMergeTransition(serverName, transition.getTransitionCode(), merged, mergeA, mergeB); break; } } } catch (PleaseHoldException e) { if (LOG.isTraceEnabled()) LOG.trace("Failed transition " + e.getMessage()); throw e; } catch (UnsupportedOperationException|IOException e) { // TODO: at the moment we have a single error message and the RS will abort // if the master says that one of the region transitions failed. LOG.warn("Failed transition", e); builder.setErrorMessage("Failed transition " + e.getMessage()); } return builder.build(); }
void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException, MasterNotRunningException { checkServiceStarted(); if (!isInitialized()) throw new PleaseHoldException("Master is initializing"); if (isStopped()) throw new MasterNotRunningException(); }
/** * Verify that PleaseHoldException gets retried. * HBASE-8764 */ //TODO: Clean up, with Procedure V2 and nonce to prevent the same procedure to call mulitple // time, this test is invalid anymore. Just keep the test around for some time before // fully removing it. @Ignore @Test public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, org.apache.hbase.thirdparty.com.google.protobuf.ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1); final int count = 10; configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); fail(); } catch (RetriesExhaustedException e) { LOG.info("Expected fail", e); } // Assert we were called 'count' times. Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any()); } finally { admin.close(); if (connection != null) connection.close(); } }
