private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicasConfigured) { // unassign the unneeded replicas (for e.g., if the previous master was configured // with a replication of 3 and now it is 2, we need to unassign the 1 unneeded replica) try { List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes(); for (String metaReplicaZnode : metaReplicaZnodes) { int replicaId = zooKeeper.getMetaReplicaIdFromZnode(metaReplicaZnode); if (replicaId >= numMetaReplicasConfigured) { RegionState r = MetaTableLocator.getMetaRegionState(zkw, replicaId); LOG.info("Closing excess replica of meta region " + r.getRegion()); // send a close and wait for a max of 30 seconds ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(), r.getRegion(), 30000); ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId)); } } } catch (Exception ex) { // ignore the exception since we don't want the master to be wedged due to potential // issues in the cleanup of the extra regions. We can do that cleanup via hbck or manually LOG.warn("Ignoring exception " + ex); } }
/** * A quick test that hbase:meta is assigned; blocks for short time only. * @return True if hbase:meta location is available and verified as good. * @throws InterruptedException * @throws IOException */ private boolean isMetaAssignedQuickTest(final MasterProcedureEnv env) throws InterruptedException, IOException { ZooKeeperWatcher zkw = env.getMasterServices().getZooKeeper(); MetaTableLocator mtl = env.getMasterServices().getMetaTableLocator(); boolean metaAssigned = false; // Is hbase:meta location available yet? if (mtl.isLocationAvailable(zkw)) { ClusterConnection connection = env.getMasterServices().getConnection(); // Is hbase:meta location good yet? long timeout = env.getMasterConfiguration().getLong(KEY_SHORT_WAIT_ON_META, DEFAULT_SHORT_WAIT_ON_META); if (mtl.verifyMetaRegionLocation(connection, zkw, timeout)) { metaAssigned = true; } } return metaAssigned; }
/** * Used by the client to identify if all regions have the schema updates * * @param tableName * @return Pair indicating the status of the alter command * @throws IOException */ public Pair<Integer, Integer> getReopenStatus(TableName tableName) throws IOException { List<HRegionInfo> hris; if (TableName.META_TABLE_NAME.equals(tableName)) { hris = new MetaTableLocator().getMetaRegions(server.getZooKeeper()); } else { hris = MetaTableAccessor.getTableRegions(server.getZooKeeper(), server.getConnection(), tableName, true); } Integer pending = 0; for (HRegionInfo hri : hris) { String name = hri.getEncodedName(); // no lock concurrent access ok: sequential consistency respected. if (regionsToReopen.containsKey(name) || regionStates.isRegionInTransition(name)) { pending++; } } return new Pair<Integer, Integer>(pending, hris.size()); }
private void testVerifyMetaRegionLocationWithException(Exception ex) throws IOException, InterruptedException, KeeperException, ServiceException { // Mock an ClientProtocol. final ClientProtos.ClientService.BlockingInterface implementation = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); ClusterConnection connection = mockConnection(null, implementation); // If a 'get' is called on mocked interface, throw connection refused. Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())). thenThrow(new ServiceException(ex)); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING); assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); }
/** * Test get of meta region fails properly if nothing to connect to. * @throws IOException * @throws InterruptedException * @throws KeeperException * @throws ServiceException */ @Test public void testVerifyMetaRegionLocationFails() throws IOException, InterruptedException, KeeperException, ServiceException { ClusterConnection connection = Mockito.mock(ClusterConnection.class); ServiceException connectException = new ServiceException(new ConnectException("Connection refused")); final AdminProtos.AdminService.BlockingInterface implementation = Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(), (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException); Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))). thenReturn(implementation); RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class); Mockito.when(controllerFactory.newController()).thenReturn( Mockito.mock(PayloadCarryingRpcController.class)); Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory); ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis()); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPENING); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); }
/** * Test waiting on meat w/ no timeout specified. * @throws IOException * @throws InterruptedException * @throws KeeperException */ @Test public void testNoTimeoutWaitForMeta() throws IOException, InterruptedException, KeeperException { final MetaTableLocator mtl = new MetaTableLocator(); ServerName hsa = mtl.getMetaRegionLocation(watcher); assertNull(hsa); // Now test waiting on meta location getting set. Thread t = new WaitOnMetaThread(); startWaitAliveThenWaitItLives(t, 1); // Set a meta location. MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); hsa = SN; // Join the thread... should exit shortly. t.join(); // Now meta is available. assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa)); }
/** * Test get of meta region fails properly if nothing to connect to. * @throws IOException * @throws InterruptedException * @throws KeeperException * @throws ServiceException */ @Test public void testVerifyMetaRegionLocationFails() throws IOException, InterruptedException, KeeperException, ServiceException { ClusterConnection connection = Mockito.mock(ClusterConnection.class); ServiceException connectException = new ServiceException(new ConnectException("Connection refused")); final AdminProtos.AdminService.BlockingInterface implementation = Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(), (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException); Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))). thenReturn(implementation); ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis()); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPENING); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); }
@Override public RegionLocations getMetaRegionLocation() throws IOException { ZooKeeperKeepAliveConnection zkw = hci.getKeepAliveZooKeeperWatcher(); try { if (LOG.isTraceEnabled()) { LOG.trace("Looking up meta region location in ZK," + " connection=" + this); } ServerName servername = new MetaTableLocator().blockUntilAvailable(zkw, hci.rpcTimeout); if (LOG.isTraceEnabled()) { LOG.trace("Looked up meta region location, connection=" + this + "; serverName=" + ((servername == null) ? "null" : servername)); } if (servername == null) return null; HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername, 0); return new RegionLocations(new HRegionLocation[] {loc}); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } finally { zkw.close(); } }
/** * Ensure Armeria's dependencies do not cause a trouble with hbase-shaded-client. * * @see <a href="https://issues.apache.org/jira/browse/HBASE-14963">HBASE-14963</a> */ @Test(expected = NotAllMetaRegionsOnlineException.class) public void testGuavaConflict() throws Exception { // Make sure Armeria is available in the class path. assertThat(Version.identify(Server.class.getClassLoader())).isNotNull(); // Make sure newer Guava is available in the class path. assertThat(Stopwatch.class.getDeclaredConstructor().getModifiers()).is(new Condition<>( value -> !Modifier.isPublic(value), "Recent Guava Stopwatch should have non-public default constructor.")); final MetaTableLocator locator = new MetaTableLocator(); final ZooKeeperWatcher zkw = mock(ZooKeeperWatcher.class); final RecoverableZooKeeper zk = mock(RecoverableZooKeeper.class); when(zkw.getRecoverableZooKeeper()).thenReturn(zk); when(zk.exists(any(), any())).thenReturn(new Stat(0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0)); locator.waitMetaRegionLocation(zkw, 100); }
private void unassignExcessMetaReplica(int numMetaReplicasConfigured) { final ZKWatcher zooKeeper = master.getZooKeeper(); // unassign the unneeded replicas (for e.g., if the previous master was configured // with a replication of 3 and now it is 2, we need to unassign the 1 unneeded replica) try { List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes(); for (String metaReplicaZnode : metaReplicaZnodes) { int replicaId = zooKeeper.znodePaths.getMetaReplicaIdFromZnode(metaReplicaZnode); if (replicaId >= numMetaReplicasConfigured) { RegionState r = MetaTableLocator.getMetaRegionState(zooKeeper, replicaId); LOG.info("Closing excess replica of meta region " + r.getRegion()); // send a close and wait for a max of 30 seconds ServerManager.closeRegionSilentlyAndWait(master.getClusterConnection(), r.getServerName(), r.getRegion(), 30000); ZKUtil.deleteNode(zooKeeper, zooKeeper.znodePaths.getZNodeForReplica(replicaId)); } } } catch (Exception ex) { // ignore the exception since we don't want the master to be wedged due to potential // issues in the cleanup of the extra regions. We can do that cleanup via hbck or manually LOG.warn("Ignoring exception " + ex); } }
/** * Test get of meta region fails properly if nothing to connect to. * @throws IOException * @throws InterruptedException * @throws KeeperException * @throws ServiceException */ @Test public void testVerifyMetaRegionLocationFails() throws IOException, InterruptedException, KeeperException, ServiceException { ClusterConnection connection = Mockito.mock(ClusterConnection.class); ServiceException connectException = new ServiceException(new ConnectException("Connection refused")); final AdminProtos.AdminService.BlockingInterface implementation = Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(), (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException); Mockito.when(connection.getAdmin(Mockito.any())). thenReturn(implementation); RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class); Mockito.when(controllerFactory.newController()).thenReturn( Mockito.mock(HBaseRpcController.class)); Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory); ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis()); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPENING); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN); assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); }
protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env, final TableName tableName) throws IOException { return ProcedureSyncWait.waitFor(env, "regions of table=" + tableName + " from meta", new ProcedureSyncWait.Predicate<List<HRegionInfo>>() { @Override public List<HRegionInfo> evaluate() throws IOException { if (TableName.META_TABLE_NAME.equals(tableName)) { return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper()); } return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(), env.getMasterServices().getConnection(), tableName); } }); }
private ServerName getMetaRegionServerName(int replicaId) throws IOException, KeeperException { ZooKeeperWatcher zkw = createZooKeeperWatcher(); ServerName sn = null; try { sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId); } finally { zkw.close(); } return sn; }
/** * Setup our cluster connection if not already initialized. * * @throws IOException */ protected synchronized void setupClusterConnection() throws IOException { if (clusterConnection == null) { clusterConnection = createClusterConnection(); metaTableLocator = new MetaTableLocator(); } }
@After public void after() { try { // Clean out meta location or later tests will be confused... they presume // start fresh in zk. new MetaTableLocator().deleteMetaLocation(this.watcher); } catch (KeeperException e) { LOG.warn("Unable to delete hbase:meta location", e); } this.watcher.close(); }
/** * Test normal operations */ @Test public void testMetaLookup() throws IOException, InterruptedException, ServiceException, KeeperException { final ClientProtos.ClientService.BlockingInterface client = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). thenReturn(GetResponse.newBuilder().build()); final MetaTableLocator mtl = new MetaTableLocator(); assertNull(mtl.getMetaRegionLocation(this.watcher)); for (RegionState.State state : RegionState.State.values()) { if (state.equals(RegionState.State.OPEN)) continue; MetaTableLocator.setMetaLocation(this.watcher, SN, state); assertNull(mtl.getMetaRegionLocation(this.watcher)); assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState()); } MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); assertEquals(mtl.getMetaRegionLocation(this.watcher), SN); assertEquals(RegionState.State.OPEN, MetaTableLocator.getMetaRegionState(this.watcher).getState()); mtl.deleteMetaLocation(this.watcher); assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName()); assertEquals(MetaTableLocator.getMetaRegionState(this.watcher).getState(), RegionState.State.OFFLINE); assertNull(mtl.getMetaRegionLocation(this.watcher)); }
/** * Test interruptable while blocking wait on meta. * @throws IOException * @throws ServiceException * @throws InterruptedException */ @Test public void testInterruptWaitOnMeta() throws IOException, InterruptedException, ServiceException { final ClientProtos.ClientService.BlockingInterface client = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). thenReturn(GetResponse.newBuilder().build()); final MetaTableLocator mtl = new MetaTableLocator(); ServerName meta = new MetaTableLocator().getMetaRegionLocation(this.watcher); assertNull(meta); Thread t = new Thread() { @Override public void run() { try { mtl.waitMetaRegionLocation(watcher); } catch (InterruptedException e) { throw new RuntimeException("Interrupted", e); } } }; t.start(); while (!t.isAlive()) Threads.sleep(1); Threads.sleep(1); assertTrue(t.isAlive()); mtl.stop(); // Join the thread... should exit shortly. t.join(); }
void doWaiting() throws InterruptedException { try { while (new MetaTableLocator().waitMetaRegionLocation(watcher, 10000) == null); } catch (NotAllMetaRegionsOnlineException e) { //Ignore } }
@Test(timeout=180000) public void testFixAssignmentsWhenMETAinTransition() throws Exception { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO); regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper()); assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO)); HBaseFsck hbck = doFsck(conf, true); assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION, ERROR_CODE.NULL_META_REGION }); assertNoErrors(doFsck(conf, false)); }
private ServerName getMetaRegionServerName() throws IOException, KeeperException { ZooKeeperWatcher zkw = createZooKeeperWatcher(); ServerName sn = null; try { sn = new MetaTableLocator().getMetaRegionLocation(zkw); } finally { zkw.close(); } return sn; }
@Override public void postOpenDeployTasks(final HRegion r) throws KeeperException, IOException { rpcServices.checkOpen(); LOG.info("Post open deploy tasks for " + r.getRegionNameAsString()); // Do checks to see if we need to compact (references or too many files) for (Store s : r.getStores().values()) { if (s.hasReferences() || s.needsCompaction()) { this.compactSplitThread.requestSystemCompaction(r, s, "Opening Region"); } } long openSeqNum = r.getOpenSeqNum(); if (openSeqNum == HConstants.NO_SEQNUM) { // If we opened a region, we should have read some sequence number from it. LOG.error("No sequence number found when opening " + r.getRegionNameAsString()); openSeqNum = 0; } // Update flushed sequence id of a recovering region in ZK updateRecoveringRegionLastFlushedSequenceId(r); // Update ZK, or META if (r.getRegionInfo().isMetaRegion()) { MetaTableLocator.setMetaLocation(getZooKeeper(), serverName, State.OPEN); } else if (useZKForAssignment) { MetaTableAccessor.updateRegionLocation(getConnection(), r.getRegionInfo(), this.serverName, openSeqNum); } if (!useZKForAssignment && !reportRegionStateTransition( TransitionCode.OPENED, openSeqNum, r.getRegionInfo())) { throw new IOException("Failed to report opened region to master: " + r.getRegionNameAsString()); } LOG.debug("Finished post open deploy task for " + r.getRegionNameAsString()); }
/** * Test verifies whether stale znodes of unknown tables as for the hbase:meta will be removed or * not. * @throws KeeperException * @throws IOException * @throws Exception */ @Test (timeout=180000) public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta() throws Exception { List<ServerName> destServers = new ArrayList<ServerName>(1); destServers.add(SERVERNAME_A); Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager( HTU.getConfiguration()); Server server = new HMaster(HTU.getConfiguration(), csm); Whitebox.setInternalState(server, "serverManager", this.serverManager); AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, this.serverManager); Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class)); // Make it so we can get a catalogtracker from servermanager.. .needed // down in guts of server shutdown handler. Whitebox.setInternalState(server, "clusterConnection", am.getConnection()); try { TableName tableName = TableName.valueOf("dummyTable"); // set table in enabling state. am.getTableStateManager().setTableState(tableName, Table.State.ENABLING); am.joinCluster(); assertFalse("Table should not be present in zookeeper.", am.getTableStateManager().isTablePresent(tableName)); } finally { am.shutdown(); } }
protected void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName) throws IOException { try { MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(), State.OPEN); } catch (KeeperException e) { throw new IOException(e); } }
/** * For assigning hbase:meta replicas only. * TODO: The way this assign runs, nothing but chance to stop all replicas showing up on same * server as the hbase:meta region. */ protected void assignMetaReplicas() throws IOException, InterruptedException, KeeperException { int numReplicas = master.getConfiguration().getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM); if (numReplicas <= 1) { // No replicaas to assign. Return. return; } final AssignmentManager assignmentManager = master.getAssignmentManager(); if (!assignmentManager.isMetaInitialized()) { throw new IllegalStateException("hbase:meta must be initialized first before we can " + "assign out its replicas"); } ServerName metaServername = this.master.getMetaTableLocator().getMetaRegionLocation(this.master.getZooKeeper()); for (int i = 1; i < numReplicas; i++) { // Get current meta state for replica from zk. RegionState metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper(), i); RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica( RegionInfoBuilder.FIRST_META_REGIONINFO, i); LOG.debug(hri.getRegionNameAsString() + " replica region state from zookeeper=" + metaState); if (metaServername.equals(metaState.getServerName())) { metaState = null; LOG.info(hri.getRegionNameAsString() + " old location is same as current hbase:meta location; setting location as null..."); } // These assigns run inline. All is blocked till they complete. Only interrupt is shutting // down hosting server which calls AM#stop. if (metaState != null && metaState.getServerName() != null) { // Try to retain old assignment. assignmentManager.assign(hri, metaState.getServerName()); } else { assignmentManager.assign(hri); } } unassignExcessMetaReplica(numReplicas); }
/** * Setup our cluster connection if not already initialized. * @throws IOException */ protected synchronized void setupClusterConnection() throws IOException { if (clusterConnection == null) { clusterConnection = createClusterConnection(); metaTableLocator = new MetaTableLocator(); } }
@Test public void testGetRegionsFromMetaTable() throws IOException, InterruptedException { List<RegionInfo> regions = new MetaTableLocator().getMetaRegions(UTIL.getZooKeeperWatcher()); assertTrue(regions.size() >= 1); assertTrue(new MetaTableLocator().getMetaRegionsAndLocations( UTIL.getZooKeeperWatcher()).size() >= 1); }
/** * Test normal operations */ @Test public void testMetaLookup() throws IOException, InterruptedException, ServiceException, KeeperException { final ClientProtos.ClientService.BlockingInterface client = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). thenReturn(GetResponse.newBuilder().build()); final MetaTableLocator mtl = new MetaTableLocator(); assertNull(mtl.getMetaRegionLocation(this.watcher)); for (RegionState.State state : RegionState.State.values()) { if (state.equals(RegionState.State.OPEN)) continue; MetaTableLocator.setMetaLocation(this.watcher, SN, state); assertNull(mtl.getMetaRegionLocation(this.watcher)); assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState()); } MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); assertEquals(SN, mtl.getMetaRegionLocation(this.watcher)); assertEquals(RegionState.State.OPEN, MetaTableLocator.getMetaRegionState(this.watcher).getState()); mtl.deleteMetaLocation(this.watcher); assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName()); assertEquals(RegionState.State.OFFLINE, MetaTableLocator.getMetaRegionState(this.watcher).getState()); assertNull(mtl.getMetaRegionLocation(this.watcher)); }
