/** * At master failover, for pending_close region, make sure * sendRegionClose RPC call is sent to the target regionserver */ private void retrySendRegionClose(final RegionState regionState) { this.executorService.submit( new EventHandler(server, EventType.M_MASTER_RECOVERY) { @Override public void process() throws IOException { HRegionInfo hri = regionState.getRegion(); ServerName serverName = regionState.getServerName(); ReentrantLock lock = locker.acquireLock(hri.getEncodedName()); try { for (int i = 1; i <= maximumAttempts; i++) { if (!serverManager.isServerOnline(serverName) || server.isStopped() || server.isAborted()) { return; // No need any more } try { if (!regionState.equals(regionStates.getRegionState(hri))) { return; // Region is not in the expected state any more } if (!serverManager.sendRegionClose(serverName, hri, -1, null, false)) { // This means the region is still on the target server LOG.debug("Got false in retry sendRegionClose for " + regionState + ", re-close it"); invokeUnAssign(hri); } return; // Done. } catch (Throwable t) { if (t instanceof RemoteException) { t = ((RemoteException) t).unwrapRemoteException(); } // In case SocketTimeoutException/FailedServerException, retry if (t instanceof java.net.SocketTimeoutException || t instanceof FailedServerException) { Threads.sleep(100); continue; } if (!(t instanceof NotServingRegionException || t instanceof RegionAlreadyInTransitionException)) { // NotServingRegionException/RegionAlreadyInTransitionException // means the target server got the original region close request. // For other exceptions, re-close it LOG.debug("Got exception in retry sendRegionClose for " + regionState + ", re-close it", t); invokeUnAssign(hri); } return; // Done. } } } finally { lock.unlock(); } } }); }
@Override public boolean addRegionsInTransition(HRegionInfo region, String currentAction) throws RegionAlreadyInTransitionException { return super.addRegionsInTransition(region, currentAction); }
/** * Send CLOSE RPC if the server is online, otherwise, offline the region */ private void unassign(final HRegionInfo region, final RegionState state, final int versionOfClosingNode, final ServerName dest, final boolean transitionInZK) { // Send CLOSE RPC ServerName server = state.getServerName(); // ClosedRegionhandler can remove the server from this.regions if (!serverManager.isServerOnline(server)) { if (transitionInZK) { // delete the node. if no node exists need not bother. deleteClosingOrClosedNode(region); } regionOffline(region); return; } for (int i = 1; i <= this.maximumAttempts; i++) { try { if (serverManager.sendRegionClose(server, region, versionOfClosingNode, dest, transitionInZK)) { LOG.debug("Sent CLOSE to " + server + " for region " + region.getRegionNameAsString()); return; } // This never happens. Currently regionserver close always return true. // Todo; this can now happen (0.96) if there is an exception in a coprocessor LOG.warn("Server " + server + " region CLOSE RPC returned false for " + region.getRegionNameAsString()); } catch (Throwable t) { if (t instanceof RemoteException) { t = ((RemoteException)t).unwrapRemoteException(); } if (t instanceof NotServingRegionException) { if (transitionInZK) { deleteClosingOrClosedNode(region); } regionOffline(region); return; } else if (t instanceof RegionAlreadyInTransitionException) { // RS is already processing this region, only need to update the timestamp LOG.debug("update " + state + " the timestamp."); state.updateTimestampToNow(); } LOG.info("Server " + server + " returned " + t + " for " + region.getRegionNameAsString() + ", try=" + i + " of " + this.maximumAttempts, t); // Presume retry or server will expire. } } }
