/** * Check if the exception is something that indicates that we cannot * contact/communicate with the server. * * @param e * @return true when exception indicates that the client wasn't able to make contact with server */ private boolean isConnectionException(Throwable e) { if (e == null) return false; // This list covers most connectivity exceptions but not all. // For example, in SocketOutputStream a plain IOException is thrown // at times when the channel is closed. return (e instanceof SocketTimeoutException || e instanceof ConnectException || e instanceof ClosedChannelException || e instanceof SyncFailedException || e instanceof EOFException || e instanceof TimeoutException || e instanceof ConnectionClosingException || e instanceof FailedServerException); }
/** * Check if the exception is something that indicates that we cannot * contact/communicate with the server. * * @param e exception to check * @return true when exception indicates that the client wasn't able to make contact with server */ public static boolean isConnectionException(Throwable e) { if (e == null) { return false; } // This list covers most connectivity exceptions but not all. // For example, in SocketOutputStream a plain IOException is thrown // at times when the channel is closed. return (e instanceof SocketTimeoutException || e instanceof ConnectException || e instanceof ClosedChannelException || e instanceof SyncFailedException || e instanceof EOFException || e instanceof TimeoutException || e instanceof CallTimeoutException || e instanceof ConnectionClosingException || e instanceof FailedServerException); }
/** * At master failover, for pending_open region, make sure * sendRegionOpen RPC call is sent to the target regionserver */ private void retrySendRegionOpen(final RegionState regionState) { this.executorService.submit( new EventHandler(server, EventType.M_MASTER_RECOVERY) { @Override public void process() throws IOException { HRegionInfo hri = regionState.getRegion(); ServerName serverName = regionState.getServerName(); ReentrantLock lock = locker.acquireLock(hri.getEncodedName()); try { for (int i = 1; i <= maximumAttempts; i++) { if (!serverManager.isServerOnline(serverName) || server.isStopped() || server.isAborted()) { return; // No need any more } try { if (!regionState.equals(regionStates.getRegionState(hri))) { return; // Region is not in the expected state any more } List<ServerName> favoredNodes = ServerName.EMPTY_SERVER_LIST; if (shouldAssignRegionsWithFavoredNodes) { favoredNodes = ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(hri); } RegionOpeningState regionOpenState = serverManager.sendRegionOpen( serverName, hri, -1, favoredNodes); if (regionOpenState == RegionOpeningState.FAILED_OPENING) { // Failed opening this region, this means the target server didn't get // the original region open RPC, so re-assign it with a new plan LOG.debug("Got failed_opening in retry sendRegionOpen for " + regionState + ", re-assign it"); invokeAssign(hri, true); } return; // Done. } catch (Throwable t) { if (t instanceof RemoteException) { t = ((RemoteException) t).unwrapRemoteException(); } // In case SocketTimeoutException/FailedServerException, retry if (t instanceof java.net.SocketTimeoutException || t instanceof FailedServerException) { Threads.sleep(100); continue; } // For other exceptions, re-assign it LOG.debug("Got exception in retry sendRegionOpen for " + regionState + ", re-assign it", t); invokeAssign(hri); return; // Done. } } } finally { lock.unlock(); } } }); }
/** * At master failover, for pending_close region, make sure * sendRegionClose RPC call is sent to the target regionserver */ private void retrySendRegionClose(final RegionState regionState) { this.executorService.submit( new EventHandler(server, EventType.M_MASTER_RECOVERY) { @Override public void process() throws IOException { HRegionInfo hri = regionState.getRegion(); ServerName serverName = regionState.getServerName(); ReentrantLock lock = locker.acquireLock(hri.getEncodedName()); try { for (int i = 1; i <= maximumAttempts; i++) { if (!serverManager.isServerOnline(serverName) || server.isStopped() || server.isAborted()) { return; // No need any more } try { if (!regionState.equals(regionStates.getRegionState(hri))) { return; // Region is not in the expected state any more } if (!serverManager.sendRegionClose(serverName, hri, -1, null, false)) { // This means the region is still on the target server LOG.debug("Got false in retry sendRegionClose for " + regionState + ", re-close it"); invokeUnAssign(hri); } return; // Done. } catch (Throwable t) { if (t instanceof RemoteException) { t = ((RemoteException) t).unwrapRemoteException(); } // In case SocketTimeoutException/FailedServerException, retry if (t instanceof java.net.SocketTimeoutException || t instanceof FailedServerException) { Threads.sleep(100); continue; } if (!(t instanceof NotServingRegionException || t instanceof RegionAlreadyInTransitionException)) { // NotServingRegionException/RegionAlreadyInTransitionException // means the target server got the original region close request. // For other exceptions, re-close it LOG.debug("Got exception in retry sendRegionClose for " + regionState + ", re-close it", t); invokeUnAssign(hri); } return; // Done. } } } finally { lock.unlock(); } } }); }
@Test public void testRetryNoRouteToHostException() throws Throwable { WriteConfiguration config = new DefaultWriteConfiguration(new Monitor(0,0,10,10L,0),pef); WriteResult writeResult = new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host"); BulkWriteResult bulkWriteResult = new BulkWriteResult(writeResult); WriteResponse response = config.processGlobalResult(bulkWriteResult); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list"); bulkWriteResult = new BulkWriteResult(writeResult); response = config.processGlobalResult(bulkWriteResult); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.FAILED, "ServerNotRunningYetException"); bulkWriteResult = new BulkWriteResult(writeResult); response = config.processGlobalResult(bulkWriteResult); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.FAILED, "ConnectTimeoutException"); bulkWriteResult = new BulkWriteResult(writeResult); response = config.processGlobalResult(bulkWriteResult); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.PARTIAL); IntObjectOpenHashMap<WriteResult> failedRows = new IntObjectOpenHashMap<>(); failedRows.put(1, new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host")); bulkWriteResult = new BulkWriteResult(writeResult, new IntOpenHashSet(), failedRows); response = config.partialFailure(bulkWriteResult, null); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.PARTIAL); failedRows = new IntObjectOpenHashMap<>(); failedRows.put(1, new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list")); bulkWriteResult = new BulkWriteResult(writeResult, new IntOpenHashSet(), failedRows); response = config.partialFailure(bulkWriteResult, null); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.PARTIAL); failedRows = new IntObjectOpenHashMap<>(); failedRows.put(1, new WriteResult(Code.FAILED, "ServerNotRunningYetException")); bulkWriteResult = new BulkWriteResult(writeResult, new IntOpenHashSet(), failedRows); response = config.partialFailure(bulkWriteResult, null); Assert.assertEquals(WriteResponse.RETRY, response); writeResult = new WriteResult(Code.PARTIAL); failedRows = new IntObjectOpenHashMap<>(); failedRows.put(1, new WriteResult(Code.FAILED, "ConnectTimeoutException")); bulkWriteResult = new BulkWriteResult(writeResult, new IntOpenHashSet(), failedRows); response = config.partialFailure(bulkWriteResult, null); Assert.assertEquals(WriteResponse.RETRY, response); NoRouteToHostException nrthe = new NoRouteToHostException(); response = config.globalError(nrthe); Assert.assertEquals(WriteResponse.RETRY, response); FailedServerException failedServerException = new FailedServerException("Failed server"); response = config.globalError(failedServerException); Assert.assertEquals(WriteResponse.RETRY, response); ServerNotRunningYetException serverNotRunningYetException = new ServerNotRunningYetException("Server not running"); response = config.globalError(serverNotRunningYetException); Assert.assertEquals(WriteResponse.RETRY, response); ConnectTimeoutException connectTimeoutException = new ConnectTimeoutException("connect timeout"); response = config.globalError(connectTimeoutException); Assert.assertEquals(WriteResponse.RETRY, response); }
