@Test public void startReturnDifferentInstances() { ExponentialBackOff backOff = new ExponentialBackOff(); backOff.setInitialInterval(2000L); backOff.setMultiplier(2.0); backOff.setMaxElapsedTime(4000L); BackOffExecution execution = backOff.start(); BackOffExecution execution2 = backOff.start(); assertEquals(2000l, execution.nextBackOff()); assertEquals(2000l, execution2.nextBackOff()); assertEquals(4000l, execution.nextBackOff()); assertEquals(4000l, execution2.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution2.nextBackOff()); }
/** * Apply the next back off time using the specified {@link BackOffExecution}. * <p>Return {@code true} if the back off period has been applied and a new * attempt to recover should be made, {@code false} if no further attempt * should be made. */ protected boolean applyBackOffTime(BackOffExecution execution) { long interval = execution.nextBackOff(); if (interval == BackOffExecution.STOP) { return false; } else { try { Thread.sleep(interval); } catch (InterruptedException interEx) { // Re-interrupt current thread, to allow other threads to react. Thread.currentThread().interrupt(); } } return true; }
@Test public void applyBackOff() { BackOff mock = mock(BackOff.class); BackOffExecution execution = mock(BackOffExecution.class); given(execution.nextBackOff()).willReturn(BackOffExecution.STOP); given(mock.start()).willReturn(execution); DefaultMessageListenerContainer container = createContainer(mock, createFailingContainerFactory()); container.start(); assertEquals(true, container.isRunning()); container.refreshConnectionUntilSuccessful(); assertEquals(false, container.isRunning()); verify(mock).start(); verify(execution).nextBackOff(); }
@Test public void defaultInstance() { ExponentialBackOff backOff = new ExponentialBackOff(); BackOffExecution execution = backOff.start(); assertEquals(2000l, execution.nextBackOff()); assertEquals(3000l, execution.nextBackOff()); assertEquals(4500l, execution.nextBackOff()); }
@Test public void simpleIncrease() { ExponentialBackOff backOff = new ExponentialBackOff(100L, 2.0); BackOffExecution execution = backOff.start(); assertEquals(100l, execution.nextBackOff()); assertEquals(200l, execution.nextBackOff()); assertEquals(400l, execution.nextBackOff()); assertEquals(800l, execution.nextBackOff()); }
@Test public void fixedIncrease() { ExponentialBackOff backOff = new ExponentialBackOff(100L, 1.0); backOff.setMaxElapsedTime(300l); BackOffExecution execution = backOff.start(); assertEquals(100l, execution.nextBackOff()); assertEquals(100l, execution.nextBackOff()); assertEquals(100l, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); }
@Test public void maxIntervalReached() { ExponentialBackOff backOff = new ExponentialBackOff(2000L, 2.0); backOff.setMaxInterval(4000L); BackOffExecution execution = backOff.start(); assertEquals(2000l, execution.nextBackOff()); assertEquals(4000l, execution.nextBackOff()); assertEquals(4000l, execution.nextBackOff()); // max reached assertEquals(4000l, execution.nextBackOff()); }
@Test public void maxAttemptsReached() { ExponentialBackOff backOff = new ExponentialBackOff(2000L, 2.0); backOff.setMaxElapsedTime(4000L); BackOffExecution execution = backOff.start(); assertEquals(2000l, execution.nextBackOff()); assertEquals(4000l, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); // > 4 sec wait in total }
@Test public void maxIntervalReachedImmediately() { ExponentialBackOff backOff = new ExponentialBackOff(1000L, 2.0); backOff.setMaxInterval(50L); BackOffExecution execution = backOff.start(); assertEquals(50L, execution.nextBackOff()); assertEquals(50L, execution.nextBackOff()); }
@Test public void toStringContent() { ExponentialBackOff backOff = new ExponentialBackOff(2000L, 2.0); BackOffExecution execution = backOff.start(); assertEquals("ExponentialBackOff{currentInterval=n/a, multiplier=2.0}", execution.toString()); execution.nextBackOff(); assertEquals("ExponentialBackOff{currentInterval=2000ms, multiplier=2.0}", execution.toString()); execution.nextBackOff(); assertEquals("ExponentialBackOff{currentInterval=4000ms, multiplier=2.0}", execution.toString()); }
@Test public void defaultInstance() { FixedBackOff backOff = new FixedBackOff(); BackOffExecution execution = backOff.start(); for (int i = 0; i < 100; i++) { assertEquals(FixedBackOff.DEFAULT_INTERVAL, execution.nextBackOff()); } }
@Test public void maxAttemptsReached() { FixedBackOff backOff = new FixedBackOff(200L, 2); BackOffExecution execution = backOff.start(); assertEquals(200l, execution.nextBackOff()); assertEquals(200l, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); }
@Test public void startReturnDifferentInstances() { FixedBackOff backOff = new FixedBackOff(100L, 1); BackOffExecution execution = backOff.start(); BackOffExecution execution2 = backOff.start(); assertEquals(100l, execution.nextBackOff()); assertEquals(100l, execution2.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution2.nextBackOff()); }
@Test public void liveUpdate() { FixedBackOff backOff = new FixedBackOff(100L, 1); BackOffExecution execution = backOff.start(); assertEquals(100l, execution.nextBackOff()); backOff.setInterval(200l); backOff.setMaxAttempts(2); assertEquals(200l, execution.nextBackOff()); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); }
@Test public void toStringContent() { FixedBackOff backOff = new FixedBackOff(200L, 10); BackOffExecution execution = backOff.start(); assertEquals("FixedBackOff{interval=200, currentAttempts=0, maxAttempts=10}", execution.toString()); execution.nextBackOff(); assertEquals("FixedBackOff{interval=200, currentAttempts=1, maxAttempts=10}", execution.toString()); execution.nextBackOff(); assertEquals("FixedBackOff{interval=200, currentAttempts=2, maxAttempts=10}", execution.toString()); }
@Test public void applyBackOffRetry() { BackOff mock = mock(BackOff.class); BackOffExecution execution = mock(BackOffExecution.class); given(execution.nextBackOff()).willReturn(50L, BackOffExecution.STOP); given(mock.start()).willReturn(execution); DefaultMessageListenerContainer container = createContainer(mock, createFailingContainerFactory()); container.start(); container.refreshConnectionUntilSuccessful(); assertEquals(false, container.isRunning()); verify(mock).start(); verify(execution, times(2)).nextBackOff(); }
@Test public void recoverResetBackOff() { BackOff mock = mock(BackOff.class); BackOffExecution execution = mock(BackOffExecution.class); given(execution.nextBackOff()).willReturn(50L, 50L, 50L); // 3 attempts max given(mock.start()).willReturn(execution); DefaultMessageListenerContainer container = createContainer(mock, createRecoverableContainerFactory(1)); container.start(); container.refreshConnectionUntilSuccessful(); assertEquals(true, container.isRunning()); verify(mock).start(); verify(execution, times(1)).nextBackOff(); // only on attempt as the second one lead to a recovery }
@Override public Response intercept(Chain chain) throws IOException { ExponentialBackOff backoff = new ExponentialBackOff(); backoff.setMaxElapsedTime(maxElapsedBackoffMs); BackOffExecution backOffExec = backoff.start(); Response response = null; long waitTime = 0; while (waitTime != BackOffExecution.STOP) { Request request = chain.request(); response = chain.proceed(request); if (response.isSuccessful() || NON_RETRYABLE_METHODS.contains(request.method()) || response.code() == 404) { return response; } try { waitTime = backOffExec.nextBackOff(); if (waitTime != BackOffExecution.STOP) { response.body().close(); log.warn("Request for " + request.urlString() + " failed. Backing off for " + waitTime + "ms"); Thread.sleep(waitTime); } } catch (Throwable ignored) { break; } } return response; }
public static void handleRegisterClientException(HystrixRuntimeException e, BackOff exponentialBackOff, BackOffExecution backOffExecution, String configServiceUrl) { Throwable cause = e.getCause(); log.debug("Exception registering client, exception getMessage={}", e.getMessage()); log.debug("Exception registering client, cause getMessage={}", cause.getMessage()); if (cause instanceof ConnectException) { log.debug("Connection refused to ConfigService url={}", configServiceUrl); } else if (cause instanceof InternalServerErrorException) { log.debug("Internal server error in ConfigService url={}", configServiceUrl); } else if(cause instanceof NotFoundException) { log.debug("404 not found to ConfigService url={}", configServiceUrl); } else if (cause instanceof BadRequestException) { log.error("400 Bad Request. Probably need to fix something on the client. Exiting after a" + " wait, so as to not DDoS the server."); // TODO Do a sensible BackOff implementation class comparissmnet before this!!! SleepUtil.sleepWithLogging(((ExponentialBackOff)exponentialBackOff).getMaxInterval() * 2); System.exit(1); } else if (cause instanceof TimeoutException) { log.debug("CommandRegisterClient timed out."); } else { log.error("Couldn't handle exception: {}", e); } SleepUtil.sleepWithLogging(backOffExecution.nextBackOff()); }
public ClientConfig registerClient() { BackOff exponentialBackOff = new ExponentialBackOff(); BackOffExecution backOffExecution = exponentialBackOff.start(); while (true) { try { return new CommandRegisterClient(artifactId, configServiceClient, clientName, clientId).execute(); } catch (HystrixRuntimeException e) { RegisterClientExceptionHandler.handleRegisterClientException(e, exponentialBackOff, backOffExecution, configServiceClient.getUrl()); } } }
@Test public void testShouldHandleConnectExceptionWithRetry() { BackOffExecution backOffExecution = mock(BackOffExecution.class); when(backOffExecution.nextBackOff()).thenReturn((long) 1); HystrixRuntimeException hystrixRuntimeException = mock(HystrixRuntimeException.class); when(hystrixRuntimeException.getCause()).thenReturn(new ConnectException()); RegisterClientExceptionHandler.handleRegisterClientException(hystrixRuntimeException, null, backOffExecution, ""); verify(backOffExecution).nextBackOff(); }
@Test public void testShouldHandleInternalServerErrorExceptionWithRetry() { BackOffExecution backOffExecution = mock(BackOffExecution.class); when(backOffExecution.nextBackOff()).thenReturn((long) 1); HystrixRuntimeException hystrixRuntimeException = mock(HystrixRuntimeException.class); when(hystrixRuntimeException.getCause()).thenReturn(new InternalServerErrorException()); RegisterClientExceptionHandler.handleRegisterClientException(hystrixRuntimeException, null, backOffExecution, ""); verify(backOffExecution).nextBackOff(); }
@Test public void testShouldHandleNotFoundExceptionWithRetry() { BackOffExecution backOffExecution = mock(BackOffExecution.class); when(backOffExecution.nextBackOff()).thenReturn((long) 1); HystrixRuntimeException hystrixRuntimeException = mock(HystrixRuntimeException.class); NotFoundException notFoundException = new NotFoundException(); when(hystrixRuntimeException.getCause()).thenReturn(notFoundException); RegisterClientExceptionHandler.handleRegisterClientException(hystrixRuntimeException, null, backOffExecution, ""); verify(backOffExecution).nextBackOff(); }
@Test public void testShouldHandleTimeoutExceptionWithRetry() { BackOffExecution backOffExecution = mock(BackOffExecution.class); when(backOffExecution.nextBackOff()).thenReturn((long) 1); HystrixRuntimeException hystrixRuntimeException = mock(HystrixRuntimeException.class); when(hystrixRuntimeException.getCause()).thenReturn(new TimeoutException()); RegisterClientExceptionHandler.handleRegisterClientException(hystrixRuntimeException, null, backOffExecution, ""); verify(backOffExecution).nextBackOff(); }
public static void main(String[] args) { ExponentialBackOff backOff = new ExponentialBackOff(100, 1.5);// 初始间隔,递增倍数(上次基础上) backOff.setMaxInterval(5 * 1000L);// 最大间隔(上一次和下一次最大间隔) backOff.setMaxElapsedTime(50 * 1000L);// 最大总时间间隔(第一次和最后一次间隔) BackOffExecution execution = backOff.start(); for (int i = 1; i <= 18; i++) { System.out.println(execution.nextBackOff()); } System.out.println(execution.nextBackOff()); }
@Test public void noAttemptAtAll() { FixedBackOff backOff = new FixedBackOff(100L, 0L); BackOffExecution execution = backOff.start(); assertEquals(BackOffExecution.STOP, execution.nextBackOff()); }
public long syncAndReturn() { FixedBackOff backoff = new FixedBackOff(); backoff.setInterval(retryIntervalMs); BackOffExecution backOffExec = backoff.start(); if (!isServerHealthy()) { log.warn("Server is currently UNHEALTHY. User permission role synchronization and " + "resolution may not complete until this server becomes healthy again."); } while (true) { try { Map<String, UserPermission> combo = new HashMap<>(); Map<String, UserPermission> temp; if (!(temp = getUserPermissions()).isEmpty()) { combo.putAll(temp); } if (!(temp = getServiceAccountsAsMap()).isEmpty()) { combo.putAll(temp); } return updateUserPermissions(combo); } catch (ProviderException|PermissionResolutionException ex) { Status status = healthIndicator.health().getStatus(); long waitTime = backOffExec.nextBackOff(); if (waitTime == BackOffExecution.STOP) { log.error("Unable to resolve service account permissions.", ex); return 0; } String message = new StringBuilder("User permission sync failed. ") .append("Server status is ") .append(status) .append(". Trying again in ") .append(waitTime) .append(" ms. Cause:") .append(ex.getMessage()) .toString(); if (log.isDebugEnabled()) { log.debug(message, ex); } else { log.warn(message); } try { Thread.sleep(waitTime); } catch (InterruptedException ignored) { } } finally { isServerHealthy(); } } }
/** * Apply the back off time once. In a regular scenario, the back off is only applied if we * failed to recover with the broker. This additional sleep period avoids a burst retry * scenario when the broker is actually up but something else if failing (i.e. listener * specific). */ private void sleepBeforeRecoveryAttempt() { BackOffExecution execution = DefaultMessageListenerContainer.this.backOff.start(); applyBackOffTime(execution); }