Java 类org.apache.http.impl.client.cache.CacheConfig 实例源码

public static HttpClientBuilder getHttpClientBuilder() {
    // Common CacheConfig for both the JarCacheStorage and the underlying
    // BasicHttpCacheStorage
    final CacheConfig cacheConfig = CacheConfig.custom().setMaxCacheEntries(1000).setMaxObjectSize(1024 * 128)
            .build();

    RequestConfig config = RequestConfig.custom().setConnectTimeout(DEFAULT_TIMEOUT)
            .setConnectionRequestTimeout(DEFAULT_TIMEOUT).setSocketTimeout(DEFAULT_TIMEOUT).build();

    HttpClientBuilder clientBuilder = CachingHttpClientBuilder.create()
            // allow caching
            .setCacheConfig(cacheConfig)
            // Wrap the local JarCacheStorage around a BasicHttpCacheStorage
            .setHttpCacheStorage(new JarCacheStorage(null, cacheConfig, new BasicHttpCacheStorage(cacheConfig)))
            // Support compressed data
            // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/httpagent.html#d5e1238
            .addInterceptorFirst(new RequestAcceptEncoding()).addInterceptorFirst(new ResponseContentEncoding())
            // use system defaults for proxy etc.
            .useSystemProperties().setDefaultRequestConfig(config);
    return clientBuilder;
}

private static CloseableHttpClient newClosableCachingHttpClient(EventStoreSettings settings) {
   final CacheConfig cacheConfig = CacheConfig.custom()
         .setMaxCacheEntries(Integer.MAX_VALUE)
         .setMaxObjectSize(Integer.MAX_VALUE)
         .build();

   settings.getCacheDirectory()
         .mkdirs();

   return CachingHttpClientBuilder.create()
         .setHttpCacheStorage(new FileCacheStorage(cacheConfig, settings.getCacheDirectory()))
         .setCacheConfig(cacheConfig)
         .setDefaultRequestConfig(requestConfig(settings))
         .setDefaultCredentialsProvider(credentialsProvider(settings))
         .setRedirectStrategy(new LaxRedirectStrategy())
         .setRetryHandler(new StandardHttpRequestRetryHandler())
         .setKeepAliveStrategy(new de.qyotta.eventstore.utils.DefaultConnectionKeepAliveStrategy())
         .setConnectionManagerShared(true)

         .build();
}

public MCRRESTResolver() {
    CacheConfig cacheConfig = CacheConfig.custom()
        .setMaxObjectSize(MAX_OBJECT_SIZE)
        .setMaxCacheEntries(MAX_CACHE_ENTRIES)
        .build();
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(REQUEST_TIMEOUT)
        .setSocketTimeout(REQUEST_TIMEOUT)
        .build();
    String userAgent = MessageFormat
        .format("MyCoRe/{0} ({1}; java {2})", MCRCoreVersion.getCompleteVersion(), MCRConfiguration.instance()
            .getString("MCR.NameOfProject", "undefined"), System.getProperty("java.version"));
    this.restClient = CachingHttpClients.custom()
        .setCacheConfig(cacheConfig)
        .setDefaultRequestConfig(requestConfig)
        .setUserAgent(userAgent)
        .build();
    MCRShutdownHandler.getInstance().addCloseable(this::close);
    this.logger = LogManager.getLogger();
}

private CloseableHttpClient createDefaultClient(HttpClientConnectionManager fClientCm) {
    final CacheConfig cacheConfig = CacheConfig.custom()
            .setMaxCacheEntries(DEFAULT_CACHE_MAX_ENTRIES)
            .setMaxObjectSize(DEFAULT_CACHE_MAX_OBJ_SIZE)
            .setHeuristicCachingEnabled(true)
            .setHeuristicDefaultLifetime(DEFAULT_CACHE_TTL_IN_SECS)
            .build();

    final RequestConfig requestConfig = RequestConfig.custom()
            .setConnectTimeout(DEFAULT_REQUEST_CONN_TIMEOUT_IN_MS)
            .setSocketTimeout(DEFAULT_REQUEST_SOCK_TIMEOUT_IN_MS)
            .build();

    return CachingHttpClients.custom()
            .setCacheConfig(cacheConfig)
            .setConnectionManager(fClientCm)
            .setDefaultRequestConfig(requestConfig)
            .build();
}

public HttpEndpoint(URI endpoint, Config cfg, HttpClientContextFactory clientContextFactory) {
    if (endpoint == null) {
        throw new IllegalArgumentException("Endpoint is required");
    }
    if (cfg == null) {
        cfg = new ConfigurationBuilder().build();
    }
    CacheConfig cacheConfig = CacheConfig.custom()
            .setMaxCacheEntries(cfg.getMaxCacheEntries())
            .setMaxObjectSize(cfg.getMaxCacheObjectSize())
            .build();
    RequestConfig requestConfig = RequestConfig.custom()
            .setConnectTimeout(1000 * cfg.getConnectTimeOutSeconds())
            .setSocketTimeout(1000 * cfg.getSocketTimeOutSeconds())
            .build();

    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
    cm.setMaxTotal(cfg.getMaxConections());

    this.endpoint = endpoint;
    this.httpClient = CachingHttpClients.custom()
            .setCacheConfig(cacheConfig)
            .setDefaultRequestConfig(requestConfig)
            .setRetryHandler(new StandardHttpRequestRetryHandler())
            .setConnectionManager(cm)
            .build();
    this.clientContextFactory = clientContextFactory;
    initPingThread(cfg.getPingSeconds());
}

@Override
@Before
public void setUp() throws Exception {
    mockMemcachedClient = mock(MemcachedClientIF.class);
    mockKeyHashingScheme = mock(KeyHashingScheme.class);
    mockMemcachedCacheEntryFactory = mock(MemcachedCacheEntryFactory.class);
    mockMemcachedCacheEntry = mock(MemcachedCacheEntry.class);
    mockMemcachedCacheEntry2 = mock(MemcachedCacheEntry.class);
    mockMemcachedCacheEntry3 = mock(MemcachedCacheEntry.class);
    mockMemcachedCacheEntry4 = mock(MemcachedCacheEntry.class);
    final CacheConfig config = CacheConfig.custom().setMaxUpdateRetries(1).build();
    impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config,
            mockMemcachedCacheEntryFactory, mockKeyHashingScheme);
}

@Override
@Before
public void setUp() {
    super.setUp();
    config = CacheConfig.custom().setMaxObjectSize(MAX_BYTES).build();

    if (CACHE_MANAGER.cacheExists(TEST_EHCACHE_NAME)){
        CACHE_MANAGER.removeCache(TEST_EHCACHE_NAME);
    }
    CACHE_MANAGER.addCache(TEST_EHCACHE_NAME);
    final HttpCacheStorage storage = new EhcacheHttpCacheStorage(CACHE_MANAGER.getCache(TEST_EHCACHE_NAME));
    mockBackend = EasyMock.createNiceMock(ClientExecChain.class);

    impl = new CachingExec(mockBackend, new HeapResourceFactory(), storage, config);
}

@Override
public void setUp() {
    mockCache = mock(Ehcache.class);
    final CacheConfig config = CacheConfig.custom().setMaxUpdateRetries(1).build();
    mockSerializer = mock(HttpCacheEntrySerializer.class);
    impl = new EhcacheHttpCacheStorage(mockCache, config, mockSerializer);
}

private HttpCacheStorage createStorage()
{
    CacheConfig config = new CacheConfig();
    // if max cache entries value is not present the CacheConfig's default (CacheConfig.DEFAULT_MAX_CACHE_ENTRIES = 1000) will be used
    Integer maxCacheEntries = Integer.getInteger("bitbucket.client.cache.maxentries");
    if (maxCacheEntries != null)
    {
        config.setMaxCacheEntries(maxCacheEntries);
    }
    return new BasicHttpCacheStorage(config);
}

private static String tex2json(String tex) throws IOException {
    CachingHttpClientBuilder cachingHttpClientBuilder = CachingHttpClientBuilder.create();
    CacheConfig cacheCfg = new CacheConfig();
    cacheCfg.setMaxCacheEntries(100000);
    cacheCfg.setMaxObjectSize(8192);
    cachingHttpClientBuilder.setCacheConfig(cacheCfg);
    HttpClient client = cachingHttpClientBuilder.build();
    //HttpPost post = new HttpPost("http://localhost/convert");
    HttpPost post = new HttpPost("https://drmf-latexml.wmflabs.org");
    List<NameValuePair> nameValuePairs = new ArrayList<>(1);
    nameValuePairs.add(new BasicNameValuePair("tex",
        "$" + tex + "$"));
    //WARNING: This does not produce pmml, since there is a xstl trasformation that rewrites the output and removes pmml
//        nameValuePairs.add(new BasicNameValuePair("profile",
//                "mwsquery"));
    nameValuePairs.add(new BasicNameValuePair("preload",
        "mws.sty"));
    nameValuePairs.add(new BasicNameValuePair("profile",
        "math"));
    nameValuePairs.add(new BasicNameValuePair("noplane1",
        ""));
    nameValuePairs.add(new BasicNameValuePair("whatsout",
        "math"));
    post.setEntity(new UrlEncodedFormEntity(nameValuePairs, "utf-8"));
    HttpResponse response = client.execute(post);
    BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
    String line = "";
    String result = "";
    while ((line = rd.readLine()) != null) {
      result += line;
    }
    return result;
  }

@SuppressWarnings("restriction")
private static CloseableHttpClient getHttpClient(URI url){
    CacheConfig cacheConfig = CacheConfig.custom()
        .setMaxCacheEntries(1000)
        .setMaxObjectSize(120*1024).setHeuristicCachingEnabled(true)
        .setHeuristicDefaultLifetime(TimeUnit.HOURS.toSeconds(12))
        .build();

    CachingHttpClientBuilder builder = CachingHttpClients.custom()
            .setCacheConfig(cacheConfig)
            .setHttpCacheStorage(new BundleHttpCacheStorage(HybridCore.getContext().getBundle()));

    builder = setupProxy(builder, url);
    return builder.build();
}

/**
 * Builds the HTTP client to connect to the server.
 * 
 * @param trustSelfSigned <tt>true</tt> if the client should accept
 *            self-signed certificates
 * @return a new client instance
 */
private CloseableHttpClient buildClient(boolean trustSelfSigned) {
    try {
        // if required, define custom SSL context allowing self-signed certs
        SSLContext sslContext = !trustSelfSigned ? SSLContexts.createSystemDefault()
            : SSLContexts.custom().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build();

        // set timeouts for the HTTP client
        int globalTimeout = readFromProperty("bdTimeout", 100000);
        int connectTimeout = readFromProperty("bdConnectTimeout", globalTimeout);
        int connectionRequestTimeout = readFromProperty("bdConnectionRequestTimeout", globalTimeout);
        int socketTimeout = readFromProperty("bdSocketTimeout", globalTimeout);
        RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT).setConnectTimeout(connectTimeout)
            .setSocketTimeout(socketTimeout).setConnectionRequestTimeout(connectionRequestTimeout).build();

        // configure caching
        CacheConfig cacheConfig = CacheConfig.copy(CacheConfig.DEFAULT).setSharedCache(false).setMaxCacheEntries(1000)
            .setMaxObjectSize(2 * 1024 * 1024).build();

        // configure connection pooling
        PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(RegistryBuilder
            .<ConnectionSocketFactory> create().register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", new SSLConnectionSocketFactory(sslContext)).build());
        int connectionLimit = readFromProperty("bdMaxConnections", 40);
        // there's only one server to connect to, so max per route matters
        connManager.setMaxTotal(connectionLimit);
        connManager.setDefaultMaxPerRoute(connectionLimit);

        // create the HTTP client
        return CachingHttpClientBuilder.create().setCacheConfig(cacheConfig).setDefaultRequestConfig(requestConfig)
            .setConnectionManager(connManager).build();
    } catch (GeneralSecurityException e) {
        throw new InternalConfigurationException("Failed to set up SSL context", e);
    }
}

/**
 * Builds the HTTP client to connect to the server.
 * 
 * @param trustSelfSigned <tt>true</tt> if the client should accept
 *            self-signed certificates
 * @return a new client instance
 */
private CloseableHttpClient buildClient(boolean trustSelfSigned) {
    try {
        // if required, define custom SSL context allowing self-signed certs
        SSLContext sslContext = !trustSelfSigned ? SSLContexts.createSystemDefault()
            : SSLContexts.custom().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build();

        // set timeouts for the HTTP client
        int globalTimeout = readFromProperty("bdTimeout", 100000);
        int connectTimeout = readFromProperty("bdConnectTimeout", globalTimeout);
        int connectionRequestTimeout = readFromProperty("bdConnectionRequestTimeout", globalTimeout);
        int socketTimeout = readFromProperty("bdSocketTimeout", globalTimeout);
        RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT).setConnectTimeout(connectTimeout)
            .setSocketTimeout(socketTimeout).setConnectionRequestTimeout(connectionRequestTimeout).build();

        // configure caching
        CacheConfig cacheConfig = CacheConfig.copy(CacheConfig.DEFAULT).setSharedCache(false).setMaxCacheEntries(1000)
            .setMaxObjectSize(2 * 1024 * 1024).build();

        // configure connection pooling
        PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(RegistryBuilder
            .<ConnectionSocketFactory> create().register("http", PlainConnectionSocketFactory.getSocketFactory())
            .register("https", new SSLConnectionSocketFactory(sslContext)).build());
        int connectionLimit = readFromProperty("bdMaxConnections", 40);
        // there's only one server to connect to, so max per route matters
        connManager.setMaxTotal(connectionLimit);
        connManager.setDefaultMaxPerRoute(connectionLimit);

        // create the HTTP client
        return CachingHttpClientBuilder.create().setCacheConfig(cacheConfig).setDefaultRequestConfig(requestConfig)
            .setConnectionManager(connManager).build();
    } catch (GeneralSecurityException e) {
        throw new InternalConfigurationException("Failed to set up SSL context", e);
    }
}

/**
 * Creates a HTTP client.
 * 
 * @param parent
 *            Owning object. Used for logging.
 * @param maxCacheEntries
 *            the maximum number of cache entries the cache will retain. Set to 0 for no caching.
 */
public SimpleHttpClient(Object parent, int maxCacheEntries) {
    logger = Logger.getLogger(this.getClass().getName() + "-" + instanceCounter.incrementAndGet() + " ("
            + parent.getClass().getSimpleName() + ")");

    isShutdown = new AtomicBoolean();

    PoolingClientConnectionManager conman = new PoolingClientConnectionManager();
    // increase max number of connections per host and total, defaults are too low to take advantage of multiple
    // threads
    conman.setDefaultMaxPerRoute(50); // default is 2
    conman.setMaxTotal(100); // default is 20
    defaultClient = new DefaultHttpClient(conman);

    if (maxCacheEntries > 0) {
        CacheConfig cacheConfig = new CacheConfig();
        cacheConfig.setSharedCache(false);
        cacheConfig.setMaxCacheEntries(maxCacheEntries);
        cachingClient = new CachingHttpClient(defaultClient, cacheConfig);
        theClient = cachingClient;
    } else {
        cachingClient = null;
        theClient = defaultClient;
    }

    logger.info("Created");
}

@Bean
public CloseableHttpClient httpClient() {
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(properties().getReadTimeout())
        .setSocketTimeout(properties().getSocketTimeout())
        .build();

    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(properties().getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(properties().getMaxConnectionsPerRoute());

    CacheConfig cacheConfig = CacheConfig.custom()
        .setMaxCacheEntries(properties().getMaxCacheEntries())
        .setMaxObjectSize(properties().getMaxObjectSize())
        .setHeuristicCachingEnabled(true) // important!
        .build();

    File cacheDir = new File(properties().getCacheDir());
    try {
        Files.createDirectories(cacheDir.toPath());
    } catch (IOException e) {
        log.warn("Could not create cache directory - using temp folder", e);
        try {
            cacheDir = Files.createTempDirectory("cache").toFile();
        } catch (IOException ee) {
            log.warn("Could not create temp cache directory", ee);
        }
    }

    FileResourceFactory resourceFactory = new FileResourceFactory(cacheDir);
    cacheStorage = new ManagedHttpCacheStorage(cacheConfig);

    client = CachingHttpClients.custom()
        .setCacheConfig(cacheConfig)
        .setResourceFactory(resourceFactory)
        .setHttpCacheStorage(cacheStorage)
        .setDefaultRequestConfig(requestConfig)
        .setConnectionManager(connectionManager)
        .setUserAgent(Constants.USER_AGENT)
        .build();

    return client;
}

public FileCacheStorage(final CacheConfig config, File cacheDir) {
   this.cacheDir = cacheDir;
}

@Bean
public CloseableHttpClient httpClient() {
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(properties().getReadTimeout())
        .setSocketTimeout(properties().getSocketTimeout())
        .build();

    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(properties().getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(properties().getMaxConnectionsPerRoute());

    CacheConfig cacheConfig = CacheConfig.custom()
        .setMaxCacheEntries(properties().getMaxCacheEntries())
        .setMaxObjectSize(properties().getMaxObjectSize())
        .setHeuristicCachingEnabled(true) // important!
        .build();

    File cacheDir = new File(properties().getCacheDir());
    try {
        Files.createDirectories(cacheDir.toPath());
    } catch (IOException e) {
        log.warn("Could not create cache directory - using temp folder", e);
        try {
            cacheDir = Files.createTempDirectory("cache").toFile();
        } catch (IOException ee) {
            log.warn("Could not create temp cache directory", ee);
        }
    }

    FileResourceFactory resourceFactory = new FileResourceFactory(cacheDir);
    cacheStorage = new ManagedHttpCacheStorage(cacheConfig);

    client = CachingHttpClients.custom()
        .setCacheConfig(cacheConfig)
        .setResourceFactory(resourceFactory)
        .setHttpCacheStorage(cacheStorage)
        .setDefaultRequestConfig(requestConfig)
        .setConnectionManager(connectionManager)
        .setUserAgent(Constants.USER_AGENT)
        .build();

    return client;
}

@Test
public void testCacheUpdateThrowsExceptionsIfCASFailsEnoughTimes() throws IOException {
    final String url = "foo";
    final String key = "key";
    final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
    final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
    final byte[] oldBytes = HttpTestUtils.getRandomBytes(128);
    final CASValue<Object> casValue = new CASValue<Object>(1, oldBytes);
    final byte[] newBytes = HttpTestUtils.getRandomBytes(128);

    final CacheConfig config = CacheConfig.custom().setMaxUpdateRetries(0).build();
    impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config,
            mockMemcachedCacheEntryFactory, mockKeyHashingScheme);

    final HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
        @Override
        public HttpCacheEntry update(final HttpCacheEntry old) {
            assertSame(existingValue, old);
            return updatedValue;
        }
    };

    // get empty old entry
    when(mockKeyHashingScheme.hash(url)).thenReturn(key);
    when(mockMemcachedClient.gets(key)).thenReturn(casValue);
    when(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
        .thenReturn(mockMemcachedCacheEntry);
    when(mockMemcachedCacheEntry.getStorageKey()).thenReturn(url);
    when(mockMemcachedCacheEntry.getHttpCacheEntry()).thenReturn(existingValue);

    when(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
        .thenReturn(mockMemcachedCacheEntry2);
    when(mockMemcachedCacheEntry2.toByteArray()).thenReturn(newBytes);

    when(
            mockMemcachedClient.cas(key, casValue.getCas(),
                    newBytes)).thenReturn(CASResponse.EXISTS);

    try {
        impl.updateEntry(url, callback);
        fail("should have thrown exception");
    } catch (final HttpCacheUpdateException expected) {
    }

    verify(mockKeyHashingScheme).hash(url);
    verify(mockMemcachedClient).gets(key);
    verify(mockMemcachedCacheEntryFactory).getUnsetCacheEntry();
    verify(mockMemcachedCacheEntry).getStorageKey();
    verify(mockMemcachedCacheEntry).getHttpCacheEntry();
    verify(mockMemcachedCacheEntryFactory).getMemcachedCacheEntry(url, updatedValue);
    verify(mockMemcachedCacheEntry2).toByteArray();
    verify(mockMemcachedClient).cas(key, casValue.getCas(), newBytes);
}

public EtagCachingHttpClient()
{
    this(new DefaultHttpClient(), new BasicHttpCacheStorage(new CacheConfig()));
}

public EtagCachingHttpClient(HttpClient httpClient)
{
    this(httpClient, new BasicHttpCacheStorage(new CacheConfig()));
}

@SuppressWarnings("resource")
private static YamjHttpClient buildHttpClient() {
    LOG.trace("Create new YAMJ http client");

    // create proxy
    HttpHost proxy = null;
    CredentialsProvider credentialsProvider = null;

    if (StringUtils.isNotBlank(PROXY_HOST) && PROXY_PORT > 0) {
        proxy = new HttpHost(PROXY_HOST, PROXY_PORT);

        if (StringUtils.isNotBlank(PROXY_USERNAME) && StringUtils.isNotBlank(PROXY_PASSWORD)) {
            credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(
                    new AuthScope(PROXY_HOST, PROXY_PORT),
                    new UsernamePasswordCredentials(PROXY_USERNAME, PROXY_PASSWORD));
        }
    }


    PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
    connManager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(TIMEOUT_SOCKET).build());
    connManager.setMaxTotal(20);
    connManager.setDefaultMaxPerRoute(2);

    CacheConfig cacheConfig = CacheConfig.custom()
                    .setMaxCacheEntries(1000)
                    .setMaxObjectSize(8192)
                    .build();

    HttpClientBuilder builder = CachingHttpClientBuilder.create()
            .setCacheConfig(cacheConfig)
            .setConnectionManager(connManager)
            .setProxy(proxy)
            .setDefaultCredentialsProvider(credentialsProvider)
            .setDefaultRequestConfig(RequestConfig.custom()
                    .setConnectionRequestTimeout(TIMEOUT_READ)
                    .setConnectTimeout(TIMEOUT_CONNECT)
                    .setSocketTimeout(TIMEOUT_SOCKET)
                    .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
                    .setProxy(proxy)
                    .build());

    // show status
    showStatus();

    // build the client
    YamjHttpClient wrapper = new YamjHttpClient(builder.build(), connManager);
    wrapper.setUserAgentSelector(new WebBrowserUserAgentSelector());
    wrapper.addGroupLimit(".*", 1); // default limit, can be overwritten

    // First we have to read/create the rules
    String maxDownloadSlots = PropertiesUtil.getProperty("mjb.MaxDownloadSlots");
    if (StringUtils.isNotBlank(maxDownloadSlots)) {
        LOG.debug("Using download limits: {}", maxDownloadSlots);

        Pattern pattern = Pattern.compile(",?\\s*([^=]+)=(\\d+)");
        Matcher matcher = pattern.matcher(maxDownloadSlots);
        while (matcher.find()) {
            String group = matcher.group(1);
            try {
                final Integer maxResults = Integer.valueOf(matcher.group(2));
                wrapper.addGroupLimit(group, maxResults);
                LOG.trace("Added download slot '{}' with max results {}", group, maxResults);
            } catch (NumberFormatException error) {
                LOG.debug("Rule '{}' is no valid regexp, ignored", group);
            }
        }
    }

    return wrapper;
}

/**
 * Construct a new SpotifyHttpManager instance.
 *
 * @param builder The builder.
 */
public SpotifyHttpManager(Builder builder) {
  this.proxy = builder.proxy;
  this.proxyCredentials = builder.proxyCredentials;
  this.cacheMaxEntries = builder.cacheMaxEntries;
  this.cacheMaxObjectSize = builder.cacheMaxObjectSize;


  CacheConfig cacheConfig = CacheConfig.custom()
          .setMaxCacheEntries(cacheMaxEntries != null ? cacheMaxEntries : DEFAULT_CACHE_MAX_ENTRIES)
          .setMaxObjectSize(cacheMaxEntries != null ? cacheMaxEntries : DEFAULT_CACHE_MAX_OBJECT_SIZE)
          .setSharedCache(false)
          .build();

  ConnectionConfig connectionConfig = ConnectionConfig
          .custom()
          .setCharset(Charset.forName("UTF-8"))
          .build();

  new BasicCredentialsProvider();
  CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
  if (proxy != null) {
    credentialsProvider.setCredentials(
            new AuthScope(proxy.getHostName(), proxy.getPort(), null, proxy.getSchemeName()),
            proxyCredentials
    );
  }

  RequestConfig requestConfig = RequestConfig
          .custom()
          .setCookieSpec(CookieSpecs.DEFAULT)
          .setProxy(proxy)
          .build();


  httpClient = CachingHttpClients
          .custom()
          .setCacheConfig(cacheConfig)
          .setDefaultConnectionConfig(connectionConfig)
          .setDefaultCredentialsProvider(credentialsProvider)
          .setDefaultRequestConfig(requestConfig)
          .build();
}

@PostConstruct
protected void initialize() {
    try {
        lock.writeLock().lock();

        httpParams = new BasicHttpParams();
        String userAgentString =
                "Apache Marmotta/" + configurationService.getStringConfiguration("kiwi.version") +
                " (running at " + configurationService.getServerUri() + ")" +
                " lmf-core/" + configurationService.getStringConfiguration("kiwi.version");
        userAgentString = configurationService.getStringConfiguration("core.http.user_agent", userAgentString);

        httpParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, configurationService.getIntConfiguration("core.http.so_timeout", 60000));
        httpParams.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
                configurationService.getIntConfiguration("core.http.connection_timeout", 10000));

        httpParams.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, true);
        httpParams.setIntParameter(ClientPNames.MAX_REDIRECTS, 3);

        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

        PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
        cm.setMaxTotal(configurationService.getIntConfiguration(CoreOptions.HTTP_MAX_CONNECTIONS, 20));
        cm.setDefaultMaxPerRoute(configurationService.getIntConfiguration(CoreOptions.HTTP_MAX_CONNECTIONS_PER_ROUTE, 10));

        final DefaultHttpClient hc = new DefaultHttpClient(cm, httpParams);
        hc.setRedirectStrategy(new LMFRedirectStrategy());
        hc.setHttpRequestRetryHandler(new LMFHttpRequestRetryHandler());
        hc.removeRequestInterceptorByClass(org.apache.http.protocol.RequestUserAgent.class);
        hc.addRequestInterceptor(new LMFRequestUserAgent(userAgentString));

        if (configurationService.getBooleanConfiguration(CoreOptions.HTTP_CLIENT_CACHE_ENABLE, true)) {
            CacheConfig cacheConfig = new CacheConfig();
            // FIXME: Hardcoded constants - is this useful?
            cacheConfig.setMaxCacheEntries(1000);
            cacheConfig.setMaxObjectSize(81920);

            final HttpCacheStorage cacheStore = new MapHttpCacheStorage(httpCache);

            this.httpClient = new MonitoredHttpClient(new CachingHttpClient(hc, cacheStore, cacheConfig));
        } else {
            this.httpClient = new MonitoredHttpClient(hc);
        }
        bytesSent.set(0);
        bytesReceived.set(0);
        requestsExecuted.set(0);

        idleConnectionMonitorThread = new IdleConnectionMonitorThread(httpClient.getConnectionManager());
        idleConnectionMonitorThread.start();

        StatisticsProvider stats = new StatisticsProvider(cm);
        statisticsService.registerModule(HttpClientService.class.getSimpleName(), stats);
    } finally {
        lock.writeLock().unlock();
    }
}

/**
 * Create a storage backend using the given <i>memcached</i> client and
 * applying the given cache configuration and cache entry serialization
 * mechanism. <b>Deprecation note:</b> In the process of fixing a bug
 * based on the need to hash logical storage keys onto memcached cache
 * keys, the serialization process was revamped. This constructor still
 * works, but the serializer argument will be ignored and default
 * implementations of the new framework will be used. You can still
 * provide custom serialization by using the
 * {@link #MemcachedHttpCacheStorage(MemcachedClientIF, CacheConfig,
 * MemcachedCacheEntryFactory, KeyHashingScheme)} constructor.
 * @param client how to talk to <i>memcached</i>
 * @param config apply HTTP cache-related options
 * @param serializer <b>ignored</b>
 *
 * @deprecated (4.2) do not use
 */
@Deprecated
public MemcachedHttpCacheStorage(final MemcachedClientIF client, final CacheConfig config,
        final HttpCacheEntrySerializer serializer) {
    this(client, config, new MemcachedCacheEntryFactoryImpl(),
            new SHA256KeyHashingScheme());
}

/**
 * Create a storage backend using the given <i>memcached</i> client and
 * applying the given cache configuration, serialization, and hashing
 * mechanisms.
 * @param client how to talk to <i>memcached</i>
 * @param config apply HTTP cache-related options
 * @param memcachedCacheEntryFactory Factory pattern used for obtaining
 *   instances of alternative cache entry serialization mechanisms
 * @param keyHashingScheme how to map higher-level logical "storage keys"
 *   onto "cache keys" suitable for use with memcached
 */
public MemcachedHttpCacheStorage(final MemcachedClientIF client, final CacheConfig config,
        final MemcachedCacheEntryFactory memcachedCacheEntryFactory,
        final KeyHashingScheme keyHashingScheme) {
    this.client = client;
    this.maxUpdateRetries = config.getMaxUpdateRetries();
    this.memcachedCacheEntryFactory = memcachedCacheEntryFactory;
    this.keyHashingScheme = keyHashingScheme;
}

public HttpCache() {

        CacheConfig cacheCfg =new CacheConfig();
        cacheCfg.setMaxCacheEntries(1000);
        cacheCfg.setMaxObjectSize(8192);

        cachingClient = new CachingHttpClient(new DefaultHttpClient(), cacheCfg);

    }

/**
 * Create a storage backend using the pre-configured given
 * <i>memcached</i> client.
 * @param cache client to use for communicating with <i>memcached</i>
 */
public MemcachedHttpCacheStorage(final MemcachedClientIF cache) {
    this(cache, CacheConfig.DEFAULT, new MemcachedCacheEntryFactoryImpl(),
            new SHA256KeyHashingScheme());
}

/**
 * Constructs a storage backend using the provided Ehcache
 * with default configuration options.
 * @param cache where to store cached origin responses
 */
public EhcacheHttpCacheStorage(final Ehcache cache) {
    this(cache, CacheConfig.DEFAULT, new DefaultHttpCacheEntrySerializer());
}

/**
 * Constructs a storage backend using the provided Ehcache
 * with the given configuration options.
 * @param cache where to store cached origin responses
 * @param config cache storage configuration options - note that
 *   the setting for max object size <b>will be ignored</b> and
 *   should be configured in the Ehcache instead.
 */
public EhcacheHttpCacheStorage(final Ehcache cache, final CacheConfig config){
    this(cache, config, new DefaultHttpCacheEntrySerializer());
}

/**
 * Constructs a storage backend using the provided Ehcache
 * with the given configuration options, but using an alternative
 * cache entry serialization strategy.
 * @param cache where to store cached origin responses
 * @param config cache storage configuration options - note that
 *   the setting for max object size <b>will be ignored</b> and
 *   should be configured in the Ehcache instead.
 * @param serializer alternative serialization mechanism
 */
public EhcacheHttpCacheStorage(final Ehcache cache, final CacheConfig config, final HttpCacheEntrySerializer serializer){
    this.cache = cache;
    this.maxUpdateRetries = config.getMaxUpdateRetries();
    this.serializer = serializer;
}