public static HttpClientBuilder getHttpClientBuilder() { // Common CacheConfig for both the JarCacheStorage and the underlying // BasicHttpCacheStorage final CacheConfig cacheConfig = CacheConfig.custom().setMaxCacheEntries(1000).setMaxObjectSize(1024 * 128) .build(); RequestConfig config = RequestConfig.custom().setConnectTimeout(DEFAULT_TIMEOUT) .setConnectionRequestTimeout(DEFAULT_TIMEOUT).setSocketTimeout(DEFAULT_TIMEOUT).build(); HttpClientBuilder clientBuilder = CachingHttpClientBuilder.create() // allow caching .setCacheConfig(cacheConfig) // Wrap the local JarCacheStorage around a BasicHttpCacheStorage .setHttpCacheStorage(new JarCacheStorage(null, cacheConfig, new BasicHttpCacheStorage(cacheConfig))) // Support compressed data // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/httpagent.html#d5e1238 .addInterceptorFirst(new RequestAcceptEncoding()).addInterceptorFirst(new ResponseContentEncoding()) // use system defaults for proxy etc. .useSystemProperties().setDefaultRequestConfig(config); return clientBuilder; }
private static CloseableHttpClient newClosableCachingHttpClient(EventStoreSettings settings) { final CacheConfig cacheConfig = CacheConfig.custom() .setMaxCacheEntries(Integer.MAX_VALUE) .setMaxObjectSize(Integer.MAX_VALUE) .build(); settings.getCacheDirectory() .mkdirs(); return CachingHttpClientBuilder.create() .setHttpCacheStorage(new FileCacheStorage(cacheConfig, settings.getCacheDirectory())) .setCacheConfig(cacheConfig) .setDefaultRequestConfig(requestConfig(settings)) .setDefaultCredentialsProvider(credentialsProvider(settings)) .setRedirectStrategy(new LaxRedirectStrategy()) .setRetryHandler(new StandardHttpRequestRetryHandler()) .setKeepAliveStrategy(new de.qyotta.eventstore.utils.DefaultConnectionKeepAliveStrategy()) .setConnectionManagerShared(true) .build(); }
@SuppressWarnings("restriction") private static CachingHttpClientBuilder setupProxy(CachingHttpClientBuilder builder, URI url){ final IProxyService proxyService = HybridCore.getDefault().getProxyService(); if(proxyService != null ){ IProxyData[] proxies = proxyService.select(url); if(proxies != null && proxies.length > 0){ IProxyData proxy = proxies[0]; CredentialsProvider credsProvider = new BasicCredentialsProvider(); if(proxy.isRequiresAuthentication()){ credsProvider.setCredentials(new AuthScope(proxy.getHost(), proxy.getPort()), new UsernamePasswordCredentials(proxy.getUserId(), proxy.getPassword())); } builder.setDefaultCredentialsProvider(credsProvider); builder.setProxy(new HttpHost(proxy.getHost(), proxy.getPort())); } } return builder; }
@Test public void testDescribeRdfCached() throws IOException { try (final CloseableHttpClient cachClient = CachingHttpClientBuilder.create().setCacheConfig(DEFAULT).build()) { final String location = getLocation(postObjMethod()); try (final CloseableHttpResponse response = cachClient.execute(new HttpGet(location))) { assertEquals("Client didn't return a OK!", OK.getStatusCode(), getStatus(response)); logger.debug("Found HTTP headers:\n{}", asList(response.getAllHeaders())); assertTrue("Didn't find Last-Modified header!", response.containsHeader("Last-Modified")); final String lastModed = response.getFirstHeader("Last-Modified").getValue(); final String etag = response.getFirstHeader("ETag").getValue(); final HttpGet getObjMethod2 = new HttpGet(location); getObjMethod2.setHeader("If-Modified-Since", lastModed); getObjMethod2.setHeader("If-None-Match", etag); assertEquals("Client didn't get a NOT_MODIFIED!", NOT_MODIFIED.getStatusCode(), getStatus(getObjMethod2)); } } }
private static String tex2json(String tex) throws IOException { CachingHttpClientBuilder cachingHttpClientBuilder = CachingHttpClientBuilder.create(); CacheConfig cacheCfg = new CacheConfig(); cacheCfg.setMaxCacheEntries(100000); cacheCfg.setMaxObjectSize(8192); cachingHttpClientBuilder.setCacheConfig(cacheCfg); HttpClient client = cachingHttpClientBuilder.build(); //HttpPost post = new HttpPost("http://localhost/convert"); HttpPost post = new HttpPost("https://drmf-latexml.wmflabs.org"); List<NameValuePair> nameValuePairs = new ArrayList<>(1); nameValuePairs.add(new BasicNameValuePair("tex", "$" + tex + "$")); //WARNING: This does not produce pmml, since there is a xstl trasformation that rewrites the output and removes pmml // nameValuePairs.add(new BasicNameValuePair("profile", // "mwsquery")); nameValuePairs.add(new BasicNameValuePair("preload", "mws.sty")); nameValuePairs.add(new BasicNameValuePair("profile", "math")); nameValuePairs.add(new BasicNameValuePair("noplane1", "")); nameValuePairs.add(new BasicNameValuePair("whatsout", "math")); post.setEntity(new UrlEncodedFormEntity(nameValuePairs, "utf-8")); HttpResponse response = client.execute(post); BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); String line = ""; String result = ""; while ((line = rd.readLine()) != null) { result += line; } return result; }
@SuppressWarnings("restriction") private static CloseableHttpClient getHttpClient(URI url){ CacheConfig cacheConfig = CacheConfig.custom() .setMaxCacheEntries(1000) .setMaxObjectSize(120*1024).setHeuristicCachingEnabled(true) .setHeuristicDefaultLifetime(TimeUnit.HOURS.toSeconds(12)) .build(); CachingHttpClientBuilder builder = CachingHttpClients.custom() .setCacheConfig(cacheConfig) .setHttpCacheStorage(new BundleHttpCacheStorage(HybridCore.getContext().getBundle())); builder = setupProxy(builder, url); return builder.build(); }
/** * Builds the HTTP client to connect to the server. * * @param trustSelfSigned <tt>true</tt> if the client should accept * self-signed certificates * @return a new client instance */ private CloseableHttpClient buildClient(boolean trustSelfSigned) { try { // if required, define custom SSL context allowing self-signed certs SSLContext sslContext = !trustSelfSigned ? SSLContexts.createSystemDefault() : SSLContexts.custom().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build(); // set timeouts for the HTTP client int globalTimeout = readFromProperty("bdTimeout", 100000); int connectTimeout = readFromProperty("bdConnectTimeout", globalTimeout); int connectionRequestTimeout = readFromProperty("bdConnectionRequestTimeout", globalTimeout); int socketTimeout = readFromProperty("bdSocketTimeout", globalTimeout); RequestConfig requestConfig = RequestConfig.copy(RequestConfig.DEFAULT).setConnectTimeout(connectTimeout) .setSocketTimeout(socketTimeout).setConnectionRequestTimeout(connectionRequestTimeout).build(); // configure caching CacheConfig cacheConfig = CacheConfig.copy(CacheConfig.DEFAULT).setSharedCache(false).setMaxCacheEntries(1000) .setMaxObjectSize(2 * 1024 * 1024).build(); // configure connection pooling PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(RegistryBuilder .<ConnectionSocketFactory> create().register("http", PlainConnectionSocketFactory.getSocketFactory()) .register("https", new SSLConnectionSocketFactory(sslContext)).build()); int connectionLimit = readFromProperty("bdMaxConnections", 40); // there's only one server to connect to, so max per route matters connManager.setMaxTotal(connectionLimit); connManager.setDefaultMaxPerRoute(connectionLimit); // create the HTTP client return CachingHttpClientBuilder.create().setCacheConfig(cacheConfig).setDefaultRequestConfig(requestConfig) .setConnectionManager(connManager).build(); } catch (GeneralSecurityException e) { throw new InternalConfigurationException("Failed to set up SSL context", e); } }
@SuppressWarnings("resource") private static YamjHttpClient buildHttpClient() { LOG.trace("Create new YAMJ http client"); // create proxy HttpHost proxy = null; CredentialsProvider credentialsProvider = null; if (StringUtils.isNotBlank(PROXY_HOST) && PROXY_PORT > 0) { proxy = new HttpHost(PROXY_HOST, PROXY_PORT); if (StringUtils.isNotBlank(PROXY_USERNAME) && StringUtils.isNotBlank(PROXY_PASSWORD)) { credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials( new AuthScope(PROXY_HOST, PROXY_PORT), new UsernamePasswordCredentials(PROXY_USERNAME, PROXY_PASSWORD)); } } PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); connManager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(TIMEOUT_SOCKET).build()); connManager.setMaxTotal(20); connManager.setDefaultMaxPerRoute(2); CacheConfig cacheConfig = CacheConfig.custom() .setMaxCacheEntries(1000) .setMaxObjectSize(8192) .build(); HttpClientBuilder builder = CachingHttpClientBuilder.create() .setCacheConfig(cacheConfig) .setConnectionManager(connManager) .setProxy(proxy) .setDefaultCredentialsProvider(credentialsProvider) .setDefaultRequestConfig(RequestConfig.custom() .setConnectionRequestTimeout(TIMEOUT_READ) .setConnectTimeout(TIMEOUT_CONNECT) .setSocketTimeout(TIMEOUT_SOCKET) .setCookieSpec(CookieSpecs.IGNORE_COOKIES) .setProxy(proxy) .build()); // show status showStatus(); // build the client YamjHttpClient wrapper = new YamjHttpClient(builder.build(), connManager); wrapper.setUserAgentSelector(new WebBrowserUserAgentSelector()); wrapper.addGroupLimit(".*", 1); // default limit, can be overwritten // First we have to read/create the rules String maxDownloadSlots = PropertiesUtil.getProperty("mjb.MaxDownloadSlots"); if (StringUtils.isNotBlank(maxDownloadSlots)) { LOG.debug("Using download limits: {}", maxDownloadSlots); Pattern pattern = Pattern.compile(",?\\s*([^=]+)=(\\d+)"); Matcher matcher = pattern.matcher(maxDownloadSlots); while (matcher.find()) { String group = matcher.group(1); try { final Integer maxResults = Integer.valueOf(matcher.group(2)); wrapper.addGroupLimit(group, maxResults); LOG.trace("Added download slot '{}' with max results {}", group, maxResults); } catch (NumberFormatException error) { LOG.debug("Rule '{}' is no valid regexp, ignored", group); } } } return wrapper; }
