@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); ThriftHBaseServiceHandler hbaseHandler = createHandler(); THBaseService.Iface handler = ThriftHBaseServiceHandler.newInstance(hbaseHandler, metrics); byte[] rowName = "testMetrics".getBytes(); ByteBuffer table = wrap(tableAname); TGet get = new TGet(wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<TColumnValue>(); columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); TPut put = new TPut(wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource()); metricsHelper.assertCounter( "exists_num_ops", 2, metrics.getSource()); }
@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); THBaseService.Iface handler = ThriftHBaseServiceHandler.newInstance(conf, metrics); byte[] rowName = "testMetrics".getBytes(); ByteBuffer table = wrap(tableAname); TGet get = new TGet(wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<TColumnValue>(); columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); TPut put = new TPut(wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); logMetrics(metrics); verifyMetrics(metrics, "put_num_ops", 1); verifyMetrics(metrics, "exists_num_ops", 2); }
private static void logMetrics(ThriftMetrics metrics) throws Exception { if (LOG.isDebugEnabled()) { return; } MetricsContext context = MetricsUtil.getContext( ThriftMetrics.CONTEXT_NAME); metrics.doUpdates(context); for (String key : context.getAllRecords().keySet()) { for (OutputRecord record : context.getAllRecords().get(key)) { for (String name : record.getMetricNames()) { LOG.debug("metrics:" + name + " value:" + record.getMetric(name).intValue()); } } } }
@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); THBaseService.Iface handler = ThriftHBaseServiceHandler.newInstance(conf, metrics); byte[] rowName = "testMetrics".getBytes(); ByteBuffer table = wrap(tableAname); TGet get = new TGet(wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<TColumnValue>(); columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); TPut put = new TPut(wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource()); metricsHelper.assertCounter( "exists_num_ops", 2, metrics.getSource()); }
private static TServer getTHsHaServer(TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, int workerThreads, int maxCallQueueSize, InetSocketAddress inetSocketAddress, ThriftMetrics metrics) throws TTransportException { TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString()); THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport); if (workerThreads > 0) { // Could support the min & max threads, avoiding to preserve existing functionality. serverArgs.minWorkerThreads(workerThreads).maxWorkerThreads(workerThreads); } ExecutorService executorService = createExecutor( workerThreads, maxCallQueueSize, metrics); serverArgs.executorService(executorService); serverArgs.processor(processor); serverArgs.transportFactory(transportFactory); serverArgs.protocolFactory(protocolFactory); return new THsHaServer(serverArgs); }
private static TServer getTThreadedSelectorServer(TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, int workerThreads, int selectorThreads, int maxCallQueueSize, InetSocketAddress inetSocketAddress, ThriftMetrics metrics) throws TTransportException { TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); log.info("starting HBase ThreadedSelector Thrift server on " + inetSocketAddress.toString()); TThreadedSelectorServer.Args serverArgs = new TThreadedSelectorServer.Args(serverTransport); if (workerThreads > 0) { serverArgs.workerThreads(workerThreads); } if (selectorThreads > 0) { serverArgs.selectorThreads(selectorThreads); } ExecutorService executorService = createExecutor( workerThreads, maxCallQueueSize, metrics); serverArgs.executorService(executorService); serverArgs.processor(processor); serverArgs.transportFactory(transportFactory); serverArgs.protocolFactory(protocolFactory); return new TThreadedSelectorServer(serverArgs); }
private TServer getServer(int workerThreads, int selectorThreads, int maxCallQueueSize, int readTimeout, int backlog, boolean nonblocking, boolean hsha, boolean selector, ThriftMetrics metrics, TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException { TServer server; if (nonblocking) { server = getTNonBlockingServer(protocolFactory, processor, transportFactory, inetSocketAddress); } else if (hsha) { server = getTHsHaServer(protocolFactory, processor, transportFactory, workerThreads, maxCallQueueSize, inetSocketAddress, metrics); } else if (selector) { server = getTThreadedSelectorServer(protocolFactory, processor, transportFactory, workerThreads, selectorThreads, maxCallQueueSize, inetSocketAddress, metrics); } else { server = getTThreadPoolServer(protocolFactory, processor, transportFactory, workerThreads, inetSocketAddress, backlog, readTimeout, metrics); } return server; }
@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); ThriftHBaseServiceHandler hbaseHandler = createHandler(); THBaseService.Iface handler = ThriftHBaseServiceHandler.newInstance(hbaseHandler, metrics); byte[] rowName = Bytes.toBytes("testMetrics"); ByteBuffer table = wrap(tableAname); TGet get = new TGet(wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<>(2); columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); TPut put = new TPut(wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource()); metricsHelper.assertCounter("exists_num_ops", 2, metrics.getSource()); }
private void testExceptionType(THBaseService.Iface handler, ThriftMetrics metrics, ByteBuffer tTableName, byte[] rowkey, ErrorThrowingGetObserver.ErrorType errorType) { long preGetCounter = metricsHelper.getCounter("get_num_ops", metrics.getSource()); String exceptionKey = errorType.getMetricName(); long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ? metricsHelper.getCounter(exceptionKey, metrics.getSource()) : 0; TGet tGet = new TGet(wrap(rowkey)); Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>(); attributes.put(wrap(Bytes.toBytes(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE)), wrap(Bytes.toBytes(errorType.name()))); tGet.setAttributes(attributes); try { TResult tResult = handler.get(tTableName, tGet); fail("Get with error attribute should have thrown an exception"); } catch (TException e) { LOG.info("Received exception: ", e); metricsHelper.assertCounter("get_num_ops", preGetCounter + 1, metrics.getSource()); metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource()); } }
private static TServer getTHsHaServer(TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, int workerThreads, InetSocketAddress inetSocketAddress, ThriftMetrics metrics) throws TTransportException { TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString()); THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport); if (workerThreads > 0) { // Could support the min & max threads, avoiding to preserve existing functionality. serverArgs.minWorkerThreads(workerThreads).maxWorkerThreads(workerThreads); } ExecutorService executorService = createExecutor( workerThreads, metrics); serverArgs.executorService(executorService); serverArgs.processor(processor); serverArgs.transportFactory(transportFactory); serverArgs.protocolFactory(protocolFactory); return new THsHaServer(serverArgs); }
private static ExecutorService createExecutor( int workerThreads, ThriftMetrics metrics) { CallQueue callQueue = new CallQueue( new LinkedBlockingQueue<Call>(), metrics); ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); tfb.setDaemon(true); tfb.setNameFormat("thrift2-worker-%d"); ThreadPoolExecutor pool = new ThreadPoolExecutor(workerThreads, workerThreads, Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build()); pool.prestartAllCoreThreads(); return pool; }
private static TServer getTHsHaServer(TProtocolFactory protocolFactory, THBaseService.Processor processor, TTransportFactory transportFactory, InetSocketAddress inetSocketAddress, ThriftMetrics metrics) throws TTransportException { TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress); log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString()); THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport); ExecutorService executorService = createExecutor( serverArgs.getWorkerThreads(), metrics); serverArgs.executorService(executorService); serverArgs.processor(processor); serverArgs.transportFactory(transportFactory); serverArgs.protocolFactory(protocolFactory); return new THsHaServer(serverArgs); }
private static ExecutorService createExecutor( int workerThreads, ThriftMetrics metrics) { CallQueue callQueue = new CallQueue( new LinkedBlockingQueue<Call>(), metrics); ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); tfb.setDaemon(true); tfb.setNameFormat("thrift2-worker-%d"); return new ThreadPoolExecutor(workerThreads, workerThreads, Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build()); }
private static void setupMetricsContext() throws IOException { ContextFactory factory = ContextFactory.getFactory(); factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class", NoEmitMetricsContext.class.getName()); MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME) .createRecord(ThriftMetrics.CONTEXT_NAME).remove(); }
private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue) throws Exception { MetricsContext context = MetricsUtil.getContext( ThriftMetrics.CONTEXT_NAME); metrics.doUpdates(context); OutputRecord record = context.getAllRecords().get( ThriftMetrics.CONTEXT_NAME).iterator().next(); assertEquals(expectValue, record.getMetric(name).intValue()); }
private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory, TProcessor processor, TTransportFactory transportFactory, int workerThreads, InetSocketAddress inetSocketAddress, int backlog, int clientTimeout, ThriftMetrics metrics) throws TTransportException { TServerTransport serverTransport = new TServerSocket( new TServerSocket.ServerSocketTransportArgs(). bindAddr(inetSocketAddress).backlog(backlog). clientTimeout(clientTimeout)); log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString()); TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport); serverArgs.processor(processor); serverArgs.transportFactory(transportFactory); serverArgs.protocolFactory(protocolFactory); if (workerThreads > 0) { serverArgs.maxWorkerThreads(workerThreads); } ThreadPoolExecutor executor = new THBaseThreadPoolExecutor(serverArgs.minWorkerThreads, serverArgs.maxWorkerThreads, serverArgs.stopTimeoutVal, TimeUnit.SECONDS, new SynchronousQueue<>(), metrics); serverArgs.executorService(executor); return new TThreadPoolServer(serverArgs); }
public static THBaseService.Iface newInstance( Configuration conf, ThriftMetrics metrics) { THBaseService.Iface handler = new ThriftHBaseServiceHandler(conf); return (THBaseService.Iface) Proxy.newProxyInstance( handler.getClass().getClassLoader(), new Class[]{THBaseService.Iface.class}, new THBaseServiceMetricsProxy(handler, metrics)); }
@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); THBaseService.Iface handler = ThriftHBaseServiceHandler.newInstance(conf, metrics); byte[] rowName = "testMetrics".getBytes(); ByteBuffer table = ByteBuffer.wrap(tableAname); TGet get = new TGet(ByteBuffer.wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<TColumnValue>(); columnValues.add(new TColumnValue(ByteBuffer.wrap(familyAname), ByteBuffer.wrap(qualifierAname), ByteBuffer.wrap(valueAname))); columnValues.add(new TColumnValue(ByteBuffer.wrap(familyBname), ByteBuffer.wrap(qualifierBname), ByteBuffer.wrap(valueBname))); TPut put = new TPut(ByteBuffer.wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); logMetrics(metrics); verifyMetrics(metrics, "put_num_ops", 1); verifyMetrics(metrics, "exists_num_ops", 2); }