public static Map<String, Long> getScanMetrics(ScanResponse response) { Map<String, Long> metricMap = new HashMap<String, Long>(); if (response == null || !response.hasScanMetrics() || response.getScanMetrics() == null) { return metricMap; } ScanMetrics metrics = response.getScanMetrics(); int numberOfMetrics = metrics.getMetricsCount(); for (int i = 0; i < numberOfMetrics; i++) { NameInt64Pair metricPair = metrics.getMetrics(i); if (metricPair != null) { String name = metricPair.getName(); Long value = metricPair.getValue(); if (name != null && value != null) { metricMap.put(name, value); } } } return metricMap; }
protected long openScanner() throws IOException { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest( getLocation().getRegionInfo().getRegionName(), this.scan, 0, false); try { ScanResponse response = getStub().scan(null, request); long id = response.getScannerId(); if (logScannerActivity) { LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + " on region " + getLocation().toString()); } return id; } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
ScanOpenNextThenExceptionThenRecoverConnection(Configuration conf, boolean managed, ExecutorService pool) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
RegionServerStoppedOnScannerOpenConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
@Override public Result[] call(int timeout) throws IOException { if (this.closed) return null; if (Thread.interrupted()) { throw new InterruptedIOException(); } ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), getScan(), getCaching(), true); ScanResponse response = null; PayloadCarryingRpcController controller = controllerFactory.newController(); try { controller.setPriority(getTableName()); controller.setCallTimeout(timeout); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
static RegionServerCallable<Result[]> getSmallScanCallable( final Scan sc, HConnection connection, TableName table, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) throws IOException { sc.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( connection, table, sc.getStartRow()) { public Result[] call() throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), sc, cacheNum, true); ScanResponse response = null; PayloadCarryingRpcController controller = rpcControllerFactory.newController(); try { controller.setPriority(getTableName()); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
static RegionServerCallable<Result[]> getSmallScanCallable( final Scan sc, HConnection connection, TableName table, byte[] localStartKey, final int cacheNum, final RpcControllerFactory rpcControllerFactory) { sc.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( connection, table, sc.getStartRow()) { public Result[] call(int callTimeout) throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), sc, cacheNum, true); PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(getTableName()); controller.setCallTimeout(callTimeout); try { ScanResponse response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
private RegionServerCallable<Result[]> getSmallScanCallable( byte[] localStartKey, final int cacheNum) { this.scan.setStartRow(localStartKey); RegionServerCallable<Result[]> callable = new RegionServerCallable<Result[]>( getConnection(), getTable(), scan.getStartRow()) { public Result[] call() throws IOException { ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), scan, cacheNum, true); ScanResponse response = null; PayloadCarryingRpcController controller = new PayloadCarryingRpcController(); try { controller.setPriority(getTableName()); response = getStub().scan(controller, request); return ResponseConverter.getResults(controller.cellScanner(), response); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return callable; }
protected long openScanner() throws IOException { incRPCcallsMetrics(); ScanRequest request = RequestConverter.buildScanRequest( this.location.getRegionInfo().getRegionName(), this.scan, 0, false); try { ScanResponse response = server.scan(null, request); long id = response.getScannerId(); if (logScannerActivity) { LOG.info("Open scanner=" + id + " for scan=" + scan.toString() + " on region " + this.location.toString() + " ip:" + this.location.getHostnamePort()); } return id; } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
@Override public ScanResponse scan(final RpcController controller, final ScanRequest request) throws ServiceException { if (request.hasScannerId()) { ScanResponse scanResponse = super.scan(controller, request); if (this.tableScannerId == request.getScannerId() && (sleepAlways || (!slept && seqNoToSleepOn == request.getNextCallSeq()))) { try { LOG.info("SLEEPING " + (rpcTimeout + 500)); Thread.sleep(rpcTimeout + 500); } catch (InterruptedException e) { } slept = true; tryNumber++; if (tryNumber > 2 * CLIENT_RETRIES_NUMBER) { sleepAlways = false; } } return scanResponse; } else { ScanResponse scanRes = super.scan(controller, request); String regionName = Bytes.toString(request.getRegion().getValue().toByteArray()); if (!regionName.contains(TableName.META_TABLE_NAME.getNameAsString())) { tableScannerId = scanRes.getScannerId(); } return scanRes; } }
@Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { ScanResponse.Builder builder = ScanResponse.newBuilder(); try { if (request.hasScan()) { byte[] regionName = request.getRegion().getValue().toByteArray(); builder.setScannerId(openScanner(regionName, null)); builder.setMoreResults(true); } else { long scannerId = request.getScannerId(); Result result = next(scannerId); if (result != null) { builder.addCellsPerResult(result.size()); List<CellScannable> results = new ArrayList<CellScannable>(1); results.add(result); ((PayloadCarryingRpcController) controller).setCellScanner(CellUtil .createCellScanner(results)); builder.setMoreResults(true); } else { builder.setMoreResults(false); close(scannerId); } } } catch (IOException ie) { throw new ServiceException(ie); } return builder.build(); }
@Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { ScanRequest.Builder builder = ScanRequest.newBuilder(request); builder.setClientHandlesHeartbeats(heartbeatsEnabled); return super.scan(controller, builder.build()); }
/** * Use the scan metrics returned by the server to add to the identically named counters in the * client side metrics. If a counter does not exist with the same name as the server side metric, * the attempt to increase the counter will fail. * @param response */ private void updateServerSideMetrics(ScanResponse response) { if (this.scanMetrics == null || response == null || !response.hasScanMetrics()) return; Map<String, Long> serverMetrics = ResponseConverter.getScanMetrics(response); for (Entry<String, Long> entry : serverMetrics.entrySet()) { this.scanMetrics.addToCounter(entry.getKey(), entry.getValue()); } }
@Override public Result[] call(int timeout) throws IOException { if (this.closed) return null; if (Thread.interrupted()) { throw new InterruptedIOException(); } ScanRequest request = RequestConverter.buildScanRequest(getLocation() .getRegionInfo().getRegionName(), getScan(), getCaching(), true); ScanResponse response = null; controller = controllerFactory.newController(); try { controller.setPriority(getTableName()); controller.setCallTimeout(timeout); response = getStub().scan(controller, request); Result[] results = ResponseConverter.getResults(controller.cellScanner(), response); if (response.hasMoreResultsInRegion()) { setHasMoreResultsContext(true); setServerHasMoreResults(response.getMoreResultsInRegion()); } else { setHasMoreResultsContext(false); } // We need to update result metrics since we are overriding call() updateResultsMetrics(results); return results; } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
@Override public ScanResponse scan(RpcController controller, ScanRequest request) throws ServiceException { // Presume it is a scan of meta for now. Not all scans provide a region spec expecting // the server to keep reference by scannerid. TODO. return doMetaScanResponse(meta, sequenceids, request); }
static ScanResponse doMetaScanResponse(final SortedMap<byte [], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final ScanRequest request) { ScanResponse.Builder builder = ScanResponse.newBuilder(); int max = request.getNumberOfRows(); int count = 0; Map<byte [], Pair<HRegionInfo, ServerName>> tail = request.hasScan()? meta.tailMap(request.getScan().getStartRow().toByteArray()): meta; ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); for (Map.Entry<byte [], Pair<HRegionInfo, ServerName>> e: tail.entrySet()) { // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only. if (max <= 0) break; if (++count > max) break; HRegionInfo hri = e.getValue().getFirst(); ByteString row = ByteStringer.wrap(hri.getRegionName()); resultBuilder.clear(); resultBuilder.addCell(getRegionInfo(row, hri)); resultBuilder.addCell(getServer(row, e.getValue().getSecond())); resultBuilder.addCell(getStartCode(row)); builder.addResults(resultBuilder.build()); // Set more to false if we are on the last region in table. if (hri.getEndKey().length <= 0) builder.setMoreResults(false); else builder.setMoreResults(true); } // If no scannerid, set one. builder.setScannerId(request.hasScannerId()? request.getScannerId(): sequenceids.incrementAndGet()); return builder.build(); }
static ScanResponse doMetaScanResponse(final SortedMap<byte [], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final ScanRequest request) { ScanResponse.Builder builder = ScanResponse.newBuilder(); int max = request.getNumberOfRows(); int count = 0; Map<byte [], Pair<HRegionInfo, ServerName>> tail = request.hasScan()? meta.tailMap(request.getScan().getStartRow().toByteArray()): meta; ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); for (Map.Entry<byte [], Pair<HRegionInfo, ServerName>> e: tail.entrySet()) { // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only. if (max <= 0) break; if (++count > max) break; HRegionInfo hri = e.getValue().getFirst(); ByteString row = HBaseZeroCopyByteString.wrap(hri.getRegionName()); resultBuilder.clear(); resultBuilder.addCell(getRegionInfo(row, hri)); resultBuilder.addCell(getServer(row, e.getValue().getSecond())); resultBuilder.addCell(getStartCode(row)); builder.addResults(resultBuilder.build()); // Set more to false if we are on the last region in table. if (hri.getEndKey().length <= 0) builder.setMoreResults(false); else builder.setMoreResults(true); } // If no scannerid, set one. builder.setScannerId(request.hasScannerId()? request.getScannerId(): sequenceids.incrementAndGet()); return builder.build(); }