private MultiResponse buildMultiResponse(MultiRequest req) { MultiResponse.Builder builder = MultiResponse.newBuilder(); RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder(); ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); for (RegionAction regionAction: req.getRegionActionList()) { regionActionResultBuilder.clear(); for (ClientProtos.Action action: regionAction.getActionList()) { roeBuilder.clear(); roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); roeBuilder.setIndex(action.getIndex()); regionActionResultBuilder.addResultOrException(roeBuilder.build()); } builder.addRegionActionResult(regionActionResultBuilder.build()); } return builder.build(); }
/** * Create a protocol buffer MultiRequest for row mutations. * Does not propagate Action absolute position. Does not set atomic action on the created * RegionAtomic. Caller should do that if wanted. * @param regionName * @param rowMutations * @return a data-laden RegionMutation.Builder * @throws IOException */ public static RegionAction.Builder buildRegionAction(final byte [] regionName, final RowMutations rowMutations) throws IOException { RegionAction.Builder builder = getRegionActionBuilderWithRegion(RegionAction.newBuilder(), regionName); ClientProtos.Action.Builder actionBuilder = ClientProtos.Action.newBuilder(); MutationProto.Builder mutationBuilder = MutationProto.newBuilder(); for (Mutation mutation: rowMutations.getMutations()) { MutationType mutateType = null; if (mutation instanceof Put) { mutateType = MutationType.PUT; } else if (mutation instanceof Delete) { mutateType = MutationType.DELETE; } else { throw new DoNotRetryIOException("RowMutations supports only put and delete, not " + mutation.getClass().getName()); } mutationBuilder.clear(); MutationProto mp = ProtobufUtil.toMutation(mutateType, mutation, mutationBuilder); actionBuilder.clear(); actionBuilder.setMutation(mp); builder.addAction(actionBuilder.build()); } return builder; }
/** * Create a protocol buffer MultiRequest for row mutations that does not hold data. Data/Cells * are carried outside of protobuf. Return references to the Cells in <code>cells</code> param. * Does not propagate Action absolute position. Does not set atomic action on the created * RegionAtomic. Caller should do that if wanted. * @param regionName * @param rowMutations * @param cells Return in here a list of Cells as CellIterable. * @return a region mutation minus data * @throws IOException */ public static RegionAction.Builder buildNoDataRegionAction(final byte[] regionName, final RowMutations rowMutations, final List<CellScannable> cells, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { for (Mutation mutation: rowMutations.getMutations()) { MutationType type = null; if (mutation instanceof Put) { type = MutationType.PUT; } else if (mutation instanceof Delete) { type = MutationType.DELETE; } else { throw new DoNotRetryIOException("RowMutations supports only put and delete, not " + mutation.getClass().getName()); } mutationBuilder.clear(); MutationProto mp = ProtobufUtil.toMutationNoData(type, mutation, mutationBuilder); cells.add(mutation); actionBuilder.clear(); regionActionBuilder.addAction(actionBuilder.setMutation(mp).build()); } return regionActionBuilder; }
static MultiResponse doMultiResponse(final SortedMap<byte [], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final MultiRequest request) { // Make a response to match the request. Act like there were no failures. ClientProtos.MultiResponse.Builder builder = ClientProtos.MultiResponse.newBuilder(); // Per Region. RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder(); ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); for (RegionAction regionAction: request.getRegionActionList()) { regionActionResultBuilder.clear(); // Per Action in a Region. for (ClientProtos.Action action: regionAction.getActionList()) { roeBuilder.clear(); // Return empty Result and proper index as result. roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); roeBuilder.setIndex(action.getIndex()); regionActionResultBuilder.addResultOrException(roeBuilder.build()); } builder.addRegionActionResult(regionActionResultBuilder.build()); } return builder.build(); }
private boolean isWriteRequest(final RequestHeader header, final Message param) { // TODO: Is there a better way to do this? String methodName = header.getMethodName(); if (methodName.equalsIgnoreCase("multi") && param instanceof MultiRequest) { MultiRequest multi = (MultiRequest)param; for (RegionAction regionAction : multi.getRegionActionList()) { for (Action action: regionAction.getActionList()) { if (action.hasMutation()) { return true; } } } } if (methodName.equalsIgnoreCase("mutate")) { return true; } return false; }
/** * {@inheritDoc} */ @Override public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable<Void> callable = new RegionServerCallable<Void>(connection, getName(), rm.getRow()) { @Override public Void call(int callTimeout) throws IOException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( getLocation().getRegionInfo().getRegionName(), rm); regionMutationBuilder.setAtomic(true); MultiRequest request = MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build(); getStub().multi(controller, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } return null; } }; rpcCallerFactory.<Void> newCaller().callWithRetries(callable, this.operationTimeout); }
/** * {@inheritDoc} */ @Override public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable<Void> callable = new RegionServerCallable<Void>(connection, getName(), rm.getRow()) { public Void call() throws IOException { try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( getLocation().getRegionInfo().getRegionName(), rm); regionMutationBuilder.setAtomic(true); MultiRequest request = MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build(); PayloadCarryingRpcController pcrc = rpcControllerFactory.newController(); pcrc.setPriority(tableName); getStub().multi(pcrc, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } return null; } }; rpcCallerFactory.<Void> newCaller().callWithRetries(callable, this.operationTimeout); }
/** * {@inheritDoc} */ @Override public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable<Void> callable = new RegionServerCallable<Void>(connection, getName(), rm.getRow()) { public Void call(int callTimeout) throws IOException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( getLocation().getRegionInfo().getRegionName(), rm); regionMutationBuilder.setAtomic(true); MultiRequest request = MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build(); getStub().multi(controller, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } return null; } }; rpcCallerFactory.<Void> newCaller().callWithRetries(callable, this.operationTimeout); }
/** * {@inheritDoc} */ @Override public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable<Void> callable = new RegionServerCallable<Void>(connection, getName(), rm.getRow()) { public Void call() throws IOException { try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( getLocation().getRegionInfo().getRegionName(), rm); regionMutationBuilder.setAtomic(true); MultiRequest request = MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build(); PayloadCarryingRpcController pcrc = new PayloadCarryingRpcController(); pcrc.setPriority(tableName); getStub().multi(null, request); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } return null; } }; rpcCallerFactory.<Void> newCaller().callWithRetries(callable, this.operationTimeout); }
private boolean isWriteRequest(final RequestHeader header, final Message param) { // TODO: Is there a better way to do this? if (param instanceof MultiRequest) { MultiRequest multi = (MultiRequest)param; for (RegionAction regionAction : multi.getRegionActionList()) { for (Action action: regionAction.getActionList()) { if (action.hasMutation()) { return true; } } } } if (param instanceof MutateRequest) { return true; } // Below here are methods for master. It's a pretty brittle version of this. // Not sure that master actually needs a read/write queue since 90% of requests to // master are writing to status or changing the meta table. // All other read requests are admin generated and can be processed whenever. // However changing that would require a pretty drastic change and should be done for // the next major release and not as a fix for HBASE-14239 if (param instanceof RegionServerStatusProtos.ReportRegionStateTransitionRequest) { return true; } if (param instanceof RegionServerStatusProtos.RegionServerStartupRequest) { return true; } if (param instanceof RegionServerStatusProtos.RegionServerReportRequest) { return true; } return false; }
/** * Create a protocol buffer MutateRequest for conditioned row mutations * * @param regionName * @param row * @param family * @param qualifier * @param comparator * @param compareType * @param rowMutations * @return a mutate request * @throws IOException */ public static ClientProtos.MultiRequest buildMutateRequest( final byte[] regionName, final byte[] row, final byte[] family, final byte [] qualifier, final ByteArrayComparable comparator, final CompareType compareType, final RowMutations rowMutations) throws IOException { RegionAction.Builder builder = getRegionActionBuilderWithRegion(RegionAction.newBuilder(), regionName); builder.setAtomic(true); ClientProtos.Action.Builder actionBuilder = ClientProtos.Action.newBuilder(); MutationProto.Builder mutationBuilder = MutationProto.newBuilder(); Condition condition = buildCondition( row, family, qualifier, comparator, compareType); for (Mutation mutation: rowMutations.getMutations()) { MutationType mutateType = null; if (mutation instanceof Put) { mutateType = MutationType.PUT; } else if (mutation instanceof Delete) { mutateType = MutationType.DELETE; } else { throw new DoNotRetryIOException("RowMutations supports only put and delete, not " + mutation.getClass().getName()); } mutationBuilder.clear(); MutationProto mp = ProtobufUtil.toMutation(mutateType, mutation, mutationBuilder); actionBuilder.clear(); actionBuilder.setMutation(mp); builder.addAction(actionBuilder.build()); } ClientProtos.MultiRequest request = ClientProtos.MultiRequest.newBuilder().addRegionAction(builder.build()) .setCondition(condition).build(); return request; }
/** * {@inheritDoc} */ @Override public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable<Void> callable = new RegionServerCallable<Void>(connection, getName(), rm.getRow()) { @Override public Void call(int callTimeout) throws IOException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( getLocation().getRegionInfo().getRegionName(), rm); regionMutationBuilder.setAtomic(true); MultiRequest request = MultiRequest.newBuilder().addRegionAction(regionMutationBuilder.build()).build(); ClientProtos.MultiResponse response = getStub().multi(controller, request); ClientProtos.RegionActionResult res = response.getRegionActionResultList().get(0); if (res.hasException()) { Throwable ex = ProtobufUtil.toException(res.getException()); if(ex instanceof IOException) { throw (IOException)ex; } throw new IOException("Failed to mutate row: "+Bytes.toStringBinary(rm.getRow()), ex); } } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } return null; } }; rpcCallerFactory.<Void> newCaller().callWithRetries(callable, this.operationTimeout); }
/** * Create a protocol buffer multi request for a list of actions. * Propagates Actions original index. * * @param regionName * @param actions * @return a multi request * @throws IOException */ public static <R> RegionAction.Builder buildRegionAction(final byte[] regionName, final List<Action<R>> actions, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { for (Action<R> action: actions) { Row row = action.getAction(); actionBuilder.clear(); actionBuilder.setIndex(action.getOriginalIndex()); mutationBuilder.clear(); if (row instanceof Get) { Get g = (Get)row; regionActionBuilder.addAction(actionBuilder.setGet(ProtobufUtil.toGet(g))); } else if (row instanceof Put) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.PUT, (Put)row, mutationBuilder))); } else if (row instanceof Delete) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.DELETE, (Delete)row, mutationBuilder))); } else if (row instanceof Append) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.APPEND, (Append)row, mutationBuilder))); } else if (row instanceof Increment) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation((Increment)row, mutationBuilder))); } else if (row instanceof RowMutations) { throw new UnsupportedOperationException("No RowMutations in multi calls; use mutateRow"); } else { throw new DoNotRetryIOException("Multi doesn't support " + row.getClass().getName()); } } return regionActionBuilder; }
public static void main(String[] args) throws IOException, SecurityException, NoSuchMethodException, InterruptedException { if (args.length != 2) { System.out.println("Usage: TestAsyncIPC <CYCLES> <CELLS_PER_CYCLE>"); return; } // ((Log4JLogger)HBaseServer.LOG).getLogger().setLevel(Level.INFO); // ((Log4JLogger)HBaseClient.LOG).getLogger().setLevel(Level.INFO); int cycles = Integer.parseInt(args[0]); int cellcount = Integer.parseInt(args[1]); Configuration conf = HBaseConfiguration.create(); TestRpcServer rpcServer = new TestRpcServer(); MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo"); EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build(); AsyncRpcClient client = new AsyncRpcClient(conf); KeyValue kv = BIG_CELL; Put p = new Put(CellUtil.cloneRow(kv)); for (int i = 0; i < cellcount; i++) { p.add(kv); } RowMutations rm = new RowMutations(CellUtil.cloneRow(kv)); rm.add(p); try { rpcServer.start(); InetSocketAddress address = rpcServer.getListenerAddress(); if (address == null) { throw new IOException("Listener channel is closed"); } long startTime = System.currentTimeMillis(); User user = User.getCurrent(); for (int i = 0; i < cycles; i++) { List<CellScannable> cells = new ArrayList<CellScannable>(); // Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm); ClientProtos.RegionAction.Builder builder = RequestConverter.buildNoDataRegionAction(HConstants.EMPTY_BYTE_ARRAY, rm, cells, RegionAction.newBuilder(), ClientProtos.Action.newBuilder(), MutationProto.newBuilder()); builder.setRegion(RegionSpecifier .newBuilder() .setType(RegionSpecifierType.REGION_NAME) .setValue( ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()))); if (i % 100000 == 0) { LOG.info("" + i); // Uncomment this for a thread dump every so often. // ReflectionUtils.printThreadInfo(new PrintWriter(System.out), // "Thread dump " + Thread.currentThread().getName()); } PayloadCarryingRpcController pcrc = new PayloadCarryingRpcController(CellUtil.createCellScanner(cells)); // Pair<Message, CellScanner> response = client.call(pcrc, md, builder.build(), param, user, address, new MetricsConnection.CallStats()); /* * int count = 0; while (p.getSecond().advance()) { count++; } assertEquals(cells.size(), * count); */ } LOG.info("Cycled " + cycles + " time(s) with " + cellcount + " cell(s) in " + (System.currentTimeMillis() - startTime) + "ms"); } finally { client.close(); rpcServer.stop(); } }
public static void main(String[] args) throws IOException, SecurityException, NoSuchMethodException, InterruptedException { if (args.length != 2) { System.out.println("Usage: TestIPC <CYCLES> <CELLS_PER_CYCLE>"); return; } // ((Log4JLogger)HBaseServer.LOG).getLogger().setLevel(Level.INFO); // ((Log4JLogger)HBaseClient.LOG).getLogger().setLevel(Level.INFO); int cycles = Integer.parseInt(args[0]); int cellcount = Integer.parseInt(args[1]); Configuration conf = HBaseConfiguration.create(); TestRpcServer rpcServer = new TestRpcServer(); MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo"); EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build(); RpcClientImpl client = new RpcClientImpl(conf, HConstants.CLUSTER_ID_DEFAULT); KeyValue kv = BIG_CELL; Put p = new Put(CellUtil.cloneRow(kv)); for (int i = 0; i < cellcount; i++) { p.add(kv); } RowMutations rm = new RowMutations(CellUtil.cloneRow(kv)); rm.add(p); try { rpcServer.start(); long startTime = System.currentTimeMillis(); User user = User.getCurrent(); InetSocketAddress address = rpcServer.getListenerAddress(); if (address == null) { throw new IOException("Listener channel is closed"); } for (int i = 0; i < cycles; i++) { List<CellScannable> cells = new ArrayList<CellScannable>(); // Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm); ClientProtos.RegionAction.Builder builder = RequestConverter.buildNoDataRegionAction(HConstants.EMPTY_BYTE_ARRAY, rm, cells, RegionAction.newBuilder(), ClientProtos.Action.newBuilder(), MutationProto.newBuilder()); builder.setRegion(RegionSpecifier .newBuilder() .setType(RegionSpecifierType.REGION_NAME) .setValue( ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()))); if (i % 100000 == 0) { LOG.info("" + i); // Uncomment this for a thread dump every so often. // ReflectionUtils.printThreadInfo(new PrintWriter(System.out), // "Thread dump " + Thread.currentThread().getName()); } PayloadCarryingRpcController pcrc = new PayloadCarryingRpcController(CellUtil.createCellScanner(cells)); // Pair<Message, CellScanner> response = client.call(pcrc, md, builder.build(), param, user, address, new MetricsConnection.CallStats()); /* * int count = 0; while (p.getSecond().advance()) { count++; } assertEquals(cells.size(), * count); */ } LOG.info("Cycled " + cycles + " time(s) with " + cellcount + " cell(s) in " + (System.currentTimeMillis() - startTime) + "ms"); } finally { client.close(); rpcServer.stop(); } }
private static RegionAction.Builder getRegionActionBuilderWithRegion( final RegionAction.Builder regionActionBuilder, final byte [] regionName) { RegionSpecifier region = buildRegionSpecifier(RegionSpecifierType.REGION_NAME, regionName); regionActionBuilder.setRegion(region); return regionActionBuilder; }
/** * Create a protocol buffer multi request for a list of actions. * Propagates Actions original index. * * @param regionName * @param actions * @return a multi request * @throws IOException */ public static <R> RegionAction.Builder buildRegionAction(final byte[] regionName, final List<Action<R>> actions, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { for (Action<R> action: actions) { Row row = action.getAction(); actionBuilder.clear(); actionBuilder.setIndex(action.getOriginalIndex()); mutationBuilder.clear(); if (row instanceof Get) { Get g = (Get)row; regionActionBuilder.addAction(actionBuilder.setGet(ProtobufUtil.toGet(g))); } else if (row instanceof Put) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.PUT, (Put)row, mutationBuilder))); } else if (row instanceof Delete) { regionActionBuilder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.DELETE, (Delete)row, mutationBuilder))); } else if (row instanceof Append) { regionActionBuilder.addAction(actionBuilder.setMutation(ProtobufUtil.toMutation( MutationType.APPEND, (Append)row, mutationBuilder, action.getNonce()))); } else if (row instanceof Increment) { regionActionBuilder.addAction(actionBuilder.setMutation( ProtobufUtil.toMutation((Increment)row, mutationBuilder, action.getNonce()))); } else if (row instanceof RegionCoprocessorServiceExec) { RegionCoprocessorServiceExec exec = (RegionCoprocessorServiceExec) row; regionActionBuilder.addAction(actionBuilder.setServiceCall( ClientProtos.CoprocessorServiceCall.newBuilder() .setRow(ByteStringer.wrap(exec.getRow())) .setServiceName(exec.getMethod().getService().getFullName()) .setMethodName(exec.getMethod().getName()) .setRequest(exec.getRequest().toByteString()))); } else if (row instanceof RowMutations) { throw new UnsupportedOperationException("No RowMutations in multi calls; use mutateRow"); } else { throw new DoNotRetryIOException("Multi doesn't support " + row.getClass().getName()); } } return regionActionBuilder; }
/** * Create a protocol buffer multirequest with NO data for a list of actions (data is carried * otherwise than via protobuf). This means it just notes attributes, whether to write the * WAL, etc., and the presence in protobuf serves as place holder for the data which is * coming along otherwise. Note that Get is different. It does not contain 'data' and is always * carried by protobuf. We return references to the data by adding them to the passed in * <code>data</code> param. * * <p>Propagates Actions original index. * * @param regionName * @param actions * @param cells Place to stuff references to actual data. * @return a multi request that does not carry any data. * @throws IOException */ public static <R> RegionAction.Builder buildNoDataRegionAction(final byte[] regionName, final List<Action<R>> actions, final List<CellScannable> cells, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { RegionAction.Builder builder = getRegionActionBuilderWithRegion( RegionAction.newBuilder(), regionName); for (Action<R> action: actions) { Row row = action.getAction(); actionBuilder.clear(); actionBuilder.setIndex(action.getOriginalIndex()); mutationBuilder.clear(); if (row instanceof Get) { Get g = (Get)row; builder.addAction(actionBuilder.setGet(ProtobufUtil.toGet(g))); } else if (row instanceof Put) { Put p = (Put)row; cells.add(p); builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutationNoData(MutationType.PUT, p, mutationBuilder))); } else if (row instanceof Delete) { Delete d = (Delete)row; int size = d.size(); // Note that a legitimate Delete may have a size of zero; i.e. a Delete that has nothing // in it but the row to delete. In this case, the current implementation does not make // a KeyValue to represent a delete-of-all-the-row until we serialize... For such cases // where the size returned is zero, we will send the Delete fully pb'd rather than have // metadata only in the pb and then send the kv along the side in cells. if (size > 0) { cells.add(d); builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutationNoData(MutationType.DELETE, d, mutationBuilder))); } else { builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.DELETE, d, mutationBuilder))); } } else if (row instanceof Append) { Append a = (Append)row; cells.add(a); builder.addAction(actionBuilder.setMutation(ProtobufUtil.toMutationNoData( MutationType.APPEND, a, mutationBuilder, action.getNonce()))); } else if (row instanceof Increment) { Increment i = (Increment)row; cells.add(i); builder.addAction(actionBuilder.setMutation(ProtobufUtil.toMutationNoData( MutationType.INCREMENT, i, mutationBuilder, action.getNonce()))); } else if (row instanceof RegionCoprocessorServiceExec) { RegionCoprocessorServiceExec exec = (RegionCoprocessorServiceExec) row; builder.addAction(actionBuilder.setServiceCall(ClientProtos.CoprocessorServiceCall .newBuilder().setRow(ByteStringer.wrap(exec.getRow())) .setServiceName(exec.getMethod().getService().getFullName()) .setMethodName(exec.getMethod().getName()) .setRequest(exec.getRequest().toByteString()))); } else if (row instanceof RowMutations) { throw new UnsupportedOperationException("No RowMutations in multi calls; use mutateRow"); } else { throw new DoNotRetryIOException("Multi doesn't support " + row.getClass().getName()); } } return builder; }
/** * Get the results from a protocol buffer MultiResponse * * @param request the protocol buffer MultiResponse to convert * @param cells Cells to go with the passed in <code>proto</code>. Can be null. * @return the results that were in the MultiResponse (a Result or an Exception). * @throws IOException */ public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request, final MultiResponse response, final CellScanner cells) throws IOException { int requestRegionActionCount = request.getRegionActionCount(); int responseRegionActionResultCount = response.getRegionActionResultCount(); if (requestRegionActionCount != responseRegionActionResultCount) { throw new IllegalStateException("Request mutation count=" + responseRegionActionResultCount + " does not match response mutation result count=" + responseRegionActionResultCount); } org.apache.hadoop.hbase.client.MultiResponse results = new org.apache.hadoop.hbase.client.MultiResponse(); for (int i = 0; i < responseRegionActionResultCount; i++) { RegionAction actions = request.getRegionAction(i); RegionActionResult actionResult = response.getRegionActionResult(i); HBaseProtos.RegionSpecifier rs = actions.getRegion(); if (rs.hasType() && (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){ throw new IllegalArgumentException( "We support only encoded types for protobuf multi response."); } byte[] regionName = rs.getValue().toByteArray(); if (actionResult.hasException()) { Throwable regionException = ProtobufUtil.toException(actionResult.getException()); results.addException(regionName, regionException); continue; } if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) { throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() + ", actionResult.getResultOrExceptionCount=" + actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion()); } for (ResultOrException roe : actionResult.getResultOrExceptionList()) { Object responseValue; if (roe.hasException()) { responseValue = ProtobufUtil.toException(roe.getException()); } else if (roe.hasResult()) { responseValue = ProtobufUtil.toResult(roe.getResult(), cells); // add the load stats, if we got any if (roe.hasLoadStats()) { ((Result) responseValue).addResults(roe.getLoadStats()); } } else if (roe.hasServiceResult()) { responseValue = roe.getServiceResult(); } else { // no result & no exception. Unexpected. throw new IllegalStateException("No result & no exception roe=" + roe + " for region " + actions.getRegion()); } results.add(regionName, roe.getIndex(), responseValue); } } return results; }
public static void main(String[] args) throws IOException, SecurityException, NoSuchMethodException, InterruptedException { if (args.length != 2) { System.out.println("Usage: TestIPC <CYCLES> <CELLS_PER_CYCLE>"); return; } // ((Log4JLogger)HBaseServer.LOG).getLogger().setLevel(Level.INFO); // ((Log4JLogger)HBaseClient.LOG).getLogger().setLevel(Level.INFO); int cycles = Integer.parseInt(args[0]); int cellcount = Integer.parseInt(args[1]); Configuration conf = HBaseConfiguration.create(); TestRpcServer rpcServer = new TestRpcServer(); MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo"); EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build(); RpcClientImpl client = new RpcClientImpl(conf, HConstants.CLUSTER_ID_DEFAULT); KeyValue kv = KeyValueUtil.ensureKeyValue(BIG_CELL); Put p = new Put(kv.getRow()); for (int i = 0; i < cellcount; i++) { p.add(kv); } RowMutations rm = new RowMutations(kv.getRow()); rm.add(p); try { rpcServer.start(); InetSocketAddress address = rpcServer.getListenerAddress(); long startTime = System.currentTimeMillis(); User user = User.getCurrent(); for (int i = 0; i < cycles; i++) { List<CellScannable> cells = new ArrayList<CellScannable>(); // Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm); ClientProtos.RegionAction.Builder builder = RequestConverter.buildNoDataRegionAction( HConstants.EMPTY_BYTE_ARRAY, rm, cells, RegionAction.newBuilder(), ClientProtos.Action.newBuilder(), MutationProto.newBuilder()); builder.setRegion(RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME). setValue(ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()))); if (i % 100000 == 0) { LOG.info("" + i); // Uncomment this for a thread dump every so often. // ReflectionUtils.printThreadInfo(new PrintWriter(System.out), // "Thread dump " + Thread.currentThread().getName()); } CellScanner cellScanner = CellUtil.createCellScanner(cells); Pair<Message, CellScanner> response = client.call(null, md, builder.build(), cellScanner, param, user, address, 0); /* int count = 0; while (p.getSecond().advance()) { count++; } assertEquals(cells.size(), count);*/ } LOG.info("Cycled " + cycles + " time(s) with " + cellcount + " cell(s) in " + (System.currentTimeMillis() - startTime) + "ms"); } finally { client.close(); rpcServer.stop(); } }
/** * Create a protocol buffer multirequest with NO data for a list of actions (data is carried * otherwise than via protobuf). This means it just notes attributes, whether to write the * WAL, etc., and the presence in protobuf serves as place holder for the data which is * coming along otherwise. Note that Get is different. It does not contain 'data' and is always * carried by protobuf. We return references to the data by adding them to the passed in * <code>data</code> param. * * <p>Propagates Actions original index. * * @param regionName * @param actions * @param cells Place to stuff references to actual data. * @return a multi request that does not carry any data. * @throws IOException */ public static <R> RegionAction.Builder buildNoDataRegionAction(final byte[] regionName, final List<Action<R>> actions, final List<CellScannable> cells, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { RegionAction.Builder builder = getRegionActionBuilderWithRegion( RegionAction.newBuilder(), regionName); for (Action<R> action: actions) { Row row = action.getAction(); actionBuilder.clear(); actionBuilder.setIndex(action.getOriginalIndex()); mutationBuilder.clear(); if (row instanceof Get) { Get g = (Get)row; builder.addAction(actionBuilder.setGet(ProtobufUtil.toGet(g))); } else if (row instanceof Put) { Put p = (Put)row; cells.add(p); builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutationNoData(MutationType.PUT, p, mutationBuilder))); } else if (row instanceof Delete) { Delete d = (Delete)row; int size = d.size(); // Note that a legitimate Delete may have a size of zero; i.e. a Delete that has nothing // in it but the row to delete. In this case, the current implementation does not make // a KeyValue to represent a delete-of-all-the-row until we serialize... For such cases // where the size returned is zero, we will send the Delete fully pb'd rather than have // metadata only in the pb and then send the kv along the side in cells. if (size > 0) { cells.add(d); builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutationNoData(MutationType.DELETE, d, mutationBuilder))); } else { builder.addAction(actionBuilder. setMutation(ProtobufUtil.toMutation(MutationType.DELETE, d, mutationBuilder))); } } else if (row instanceof Append) { Append a = (Append)row; cells.add(a); builder.addAction(actionBuilder.setMutation(ProtobufUtil.toMutationNoData( MutationType.APPEND, a, mutationBuilder, action.getNonce()))); } else if (row instanceof Increment) { Increment i = (Increment)row; cells.add(i); builder.addAction(actionBuilder.setMutation(ProtobufUtil.toMutationNoData( MutationType.INCREMENT, i, mutationBuilder, action.getNonce()))); } else if (row instanceof RowMutations) { continue; // ignore RowMutations } else { throw new DoNotRetryIOException("Multi doesn't support " + row.getClass().getName()); } } return builder; }
public static void main(String[] args) throws IOException, SecurityException, NoSuchMethodException, InterruptedException { if (args.length != 2) { System.out.println("Usage: TestIPC <CYCLES> <CELLS_PER_CYCLE>"); return; } // ((Log4JLogger)HBaseServer.LOG).getLogger().setLevel(Level.INFO); // ((Log4JLogger)HBaseClient.LOG).getLogger().setLevel(Level.INFO); int cycles = Integer.parseInt(args[0]); int cellcount = Integer.parseInt(args[1]); Configuration conf = HBaseConfiguration.create(); TestRpcServer rpcServer = new TestRpcServer(); MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo"); EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build(); RpcClient client = new RpcClient(conf, HConstants.CLUSTER_ID_DEFAULT); KeyValue kv = KeyValueUtil.ensureKeyValue(BIG_CELL); Put p = new Put(kv.getRow()); for (int i = 0; i < cellcount; i++) { p.add(kv); } RowMutations rm = new RowMutations(kv.getRow()); rm.add(p); try { rpcServer.start(); InetSocketAddress address = rpcServer.getListenerAddress(); long startTime = System.currentTimeMillis(); User user = User.getCurrent(); for (int i = 0; i < cycles; i++) { List<CellScannable> cells = new ArrayList<CellScannable>(); // Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm); ClientProtos.RegionAction.Builder builder = RequestConverter.buildNoDataRegionAction( HConstants.EMPTY_BYTE_ARRAY, rm, cells, RegionAction.newBuilder(), ClientProtos.Action.newBuilder(), MutationProto.newBuilder()); builder.setRegion(RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME). setValue(ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()))); if (i % 100000 == 0) { LOG.info("" + i); // Uncomment this for a thread dump every so often. // ReflectionUtils.printThreadInfo(new PrintWriter(System.out), // "Thread dump " + Thread.currentThread().getName()); } CellScanner cellScanner = CellUtil.createCellScanner(cells); Pair<Message, CellScanner> response = client.call(md, builder.build(), cellScanner, param, user, address, 0); /* int count = 0; while (p.getSecond().advance()) { count++; } assertEquals(cells.size(), count);*/ } LOG.info("Cycled " + cycles + " time(s) with " + cellcount + " cell(s) in " + (System.currentTimeMillis() - startTime) + "ms"); } finally { client.stop(); rpcServer.stop(); } }
