private void testVerifyMetaRegionLocationWithException(Exception ex) throws IOException, InterruptedException, KeeperException, ServiceException { // Mock an ClientProtocol. final ClientProtos.ClientService.BlockingInterface implementation = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); ClusterConnection connection = mockConnection(null, implementation); // If a 'get' is called on mocked interface, throw connection refused. Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())). thenThrow(new ServiceException(ex)); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING); assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); }
private MultiResponse buildMultiResponse(MultiRequest req) { MultiResponse.Builder builder = MultiResponse.newBuilder(); RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder(); ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); for (RegionAction regionAction: req.getRegionActionList()) { regionActionResultBuilder.clear(); for (ClientProtos.Action action: regionAction.getActionList()) { roeBuilder.clear(); roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); roeBuilder.setIndex(action.getIndex()); regionActionResultBuilder.addResultOrException(roeBuilder.build()); } builder.addRegionActionResult(regionActionResultBuilder.build()); } return builder.build(); }
@SuppressWarnings("deprecation") private HConnection getMockedConnection(final Configuration conf) throws IOException, ServiceException { HConnection c = Mockito.mock(HConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, ServerName.valueOf("example.org", 1234, 0)); Mockito.when(c.getRegionLocation((TableName) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(loc); Mockito.when(c.locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any())). thenReturn(loc); ClientProtos.ClientService.BlockingInterface hri = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito.when(hri.bulkLoadHFile((RpcController)Mockito.any(), (BulkLoadHFileRequest)Mockito.any())). thenThrow(new ServiceException(new IOException("injecting bulk load error"))); Mockito.when(c.getClient(Mockito.any(ServerName.class))). thenReturn(hri); return c; }
/** * Create a new protocol buffer GetRequest to get a row, all columns in a family. * If there is no such row, return the closest row before it. * * @param regionName the name of the region to get * @param row the row to get * @param family the column family to get * should return the immediate row before * @return a protocol buffer GetReuqest */ public static GetRequest buildGetRowOrBeforeRequest( final byte[] regionName, final byte[] row, final byte[] family) { GetRequest.Builder builder = GetRequest.newBuilder(); RegionSpecifier region = buildRegionSpecifier( RegionSpecifierType.REGION_NAME, regionName); builder.setRegion(region); Column.Builder columnBuilder = Column.newBuilder(); columnBuilder.setFamily(ByteStringer.wrap(family)); ClientProtos.Get.Builder getBuilder = ClientProtos.Get.newBuilder(); getBuilder.setRow(ByteStringer.wrap(row)); getBuilder.addColumn(columnBuilder.build()); getBuilder.setClosestRowBefore(true); builder.setGet(getBuilder.build()); return builder.build(); }
/** * Create a protocol buffer MultiRequest for row mutations. * Does not propagate Action absolute position. Does not set atomic action on the created * RegionAtomic. Caller should do that if wanted. * @param regionName * @param rowMutations * @return a data-laden RegionMutation.Builder * @throws IOException */ public static RegionAction.Builder buildRegionAction(final byte [] regionName, final RowMutations rowMutations) throws IOException { RegionAction.Builder builder = getRegionActionBuilderWithRegion(RegionAction.newBuilder(), regionName); ClientProtos.Action.Builder actionBuilder = ClientProtos.Action.newBuilder(); MutationProto.Builder mutationBuilder = MutationProto.newBuilder(); for (Mutation mutation: rowMutations.getMutations()) { MutationType mutateType = null; if (mutation instanceof Put) { mutateType = MutationType.PUT; } else if (mutation instanceof Delete) { mutateType = MutationType.DELETE; } else { throw new DoNotRetryIOException("RowMutations supports only put and delete, not " + mutation.getClass().getName()); } mutationBuilder.clear(); MutationProto mp = ProtobufUtil.toMutation(mutateType, mutation, mutationBuilder); actionBuilder.clear(); actionBuilder.setMutation(mp); builder.addAction(actionBuilder.build()); } return builder; }
/** * Create a protocol buffer MultiRequest for row mutations that does not hold data. Data/Cells * are carried outside of protobuf. Return references to the Cells in <code>cells</code> param. * Does not propagate Action absolute position. Does not set atomic action on the created * RegionAtomic. Caller should do that if wanted. * @param regionName * @param rowMutations * @param cells Return in here a list of Cells as CellIterable. * @return a region mutation minus data * @throws IOException */ public static RegionAction.Builder buildNoDataRegionAction(final byte[] regionName, final RowMutations rowMutations, final List<CellScannable> cells, final RegionAction.Builder regionActionBuilder, final ClientProtos.Action.Builder actionBuilder, final MutationProto.Builder mutationBuilder) throws IOException { for (Mutation mutation: rowMutations.getMutations()) { MutationType type = null; if (mutation instanceof Put) { type = MutationType.PUT; } else if (mutation instanceof Delete) { type = MutationType.DELETE; } else { throw new DoNotRetryIOException("RowMutations supports only put and delete, not " + mutation.getClass().getName()); } mutationBuilder.clear(); MutationProto mp = ProtobufUtil.toMutationNoData(type, mutation, mutationBuilder); cells.add(mutation); actionBuilder.clear(); regionActionBuilder.addAction(actionBuilder.setMutation(mp).build()); } return regionActionBuilder; }
/** * Convert a protobuf Durability into a client Durability */ public static Durability toDurability( final ClientProtos.MutationProto.Durability proto) { switch(proto) { case USE_DEFAULT: return Durability.USE_DEFAULT; case SKIP_WAL: return Durability.SKIP_WAL; case ASYNC_WAL: return Durability.ASYNC_WAL; case SYNC_WAL: return Durability.SYNC_WAL; case FSYNC_WAL: return Durability.FSYNC_WAL; default: return Durability.USE_DEFAULT; } }
/** * Convert a client Durability into a protbuf Durability */ public static ClientProtos.MutationProto.Durability toDurability( final Durability d) { switch(d) { case USE_DEFAULT: return ClientProtos.MutationProto.Durability.USE_DEFAULT; case SKIP_WAL: return ClientProtos.MutationProto.Durability.SKIP_WAL; case ASYNC_WAL: return ClientProtos.MutationProto.Durability.ASYNC_WAL; case SYNC_WAL: return ClientProtos.MutationProto.Durability.SYNC_WAL; case FSYNC_WAL: return ClientProtos.MutationProto.Durability.FSYNC_WAL; default: return ClientProtos.MutationProto.Durability.USE_DEFAULT; } }
/** * Convert a client Result to a protocol buffer Result * * @param result the client Result to convert * @return the converted protocol buffer Result */ public static ClientProtos.Result toResult(final Result result) { if (result.getExists() != null) { return toResult(result.getExists(), result.isStale()); } Cell[] cells = result.rawCells(); if (cells == null || cells.length == 0) { return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB; } ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder(); for (Cell c : cells) { builder.addCell(toCell(c)); } builder.setStale(result.isStale()); builder.setPartial(result.isPartial()); return builder.build(); }
/** * Convert a protocol buffer Result to a client Result * * @param proto the protocol buffer Result to convert * @return the converted client Result */ public static Result toResult(final ClientProtos.Result proto) { if (proto.hasExists()) { if (proto.getStale()) { return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE; } return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE; } List<CellProtos.Cell> values = proto.getCellList(); if (values.isEmpty()){ return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT; } List<Cell> cells = new ArrayList<Cell>(values.size()); for (CellProtos.Cell c : values) { cells.add(toCell(c)); } return Result.create(cells, null, proto.getStale(), proto.getPartial()); }
@Override public Result call(int callTimeout) throws Exception { if (controller.isCanceled()) return null; if (Thread.interrupted()) { throw new InterruptedIOException(); } byte[] reg = location.getRegionInfo().getRegionName(); ClientProtos.GetRequest request = RequestConverter.buildGetRequest(reg, get); controller.setCallTimeout(callTimeout); try { ClientProtos.GetResponse response = getStub().get(controller, request); if (response == null) { return null; } return ProtobufUtil.toResult(response.getResult()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } }
/** * Update the stats for the specified region if the result is an instance of {@link * ResultStatsUtil} * * @param r object that contains the result and possibly the statistics about the region * @param serverStats stats tracker to update from the result * @param server server from which the result was obtained * @param regionName full region name for the stats. * @return the underlying {@link Result} if the passed result is an {@link * ResultStatsUtil} or just returns the result; */ public static <T> T updateStats(T r, ServerStatisticTracker serverStats, ServerName server, byte[] regionName) { if (!(r instanceof Result)) { return r; } Result result = (Result) r; // early exit if there are no stats to collect ClientProtos.RegionLoadStats stats = result.getStats(); if(stats == null){ return r; } if (regionName != null) { serverStats.updateRegionStats(server, regionName, stats); } return r; }
/** * {@inheritDoc} * @deprecated Use reversed scan instead. */ @Override @Deprecated public Result getRowOrBefore(final byte[] row, final byte[] family) throws IOException { RegionServerCallable<Result> callable = new RegionServerCallable<Result>(this.connection, tableName, row) { @Override public Result call(int callTimeout) throws IOException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); ClientProtos.GetRequest request = RequestConverter.buildGetRowOrBeforeRequest( getLocation().getRegionInfo().getRegionName(), row, family); try { ClientProtos.GetResponse response = getStub().get(controller, request); if (!response.hasResult()) return null; return ProtobufUtil.toResult(response.getResult()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return rpcCallerFactory.<Result>newCaller().callWithRetries(callable, this.operationTimeout); }
@Test public void testAttributesSerialization() throws IOException { Scan scan = new Scan(); scan.setAttribute("attribute1", Bytes.toBytes("value1")); scan.setAttribute("attribute2", Bytes.toBytes("value2")); scan.setAttribute("attribute3", Bytes.toBytes("value3")); ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan); Scan scan2 = ProtobufUtil.toScan(scanProto); Assert.assertNull(scan2.getAttribute("absent")); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3"))); Assert.assertEquals(3, scan2.getAttributesMap().size()); }
ScanOpenNextThenExceptionThenRecoverConnection(Configuration conf, boolean managed, ExecutorService pool) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
RegionServerStoppedOnScannerOpenConnection(Configuration conf, boolean managed, ExecutorService pool, User user) throws IOException { super(conf, managed); // Mock up my stub so open scanner returns a scanner id and then on next, we throw // exceptions for three times and then after that, we return no more to scan. this.stub = Mockito.mock(ClientService.BlockingInterface.class); long sid = 12345L; try { Mockito.when(stub.scan((RpcController)Mockito.any(), (ClientProtos.ScanRequest)Mockito.any())). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid).build()). thenThrow(new ServiceException(new RegionServerStoppedException("From Mockito"))). thenReturn(ClientProtos.ScanResponse.newBuilder().setScannerId(sid). setMoreResults(false).build()); } catch (ServiceException e) { throw new IOException(e); } }
static MultiResponse doMultiResponse(final SortedMap<byte [], Pair<HRegionInfo, ServerName>> meta, final AtomicLong sequenceids, final MultiRequest request) { // Make a response to match the request. Act like there were no failures. ClientProtos.MultiResponse.Builder builder = ClientProtos.MultiResponse.newBuilder(); // Per Region. RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder(); ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); for (RegionAction regionAction: request.getRegionActionList()) { regionActionResultBuilder.clear(); // Per Action in a Region. for (ClientProtos.Action action: regionAction.getActionList()) { roeBuilder.clear(); // Return empty Result and proper index as result. roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); roeBuilder.setIndex(action.getIndex()); regionActionResultBuilder.addResultOrException(roeBuilder.build()); } builder.addRegionActionResult(regionActionResultBuilder.build()); } return builder.build(); }
static GetResponse doMetaGetResponse(final SortedMap<byte [], Pair<HRegionInfo, ServerName>> meta, final GetRequest request) { ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder(); ByteString row = request.getGet().getRow(); Pair<HRegionInfo, ServerName> p = meta.get(row.toByteArray()); if (p == null) { if (request.getGet().getClosestRowBefore()) { byte [] bytes = row.toByteArray(); SortedMap<byte [], Pair<HRegionInfo, ServerName>> head = bytes != null? meta.headMap(bytes): meta; p = head == null? null: head.get(head.lastKey()); } } if (p != null) { resultBuilder.addCell(getRegionInfo(row, p.getFirst())); resultBuilder.addCell(getServer(row, p.getSecond())); } resultBuilder.addCell(getStartCode(row)); GetResponse.Builder builder = GetResponse.newBuilder(); builder.setResult(resultBuilder.build()); return builder.build(); }
@Test public void testAttributesSerialization() throws IOException { Get get = new Get(Bytes.toBytes("row")); get.setAttribute("attribute1", Bytes.toBytes("value1")); get.setAttribute("attribute2", Bytes.toBytes("value2")); get.setAttribute("attribute3", Bytes.toBytes("value3")); ClientProtos.Get getProto = ProtobufUtil.toGet(get); Get get2 = ProtobufUtil.toGet(getProto); Assert.assertNull(get2.getAttribute("absent")); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get2.getAttribute("attribute1"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get2.getAttribute("attribute2"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), get2.getAttribute("attribute3"))); Assert.assertEquals(3, get2.getAttributesMap().size()); }
@Override public void write(DataOutput out) throws IOException { super.write(out); Preconditions.checkNotNull(scans); WritableUtils.writeVInt(out, scans.size()); for (Scan scan : scans) { ClientProtos.Scan protoScan = ProtobufUtil.toScan(scan); byte[] protoScanBytes = protoScan.toByteArray(); WritableUtils.writeVInt(out, protoScanBytes.length); out.write(protoScanBytes); } WritableUtils.writeString(out, query); WritableUtils.writeVLong(out, regionSize); }
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); int count = WritableUtils.readVInt(in); scans = Lists.newArrayListWithExpectedSize(count); for (int i = 0; i < count; i++) { byte[] protoScanBytes = new byte[WritableUtils.readVInt(in)]; in.readFully(protoScanBytes); ClientProtos.Scan protoScan = ClientProtos.Scan.parseFrom(protoScanBytes); Scan scan = ProtobufUtil.toScan(protoScan); scans.add(scan); } init(); query = WritableUtils.readString(in); regionSize = WritableUtils.readVLong(in); }
/** * Add HBase and its dependencies (only) to the job configuration. * <p> * This is intended as a low-level API, facilitating code reuse between this * class and its mapred counterpart. It also of use to extenral tools that * need to build a MapReduce job that interacts with HBase but want * fine-grained control over the jars shipped to the cluster. * </p> * @param conf The Configuration object to extend with dependencies. * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a> */ public static void addHBaseDependencyJars(Configuration conf) throws IOException { addDependencyJars(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol org.apache.hadoop.hbase.client.Put.class, // hbase-client org.apache.hadoop.hbase.CompatibilityFactory.class, // hbase-hadoop-compat org.apache.hadoop.hbase.mapreduce.TableMapper.class, // hbase-server // pull necessary dependencies org.apache.zookeeper.ZooKeeper.class, io.netty.channel.Channel.class, com.google.protobuf.Message.class, com.google.common.collect.Lists.class, org.apache.htrace.Trace.class); }
/** * Convert a client Result to a protocol buffer Result * * @param result the client Result to convert * @return the converted protocol buffer Result */ public static ClientProtos.Result toResult(final Result result) { if (result.getExists() != null) { return toResult(result.getExists()); } Cell[] cells = result.rawCells(); if (cells == null || cells.length == 0) { return EMPTY_RESULT_PB; } ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder(); for (Cell c : cells) { builder.addCell(toCell(c)); } return builder.build(); }
/** * Convert a client Result to a protocol buffer Result * * @param result the client Result to convert * @return the converted protocol buffer Result */ public static ClientProtos.Result toResult(final Result result) { if (result.getExists() != null) { return toResult(result.getExists(), result.isStale()); } Cell[] cells = result.rawCells(); if (cells == null || cells.length == 0) { return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB; } ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder(); for (Cell c : cells) { builder.addCell(toCell(c)); } builder.setStale(result.isStale()); return builder.build(); }
/** * Convert a protocol buffer Result to a client Result * * @param proto the protocol buffer Result to convert * @return the converted client Result */ public static Result toResult(final ClientProtos.Result proto) { if (proto.hasExists()) { return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE; } List<CellProtos.Cell> values = proto.getCellList(); if (values.isEmpty()){ return EMPTY_RESULT; } List<Cell> cells = new ArrayList<Cell>(values.size()); for (CellProtos.Cell c : values) { cells.add(toCell(c)); } return Result.create(cells, null); }
/** * {@inheritDoc} */ @Override public boolean checkAndMutate(final byte [] row, final byte [] family, final byte [] qualifier, final CompareOp compareOp, final byte [] value, final RowMutations rm) throws IOException { RegionServerCallable<Boolean> callable = new RegionServerCallable<Boolean>(connection, getName(), row) { @Override public Boolean call(int callTimeout) throws IOException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); try { CompareType compareType = CompareType.valueOf(compareOp.name()); MultiRequest request = RequestConverter.buildMutateRequest( getLocation().getRegionInfo().getRegionName(), row, family, qualifier, new BinaryComparator(value), compareType, rm); ClientProtos.MultiResponse response = getStub().multi(controller, request); return Boolean.valueOf(response.getProcessed()); } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } } }; return rpcCallerFactory.<Boolean> newCaller().callWithRetries(callable, this.operationTimeout); }