/** * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus * might have some entries in it where the OpStatus is FAILURE. We will leave those and set in * others in the order. * @param mutations * @param finalOpStatus * @return whether we need a ZK update or not. */ private boolean mutateLabelsRegion(List<Mutation> mutations, OperationStatus[] finalOpStatus) throws IOException { OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations .toArray(new Mutation[mutations.size()]), HConstants.NO_NONCE, HConstants.NO_NONCE); int i = 0; boolean updateZk = false; for (OperationStatus status : opStatus) { // Update the zk when atleast one of the mutation was added successfully. updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS); for (; i < finalOpStatus.length; i++) { if (finalOpStatus[i] == null) { finalOpStatus[i] = status; break; } } } return updateZk; }
@Override public void run() { byte[] value = new byte[100]; Put[] in = new Put[1]; // iterate for the specified number of operations for (int i=0; i<numOps; i++) { // generate random bytes rand.nextBytes(value); // put the randombytes and verify that we can read it. This is one // way of ensuring that rwcc manipulation in HRegion.put() is fine. Put put = new Put(rowkey); put.add(fam1, qual1, value); in[0] = put; try { OperationStatus[] ret = region.batchMutate(in); assertEquals(1, ret.length); assertEquals(OperationStatusCode.SUCCESS, ret[0].getOperationStatusCode()); assertGet(this.region, rowkey, fam1, qual1, value); } catch (IOException e) { assertTrue("Thread id " + threadNumber + " operation " + i + " failed.", false); } } }
@Override public void run() { byte[] value = new byte[100]; Put[] in = new Put[1]; // iterate for the specified number of operations for (int i=0; i<numOps; i++) { // generate random bytes rand.nextBytes(value); // put the randombytes and verify that we can read it. This is one // way of ensuring that rwcc manipulation in HRegion.put() is fine. Put put = new Put(rowkey); put.add(fam1, qual1, value); in[0] = put; try { OperationStatus[] ret = region.put(in); assertEquals(1, ret.length); assertEquals(OperationStatusCode.SUCCESS, ret[0].getOperationStatusCode()); assertGet(rowkey, fam1, qual1, value); } catch (IOException e) { assertTrue("Thread id " + threadNumber + " operation " + i + " failed.", false); } } }
/** * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus * might have some entries in it where the OpStatus is FAILURE. We will leave those and set in * others in the order. * @param mutations * @param finalOpStatus * @return whether we need a ZK update or not. */ private boolean mutateLabelsRegion(List<Mutation> mutations, OperationStatus[] finalOpStatus) throws IOException { OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations .toArray(new Mutation[mutations.size()])); int i = 0; boolean updateZk = false; for (OperationStatus status : opStatus) { // Update the zk when atleast one of the mutation was added successfully. updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS); for (; i < finalOpStatus.length; i++) { if (finalOpStatus[i] == null) { finalOpStatus[i] = status; break; } } } return updateZk; }
@Override public void run() { byte[] value = new byte[100]; Put[] in = new Put[1]; // iterate for the specified number of operations for (int i=0; i<numOps; i++) { // generate random bytes rand.nextBytes(value); // put the randombytes and verify that we can read it. This is one // way of ensuring that rwcc manipulation in HRegion.put() is fine. Put put = new Put(rowkey); put.add(fam1, qual1, value); in[0] = put; try { OperationStatus[] ret = region.batchMutate(in); assertEquals(1, ret.length); assertEquals(OperationStatusCode.SUCCESS, ret[0].getOperationStatusCode()); assertGet(rowkey, fam1, qual1, value); } catch (IOException e) { assertTrue("Thread id " + threadNumber + " operation " + i + " failed.", false); } } }
@Override public void run() { byte[] value = new byte[100]; Put[] in = new Put[1]; // iterate for the specified number of operations for (int i=0; i<numOps; i++) { // generate random bytes rand.nextBytes(value); // put the randombytes and verify that we can read it. This is one // way of ensuring that rwcc manipulation in HRegion.put() is fine. Put put = new Put(rowkey); put.addColumn(fam1, qual1, value); in[0] = put; try { OperationStatus[] ret = region.batchMutate(in); assertEquals(1, ret.length); assertEquals(OperationStatusCode.SUCCESS, ret[0].getOperationStatusCode()); assertGet(this.region, rowkey, fam1, qual1, value); } catch (IOException e) { assertTrue("Thread id " + threadNumber + " operation " + i + " failed.", false); } } }
@Test public void testBatchPutWithTsSlop() throws Exception { // add data with a timestamp that is too recent for range. Ensure assert CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); final Put[] puts = new Put[10]; MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); try { long syncs = prepareRegionForBachPut(puts, source, true); OperationStatus[] codes = this.region.batchMutate(puts); assertEquals(10, codes.length); for (int i = 0; i < 10; i++) { assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); } metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); } finally { HBaseTestingUtility.closeRegionAndWAL(this.region); this.region = null; } }
private void doBatchMutate(Mutation mutation) throws IOException { // Currently this is only called for puts and deletes, so no nonces. OperationStatus[] batchMutate = this.batchMutate(new Mutation[] { mutation }); if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) { throw new FailedSanityCheckException(batchMutate[0].getExceptionMsg()); } else if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) { throw new NoSuchColumnFamilyException(batchMutate[0].getExceptionMsg()); } }
@Override public OperationStatus[] addLabels(List<byte[]> labels) throws IOException { // Not doing specific label add. We will just add labels in Mutation // visibility expression as it // is along with every cell. OperationStatus[] status = new OperationStatus[labels.size()]; for (int i = 0; i < labels.size(); i++) { status[i] = new OperationStatus(OperationStatusCode.SUCCESS); } return status; }
@Test public void testBatchPut_whileNoRowLocksHeld() throws IOException { byte[] cf = Bytes.toBytes(COLUMN_FAMILY); byte[] qual = Bytes.toBytes("qual"); byte[] val = Bytes.toBytes("val"); this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf); MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); try { long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); LOG.info("First a batch put with all valid puts"); final Put[] puts = new Put[10]; for (int i = 0; i < 10; i++) { puts[i] = new Put(Bytes.toBytes("row_" + i)); puts[i].add(cf, qual, val); } OperationStatus[] codes = this.region.batchMutate(puts); assertEquals(10, codes.length); for (int i = 0; i < 10; i++) { assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); } metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); LOG.info("Next a batch put with one invalid family"); puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); codes = this.region.batchMutate(puts); assertEquals(10, codes.length); for (int i = 0; i < 10; i++) { assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); } metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); } finally { HRegion.closeHRegion(this.region); this.region = null; } }
@Test public void testBatchPutWithTsSlop() throws Exception { byte[] b = Bytes.toBytes(getName()); byte[] cf = Bytes.toBytes(COLUMN_FAMILY); byte[] qual = Bytes.toBytes("qual"); byte[] val = Bytes.toBytes("val"); // add data with a timestamp that is too recent for range. Ensure assert CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); this.region = initHRegion(b, getName(), CONF, cf); try { MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); final Put[] puts = new Put[10]; for (int i = 0; i < 10; i++) { puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100); puts[i].add(cf, qual, val); } OperationStatus[] codes = this.region.batchMutate(puts); assertEquals(10, codes.length); for (int i = 0; i < 10; i++) { assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); } metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); } finally { HRegion.closeHRegion(this.region); this.region = null; } }
public int put(final byte[] regionName, final List<Put> puts) throws IOException { checkOpen(); HRegion region = null; int i = 0; try { region = getRegion(regionName); if (!region.getRegionInfo().isMetaTable()) { this.cacheFlusher.reclaimMemStoreMemory(); } @SuppressWarnings("unchecked") Pair<Mutation, Integer>[] putsWithLocks = new Pair[puts.size()]; for (Put p : puts) { Integer lock = getLockFromId(p.getLockId()); putsWithLocks[i++] = new Pair<Mutation, Integer>(p, lock); } this.requestCount.addAndGet(puts.size()); OperationStatus codes[] = region.batchMutate(putsWithLocks); for (i = 0; i < codes.length; i++) { if (codes[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) { return i; } } return -1; } catch (Throwable t) { throw convertThrowableToIOE(cleanup(t)); } }
private void doPreMutationHook(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp) throws IOException { /* Run coprocessor pre hook outside of locks to avoid deadlock */ WALEdit walEdit = new WALEdit(); if (coprocessorHost != null) { for (int i = 0; i < batchOp.operations.length; i++) { Pair<Mutation, Integer> nextPair = batchOp.operations[i]; Mutation m = nextPair.getFirst(); if (m instanceof Put) { if (coprocessorHost.prePut((Put) m, walEdit, m.getWriteToWAL())) { // pre hook says skip this Put // mark as success and skip in doMiniBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else if (m instanceof Delete) { if (coprocessorHost.preDelete((Delete) m, walEdit, m.getWriteToWAL())) { // pre hook says skip this Delete // mark as success and skip in doMiniBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else { // In case of passing Append mutations along with the Puts and Deletes in batchMutate // mark the operation return code as failure so that it will not be considered in // the doMiniBatchMutation batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE, "Put/Delete mutations only supported in batchMutate() now"); } if (!walEdit.isEmpty()) { batchOp.walEditsFromCoprocessors[i] = walEdit; walEdit = new WALEdit(); } } } }
/** * Remove all the keys listed in the map from the memstore. This method is called when a * Put/Delete has updated memstore but subequently fails to update the wal. This method is then * invoked to rollback the memstore. */ private void rollbackMemstore(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp, Map<byte[], List<KeyValue>>[] familyMaps, int start, int end) { int kvsRolledback = 0; for (int i = start; i < end; i++) { // skip over request that never succeeded in the first place. if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) { continue; } // Rollback all the kvs for this row. Map<byte[], List<KeyValue>> familyMap = familyMaps[i]; for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) { byte[] family = e.getKey(); List<KeyValue> edits = e.getValue(); // Remove those keys from the memstore that matches our // key's (row, cf, cq, timestamp, memstoreTS). The interesting part is // that even the memstoreTS has to match for keys that will be rolleded-back. Store store = getStore(family); for (KeyValue kv : edits) { store.rollback(kv); kvsRolledback++; } } } LOG.debug("rollbackMemstore rolled back " + kvsRolledback + " keyvalues from start:" + start + " to end:" + end); }
public void testBatchPutWithTsSlop() throws Exception { byte[] b = Bytes.toBytes(getName()); byte[] cf = Bytes.toBytes(COLUMN_FAMILY); byte[] qual = Bytes.toBytes("qual"); byte[] val = Bytes.toBytes("val"); Configuration conf = HBaseConfiguration.create(this.conf); // add data with a timestamp that is too recent for range. Ensure assert conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); this.region = initHRegion(b, getName(), conf, cf); try{ HLog.getSyncTime(); // clear counter from prior tests assertEquals(0, HLog.getSyncTime().count); final Put[] puts = new Put[10]; for (int i = 0; i < 10; i++) { puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100); puts[i].add(cf, qual, val); } OperationStatus[] codes = this.region.put(puts); assertEquals(10, codes.length); for (int i = 0; i < 10; i++) { assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i] .getOperationStatusCode()); } assertEquals(0, HLog.getSyncTime().count); } finally { HRegion.closeHRegion(this.region); this.region = null; } }
private void doPreMutationHook(BatchOperationInProgress<?> batchOp) throws IOException { /* Run coprocessor pre hook outside of locks to avoid deadlock */ WALEdit walEdit = new WALEdit(); if (coprocessorHost != null) { for (int i = 0; i < batchOp.operations.length; i++) { Mutation m = batchOp.getMutation(i); if (m instanceof Put) { if (coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) { // pre hook says skip this Put // mark as success and skip in doMiniBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else if (m instanceof Delete) { Delete curDel = (Delete) m; if (curDel.getFamilyCellMap().isEmpty()) { // handle deleting a row case prepareDelete(curDel); } if (coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) { // pre hook says skip this Delete // mark as success and skip in pBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else { // In case of passing Append mutations along with the Puts and Deletes in batchMutate // mark the operation return code as failure so that it will not be considered in // the doMiniBatchMutation batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE, "Put/Delete mutations only supported in batchMutate() now"); } if (!walEdit.isEmpty()) { batchOp.walEditsFromCoprocessors[i] = walEdit; walEdit = new WALEdit(); } } } }
private void doBatchMutate(Mutation mutation) throws IOException, DoNotRetryIOException { // Currently this is only called for puts and deletes, so no nonces. OperationStatus[] batchMutate = this.batchMutate(new Mutation[]{mutation}, HConstants.NO_NONCE, HConstants.NO_NONCE); if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) { throw new FailedSanityCheckException(batchMutate[0].getExceptionMsg()); } else if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) { throw new NoSuchColumnFamilyException(batchMutate[0].getExceptionMsg()); } }
private void doPreMutationHook(BatchOperationInProgress<?> batchOp) throws IOException { /* Run coprocessor pre hook outside of locks to avoid deadlock */ WALEdit walEdit = new WALEdit(); if (coprocessorHost != null) { for (int i = 0 ; i < batchOp.operations.length; i++) { Mutation m = batchOp.getMutation(i); if (m instanceof Put) { if (coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) { // pre hook says skip this Put // mark as success and skip in doMiniBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else if (m instanceof Delete) { Delete curDel = (Delete) m; if (curDel.getFamilyCellMap().isEmpty()) { // handle deleting a row case prepareDelete(curDel); } if (coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) { // pre hook says skip this Delete // mark as success and skip in doMiniBatchMutation batchOp.retCodeDetails[i] = OperationStatus.SUCCESS; } } else { // In case of passing Append mutations along with the Puts and Deletes in batchMutate // mark the operation return code as failure so that it will not be considered in // the doMiniBatchMutation batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE, "Put/Delete mutations only supported in batchMutate() now"); } if (!walEdit.isEmpty()) { batchOp.walEditsFromCoprocessors[i] = walEdit; walEdit = new WALEdit(); } } } }
private void doBatchMutate(Mutation mutation) throws IOException, DoNotRetryIOException { // Currently this is only called for puts and deletes, so no nonces. OperationStatus[] batchMutate = this.batchMutate(new Mutation[] { mutation }, HConstants.NO_NONCE, HConstants.NO_NONCE); if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) { throw new FailedSanityCheckException(batchMutate[0].getExceptionMsg()); } else if (batchMutate[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) { throw new NoSuchColumnFamilyException(batchMutate[0].getExceptionMsg()); } }
/** * Remove all the keys listed in the map from the memstore. This method is * called when a Put/Delete has updated memstore but subequently fails to update * the wal. This method is then invoked to rollback the memstore. */ private void rollbackMemstore(BatchOperationInProgress<?> batchOp, Map<byte[], List<Cell>>[] familyMaps, int start, int end) { int kvsRolledback = 0; for (int i = start; i < end; i++) { // skip over request that never succeeded in the first place. if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) { continue; } // Rollback all the kvs for this row. Map<byte[], List<Cell>> familyMap = familyMaps[i]; for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) { byte[] family = e.getKey(); List<Cell> cells = e.getValue(); // Remove those keys from the memstore that matches our // key's (row, cf, cq, timestamp, memstoreTS). The interesting part is // that even the memstoreTS has to match for keys that will be rolleded-back. Store store = getStore(family); for (Cell cell: cells) { store.rollback(KeyValueUtil.ensureKeyValue(cell)); kvsRolledback++; } } } LOG.debug("rollbackMemstore rolled back " + kvsRolledback + " keyvalues from start:" + start + " to end:" + end); }
@Override public void preBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> ctx, final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException { HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices(); HRegion userRegion = ctx.getEnvironment().getRegion(); HTableDescriptor userTableDesc = userRegion.getTableDesc(); String tableName = userTableDesc.getNameAsString(); if (isNotIndexedTableDescriptor(userTableDesc)) { return; } List<IndexSpecification> indices = indexManager.getIndicesForTable(tableName); if (indices == null || indices.isEmpty()) { LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices"); return; } LOG.trace("Entering preBatchMutate for the table " + tableName); LOG.trace("Indices for the table " + tableName + " are: " + indices); HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs); // Storing this found HRegion in the index table within the thread locale. IndexEdits indexEdits = threadLocal.get(); indexEdits.indexRegion = indexRegion; for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i); if (m instanceof Put) { try { prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); } catch (IOException e) { miniBatchOp.setOperationStatus(i, new OperationStatus( OperationStatusCode.SANITY_CHECK_FAILURE, e.getMessage())); } } else if (m instanceof Delete) { prepareIndexMutations(indices, userRegion, m, tableName, indexRegion); } } indexEdits.setUpdateLocked(); indexRegion.updatesLock(); LOG.trace("Exiting preBatchMutate for the table " + tableName); }
/** * Remove all the keys listed in the map from the memstore. This method is * called when a Put/Delete has updated memstore but subequently fails to update * the wal. This method is then invoked to rollback the memstore. */ private void rollbackMemstore(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp, Map<byte[], List<KeyValue>>[] familyMaps, int start, int end) { int kvsRolledback = 0; for (int i = start; i < end; i++) { // skip over request that never succeeded in the first place. if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) { continue; } // Rollback all the kvs for this row. Map<byte[], List<KeyValue>> familyMap = familyMaps[i]; for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) { byte[] family = e.getKey(); List<KeyValue> edits = e.getValue(); // Remove those keys from the memstore that matches our // key's (row, cf, cq, timestamp, memstoreTS). The interesting part is // that even the memstoreTS has to match for keys that will be rolleded-back. Store store = getStore(family); for (KeyValue kv: edits) { store.rollback(kv); kvsRolledback++; } } } LOG.debug("rollbackMemstore rolled back " + kvsRolledback + " keyvalues from start:" + start + " to end:" + end); }
/** * Runs prePut/ preDelete coprocessor hook for input mutation in a batch * @param metrics Array of 2 ints. index 0: count of puts and index 1: count of deletes */ private void callPreMutateCPHook(int index, final WALEdit walEdit, final int[] metrics) throws IOException { Mutation m = getMutation(index); if (m instanceof Put) { if (region.coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) { // pre hook says skip this Put // mark as success and skip in doMiniBatchMutation metrics[0]++; retCodeDetails[index] = OperationStatus.SUCCESS; } } else if (m instanceof Delete) { Delete curDel = (Delete) m; if (curDel.getFamilyCellMap().isEmpty()) { // handle deleting a row case // TODO: prepareDelete() has been called twice, before and after preDelete() CP hook. // Can this be avoided? region.prepareDelete(curDel); } if (region.coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) { // pre hook says skip this Delete // mark as success and skip in doMiniBatchMutation metrics[1]++; retCodeDetails[index] = OperationStatus.SUCCESS; } } else { String msg = "Put/Delete mutations only supported in a batch"; // In case of passing Append mutations along with the Puts and Deletes in batchMutate // mark the operation return code as failure so that it will not be considered in // the doMiniBatchMutation retCodeDetails[index] = new OperationStatus(OperationStatusCode.FAILURE, msg); if (isAtomic()) { // fail, atomic means all or none throw new IOException(msg); } } }
