/** * Extract the visibility tags of the given Cell into the given List * @param cell - the cell * @param tags - the array that will be populated if visibility tags are present * @return The visibility tags serialization format */ public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) { Byte serializationFormat = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = tag.getBuffer()[tag.getTagOffset()]; } else if (tag.getType() == VISIBILITY_TAG_TYPE) { tags.add(tag); } } } return serializationFormat; }
/** * Extracts and partitions the visibility tags and nonVisibility Tags * * @param cell - the cell for which we would extract and partition the * visibility and non visibility tags * @param visTags * - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would * be added to this list * @param nonVisTags - all the non visibility tags would be added to this list * @return - the serailization format of the tag. Can be null if no tags are found or * if there is no visibility tag found */ public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags, List<Tag> nonVisTags) { Byte serializationFormat = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = tag.getBuffer()[tag.getTagOffset()]; } else if (tag.getType() == VISIBILITY_TAG_TYPE) { visTags.add(tag); } else { // ignore string encoded visibility expressions, will be added in replication handling nonVisTags.add(tag); } } } return serializationFormat; }
protected static void doAssert(byte[] row, String visTag) throws Exception { if (VisibilityReplicationEndPointForTest.lastEntries == null) { return; // first call } Assert.assertEquals(1, VisibilityReplicationEndPointForTest.lastEntries.size()); List<Cell> cells = VisibilityReplicationEndPointForTest.lastEntries.get(0).getEdit().getCells(); Assert.assertEquals(4, cells.size()); boolean tagFound = false; for (Cell cell : cells) { if ((Bytes.equals(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), row, 0, row.length))) { List<Tag> tags = Tag .asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); for (Tag tag : tags) { if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { assertEquals(visTag, Bytes.toString(tag.getValue())); tagFound = true; break; } } } } assertTrue(tagFound); }
/** * This verifies that each cell has a tag that is equal to its rowkey name. For this to work * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to * KeyValueCodecWithTags.class.getCanonicalName()); * @param table table containing tagged cells * @throws IOException if problems reading table */ public static void verifyTags(Table table) throws IOException { ResultScanner s = table.getScanner(new Scan()); for (Result r : s) { for (Cell c : r.listCells()) { byte[] ta = c.getTagsArray(); int toff = c.getTagsOffset(); int tlen = c.getTagsLength(); Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE); if (t == null) { fail(c.toString() + " has null tag"); continue; } byte[] tval = t.getValue(); assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval), r.getRow(), tval); } } }
/** * Tag original sequence number for each edit to be replayed * @param entry * @param cell * @return */ private static Cell tagReplayLogSequenceNumber(WALEntry entry, Cell cell) { // Tag puts with original sequence number if there is no LOG_REPLAY_TAG yet boolean needAddRecoveryTag = true; if (cell.getTagsLength() > 0) { Tag tmpTag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), TagType.LOG_REPLAY_TAG_TYPE); if (tmpTag != null) { // found an existing log replay tag so reuse it needAddRecoveryTag = false; } } if (needAddRecoveryTag) { List<Tag> newTags = new ArrayList<Tag>(); Tag replayTag = new Tag(TagType.LOG_REPLAY_TAG_TYPE, Bytes.toBytes(entry.getKey() .getLogSequenceNumber())); newTags.add(replayTag); return KeyValue.cloneAndAddTags(cell, newTags); } return cell; }
/** * Extracts and partitions the visibility tags and nonVisibility Tags * * @param cell - the cell for which we would extract and partition the * visibility and non visibility tags * @param visTags * - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would * be added to this list * @param nonVisTags - all the non visibility tags would be added to this list * @return - the serailization format of the tag. Can be null if no tags are found or * if there is no visibility tag found */ public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags, List<Tag> nonVisTags) { Byte serializationFormat = null; Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = Tag.getValueAsByte(tag); } else if (tag.getType() == VISIBILITY_TAG_TYPE) { visTags.add(tag); } else { // ignore string encoded visibility expressions, will be added in replication handling nonVisTags.add(tag); } } return serializationFormat; }
public PartitionedMobCompactor(Configuration conf, FileSystem fs, TableName tableName, ColumnFamilyDescriptor column, ExecutorService pool) throws IOException { super(conf, fs, tableName, column, pool); mergeableSize = conf.getLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD); delFileMaxCount = conf.getInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); // default is 100 compactionBatchSize = conf.getInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE); tempPath = new Path(MobUtils.getMobHome(conf), MobConstants.TEMP_DIR_NAME); bulkloadPath = new Path(tempPath, new Path(MobConstants.BULKLOAD_DIR_NAME, new Path( tableName.getNamespaceAsString(), tableName.getQualifierAsString()))); compactionKVMax = this.conf.getInt(HConstants.COMPACTION_KV_MAX, HConstants.COMPACTION_KV_MAX_DEFAULT); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); compactionCacheConfig = new CacheConfig(copyOfConf); List<Tag> tags = new ArrayList<>(2); tags.add(MobConstants.MOB_REF_TAG); Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, tableName.getName()); tags.add(tableNameTag); this.refCellTags = TagUtil.fromList(tags); cryptoContext = EncryptionUtil.createEncryptionContext(copyOfConf, column); }
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam) throws IOException { super(region, family, confParam); this.family = family; this.mobCacheConfig = (MobCacheConfig) cacheConf; this.homePath = MobUtils.getMobHome(conf); this.mobFamilyPath = MobUtils.getMobFamilyPath(conf, this.getTableName(), family.getNameAsString()); List<Path> locations = new ArrayList<>(2); locations.add(mobFamilyPath); TableName tn = region.getTableDescriptor().getTableName(); locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn) .getEncodedName(), family.getNameAsString())); map.put(Bytes.toString(tn.getName()), locations); List<Tag> tags = new ArrayList<>(2); tags.add(MobConstants.MOB_REF_TAG); Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, getTableName().getName()); tags.add(tableNameTag); this.refCellTags = TagUtil.fromList(tags); }
protected static void doAssert(byte[] row, String visTag) throws Exception { if (VisibilityReplicationEndPointForTest.lastEntries == null) { return; // first call } Assert.assertEquals(1, VisibilityReplicationEndPointForTest.lastEntries.size()); List<Cell> cells = VisibilityReplicationEndPointForTest.lastEntries.get(0).getEdit().getCells(); Assert.assertEquals(4, cells.size()); boolean tagFound = false; for (Cell cell : cells) { if ((Bytes.equals(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), row, 0, row.length))) { List<Tag> tags = PrivateCellUtil.getTags(cell); for (Tag tag : tags) { if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { assertEquals(visTag, Tag.getValueAsString(tag)); tagFound = true; break; } } } } assertTrue(tagFound); }
/** * This verifies that each cell has a tag that is equal to its rowkey name. For this to work * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to * KeyValueCodecWithTags.class.getCanonicalName()); * @param table table containing tagged cells * @throws IOException if problems reading table */ public static void verifyTags(Table table) throws IOException { ResultScanner s = table.getScanner(new Scan()); for (Result r : s) { for (Cell c : r.listCells()) { Optional<Tag> tag = PrivateCellUtil.getTag(c, TagType.MOB_TABLE_NAME_TAG_TYPE); if (!tag.isPresent()) { fail(c.toString() + " has null tag"); continue; } Tag t = tag.get(); byte[] tval = Tag.cloneValue(t); assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval), r.getRow(), tval); } } }
/** * Tag original sequence number for each edit to be replayed * @param entry * @param cell */ private static Cell tagReplayLogSequenceNumber(WALEntry entry, Cell cell) { // Tag puts with original sequence number if there is no LOG_REPLAY_TAG yet boolean needAddRecoveryTag = true; if (cell.getTagsLength() > 0) { Tag tmpTag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), TagType.LOG_REPLAY_TAG_TYPE); if (tmpTag != null) { // found an existing log replay tag so reuse it needAddRecoveryTag = false; } } if (needAddRecoveryTag) { List<Tag> newTags = new ArrayList<Tag>(); Tag replayTag = new Tag(TagType.LOG_REPLAY_TAG_TYPE, Bytes.toBytes(entry.getKey() .getLogSequenceNumber())); newTags.add(replayTag); return KeyValue.cloneAndAddTags(cell, newTags); } return cell; }
private void removeReplicationVisibilityTag(List<Tag> tags) throws IOException { Iterator<Tag> iterator = tags.iterator(); while (iterator.hasNext()) { Tag tag = iterator.next(); if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { iterator.remove(); break; } } }
@Override public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException { List<Tag> tags = Lists.newArrayList(); CellVisibility cellVisibility = null; try { cellVisibility = mutation.getCellVisibility(); } catch (DeserializationException e) { throw new IOException(e); } if (cellVisibility == null) { return newCell; } // Prepend new visibility tags to a new list of tags for the cell // Don't check user auths for labels with Mutations when the user is super user boolean authCheck = authorizationEnabled && checkAuths && !(isSystemOrSuperUser()); tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(), true, authCheck)); // Save an object allocation where we can if (newCell.getTagsLength() > 0) { // Carry forward all other tags Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(), newCell.getTagsOffset(), newCell.getTagsLength()); while (tagsItr.hasNext()) { Tag tag = tagsItr.next(); if (tag.getType() != TagType.VISIBILITY_TAG_TYPE && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { tags.add(tag); } } } Cell rewriteCell = new TagRewriteCell(newCell, Tag.fromList(tags)); return rewriteCell; }
/** * @return Carry forward the TTL tag if the increment is carrying one */ private static List<Tag> carryForwardTTLTag(final List<Tag> tagsOrNull, final Mutation mutation) { long ttl = mutation.getTTL(); if (ttl == Long.MAX_VALUE) return tagsOrNull; List<Tag> tags = tagsOrNull; // If we are making the array in here, given we are the last thing checked, // we'll be only thing // in the array so set its size to '1' (I saw this being done in earlier // version of // tag-handling). if (tags == null) tags = new ArrayList<Tag>(1); tags.add(new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl))); return tags; }
@Override public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException { List<Tag> tags = Lists.newArrayList(); CellVisibility cellVisibility = null; try { cellVisibility = mutation.getCellVisibility(); } catch (DeserializationException e) { throw new IOException(e); } if (cellVisibility == null) { return newCell; } // Prepend new visibility tags to a new list of tags for the cell // Don't check user auths for labels with Mutations when the user is super user boolean authCheck = this.checkAuths && !(isSystemOrSuperUser()); tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(), true, authCheck)); // Save an object allocation where we can if (newCell.getTagsLength() > 0) { // Carry forward all other tags Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(), newCell.getTagsOffset(), newCell.getTagsLength()); while (tagsItr.hasNext()) { Tag tag = tagsItr.next(); if (tag.getType() != TagType.VISIBILITY_TAG_TYPE && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { tags.add(tag); } } } Cell rewriteCell = new TagRewriteCell(newCell, Tag.fromList(tags)); return rewriteCell; }
protected void populatePut(byte[] lineBytes, ImportTsv.TsvParser.ParsedLine parsed, Put put, int i) throws BadTsvLineException, IOException { Cell cell = null; if (hfileOutPath == null) { cell = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i)); if (cellVisibilityExpr != null) { // We won't be validating the expression here. The Visibility CP will do // the validation put.setCellVisibility(new CellVisibility(cellVisibilityExpr)); } if (ttl > 0) { put.setTTL(ttl); } } else { // Creating the KV which needs to be directly written to HFiles. Using the Facade // KVCreator for creation of kvs. List<Tag> tags = new ArrayList<Tag>(); if (cellVisibilityExpr != null) { tags.addAll(kvCreator.getVisibilityExpressionResolver() .createVisibilityExpTags(cellVisibilityExpr)); } // Add TTL directly to the KV so we can vary them when packing more than one KV // into puts if (ttl > 0) { tags.add(new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl))); } cell = this.kvCreator.create(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i), tags); } put.add(cell); }
/** * Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE. This * tag type is reserved and should not be explicitly set by user. * * @param cell The cell under consideration * @param pair An optional pair of type {@code <Boolean, Tag>} which would be reused if already * set and new one will be created if NULL is passed * @return If the boolean is false then it indicates that the cell has a RESERVERD_VIS_TAG and * with boolean as true, not null tag indicates that a string modified tag was found. */ private Pair<Boolean, Tag> checkForReservedVisibilityTagPresence(Cell cell, Pair<Boolean, Tag> pair) throws IOException { if (pair == null) { pair = new Pair<>(false, null); } else { pair.setFirst(false); pair.setSecond(null); } // Bypass this check when the operation is done by a system/super user. // This is done because, while Replication, the Cells coming to the peer cluster with reserved // typed tags and this is fine and should get added to the peer cluster table if (isSystemOrSuperUser()) { // Does the cell contain special tag which indicates that the replicated // cell visiblilty tags // have been modified Tag modifiedTag = null; Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { modifiedTag = tag; break; } } pair.setFirst(true); pair.setSecond(modifiedTag); return pair; } Iterator<Tag> tagsItr = PrivateCellUtil.tagsIterator(cell); while (tagsItr.hasNext()) { if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) { return pair; } } pair.setFirst(true); return pair; }
@Override public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException { List<Tag> tags = Lists.newArrayList(); CellVisibility cellVisibility = null; try { cellVisibility = mutation.getCellVisibility(); } catch (DeserializationException e) { throw new IOException(e); } if (cellVisibility == null) { return newCell; } // Prepend new visibility tags to a new list of tags for the cell // Don't check user auths for labels with Mutations when the user is super user boolean authCheck = authorizationEnabled && checkAuths && !(isSystemOrSuperUser()); tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(), true, authCheck)); // Carry forward all other tags Iterator<Tag> tagsItr = PrivateCellUtil.tagsIterator(newCell); while (tagsItr.hasNext()) { Tag tag = tagsItr.next(); if (tag.getType() != TagType.VISIBILITY_TAG_TYPE && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { tags.add(tag); } } Cell rewriteCell = PrivateCellUtil.createCell(newCell, tags); return rewriteCell; }
/** * Extract the visibility tags of the given Cell into the given List * @param cell - the cell * @param tags - the array that will be populated if visibility tags are present * @return The visibility tags serialization format */ public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) { Byte serializationFormat = null; Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = Tag.getValueAsByte(tag); } else if (tag.getType() == VISIBILITY_TAG_TYPE) { tags.add(tag); } } return serializationFormat; }
/** * Whether the current cell is a mob reference cell. * @param cell The current cell. * @return True if the cell has a mob reference tag, false if it doesn't. */ public static boolean isMobReferenceCell(Cell cell) { if (cell.getTagsLength() > 0) { Optional<Tag> tag = PrivateCellUtil.getTag(cell, TagType.MOB_REFERENCE_TAG_TYPE); if (tag.isPresent()) { return true; } } return false; }
/** * Gets the table name tag. * @param cell The current cell. * @return The table name tag. */ public static Tag getTableNameTag(Cell cell) { if (cell.getTagsLength() > 0) { Optional<Tag> tag = PrivateCellUtil.getTag(cell, TagType.MOB_TABLE_NAME_TAG_TYPE); if (tag.isPresent()) { return tag.get(); } } return null; }
/** * Whether the tag list has a mob reference tag. * @param tags The tag list. * @return True if the list has a mob reference tag, false if it doesn't. */ public static boolean hasMobReferenceTag(List<Tag> tags) { if (!tags.isEmpty()) { for (Tag tag : tags) { if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) { return true; } } } return false; }
/** * Create an HFile with the given number of rows between a given * start key and end key @ family:qualifier. * If withTag is true, we add the rowKey as the tag value for * tagtype MOB_TABLE_NAME_TAG_TYPE */ public static void createHFile( Configuration configuration, FileSystem fs, Path path, DataBlockEncoding encoding, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException { HFileContext meta = new HFileContextBuilder() .withIncludesTags(withTag) .withDataBlockEncoding(encoding) .build(); HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration)) .withPath(fs, path) .withFileContext(meta) .create(); long now = System.currentTimeMillis(); try { // subtract 2 since iterateOnSplits doesn't include boundary keys for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) { Cell kv = new KeyValue(key, family, qualifier, now, key); if (withTag) { // add a tag. Arbitrarily chose mob tag since we have a helper already. Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, key); kv = MobUtils.createMobRefCell(kv, key, tableNameTag); // verify that the kv has the tag. Optional<Tag> tag = PrivateCellUtil.getTag(kv, TagType.MOB_TABLE_NAME_TAG_TYPE); if (!tag.isPresent()) { throw new IllegalStateException("Tag didn't stick to KV " + kv.toString()); } } writer.append(kv); } } finally { writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); writer.close(); } }
private void init(Configuration conf, HColumnDescriptor hcd) throws IOException { Path basedir = FSUtils.getRootDir(conf); fs = FileSystem.get(conf); Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR + Bytes.toString(family)); fs.mkdirs(homePath); KeyValue key1 = new KeyValue(row, family, qf1, 1, value); KeyValue key2 = new KeyValue(row, family, qf2, 1, value); KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2); KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; int maxKeyCount = keys.length; StoreFileWriter mobWriter = store.createWriterInTmp(currentDate, maxKeyCount, hcd.getCompactionCompressionType(), region.getRegionInfo().getStartKey(), false); mobFilePath = mobWriter.getPath(); mobWriter.append(key1); mobWriter.append(key2); mobWriter.append(key3); mobWriter.close(); String targetPathName = MobUtils.formatDate(currentDate); byte[] referenceValue = Bytes.toBytes(targetPathName + Path.SEPARATOR + mobFilePath.getName()); Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName().getName()); KeyValue kv1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, referenceValue); KeyValue kv2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, referenceValue); KeyValue kv3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, referenceValue); seekKey1 = MobUtils.createMobRefCell(kv1, referenceValue, tableNameTag); seekKey2 = MobUtils.createMobRefCell(kv2, referenceValue, tableNameTag); seekKey3 = MobUtils.createMobRefCell(kv3, referenceValue, tableNameTag); }
@Override public boolean replicate(ReplicateContext replicateContext) { if (!delegator.canReplicateToSameCluster()) { // Only when the replication is inter cluster replication we need to covert the visibility tags to // string based tags. But for intra cluster replication like region replicas it is not needed. List<Entry> entries = replicateContext.getEntries(); List<Tag> visTags = new ArrayList<Tag>(); List<Tag> nonVisTags = new ArrayList<Tag>(); List<Entry> newEntries = new ArrayList<Entry>(entries.size()); for (Entry entry : entries) { WALEdit newEdit = new WALEdit(); ArrayList<Cell> cells = entry.getEdit().getCells(); for (Cell cell : cells) { if (cell.getTagsLength() > 0) { visTags.clear(); nonVisTags.clear(); Byte serializationFormat = VisibilityUtils.extractAndPartitionTags(cell, visTags, nonVisTags); if (!visTags.isEmpty()) { try { byte[] modifiedVisExpression = visibilityLabelsService .encodeVisibilityForReplication(visTags, serializationFormat); if (modifiedVisExpression != null) { nonVisTags.add(new Tag(TagType.STRING_VIS_TAG_TYPE, modifiedVisExpression)); } } catch (Exception ioe) { LOG.error( "Exception while reading the visibility labels from the cell. The replication " + "would happen as per the existing format and not as string type for the cell " + cell + ".", ioe); // just return the old entries as it is without applying the string type change newEdit.add(cell); continue; } // Recreate the cell with the new tags and the existing tags Cell newCell = new TagRewriteCell(cell, Tag.fromList(nonVisTags)); newEdit.add(newCell); } else { newEdit.add(cell); } } else { newEdit.add(cell); } } newEntries.add(new Entry(entry.getKey(), newEdit)); } replicateContext.setEntries(newEntries); return delegator.replicate(replicateContext); } else { return delegator.replicate(replicateContext); } }
/** * Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE. This * tag type is reserved and should not be explicitly set by user. * * @param cell * - the cell under consideration * @param pair - an optional pair of type <Boolean, Tag> which would be reused * if already set and new one will be created if null is passed * @return a pair<Boolean, Tag> - if the boolean is false then it indicates * that the cell has a RESERVERD_VIS_TAG and with boolean as true, not * null tag indicates that a string modified tag was found. */ private Pair<Boolean, Tag> checkForReservedVisibilityTagPresence(Cell cell, Pair<Boolean, Tag> pair) throws IOException { if (pair == null) { pair = new Pair<Boolean, Tag>(false, null); } else { pair.setFirst(false); pair.setSecond(null); } // Bypass this check when the operation is done by a system/super user. // This is done because, while Replication, the Cells coming to the peer cluster with reserved // typed tags and this is fine and should get added to the peer cluster table if (isSystemOrSuperUser()) { // Does the cell contain special tag which indicates that the replicated // cell visiblilty tags // have been modified Tag modifiedTag = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { modifiedTag = tag; break; } } } pair.setFirst(true); pair.setSecond(modifiedTag); return pair; } if (cell.getTagsLength() > 0) { Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsItr.hasNext()) { if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) { return pair; } } } pair.setFirst(true); return pair; }
@Override public boolean replicate(ReplicateContext replicateContext) { if (!delegator.canReplicateToSameCluster()) { // Only when the replication is inter cluster replication we need to // convert the visibility tags to // string based tags. But for intra cluster replication like region // replicas it is not needed. List<Entry> entries = replicateContext.getEntries(); List<Tag> visTags = new ArrayList<>(); List<Tag> nonVisTags = new ArrayList<>(); List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry entry : entries) { WALEdit newEdit = new WALEdit(); ArrayList<Cell> cells = entry.getEdit().getCells(); for (Cell cell : cells) { if (cell.getTagsLength() > 0) { visTags.clear(); nonVisTags.clear(); Byte serializationFormat = VisibilityUtils.extractAndPartitionTags(cell, visTags, nonVisTags); if (!visTags.isEmpty()) { try { byte[] modifiedVisExpression = visibilityLabelsService .encodeVisibilityForReplication(visTags, serializationFormat); if (modifiedVisExpression != null) { nonVisTags .add(new ArrayBackedTag(TagType.STRING_VIS_TAG_TYPE, modifiedVisExpression)); } } catch (Exception ioe) { LOG.error( "Exception while reading the visibility labels from the cell. The replication " + "would happen as per the existing format and not as " + "string type for the cell " + cell + ".", ioe); // just return the old entries as it is without applying the string type change newEdit.add(cell); continue; } // Recreate the cell with the new tags and the existing tags Cell newCell = PrivateCellUtil.createCell(cell, nonVisTags); newEdit.add(newCell); } else { newEdit.add(cell); } } else { newEdit.add(cell); } } newEntries.add(new Entry((entry.getKey()), newEdit)); } replicateContext.setEntries(newEntries); return delegator.replicate(replicateContext); } else { return delegator.replicate(replicateContext); } }
/** * Convert a line of TSV text into an HBase table row. */ @Override public void map(LongWritable offset, Text value, Context context) throws IOException { byte[] lineBytes = value.getBytes(); try { ImportTsv.TsvParser.ParsedLine parsed = parser.parse( lineBytes, value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength()); // Retrieve timestamp if exists ts = parsed.getTimestamp(ts); cellVisibilityExpr = parsed.getCellVisibility(); ttl = parsed.getCellTTL(); // create tags for the parsed line if (hfileOutPath != null) { tags.clear(); if (cellVisibilityExpr != null) { tags.addAll(kvCreator.getVisibilityExpressionResolver().createVisibilityExpTags( cellVisibilityExpr)); } // Add TTL directly to the KV so we can vary them when packing more than one KV // into puts if (ttl > 0) { tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl))); } } Put put = new Put(rowKey.copyBytes()); for (int i = 0; i < parsed.getColumnCount(); i++) { if (i == parser.getRowKeyColumnIndex() || i == parser.getTimestampKeyColumnIndex() || i == parser.getAttributesKeyColumnIndex() || i == parser.getCellVisibilityColumnIndex() || i == parser.getCellTTLColumnIndex() || (skipEmptyColumns && parsed.getColumnLength(i) == 0)) { continue; } populatePut(lineBytes, parsed, put, i); } context.write(rowKey, put); } catch (ImportTsv.TsvParser.BadTsvLineException | IllegalArgumentException | InvalidLabelException badLine) { if (logBadLines) { System.err.println(value); } System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage()); if (skipBadLines) { incrementBadLineCount(1); return; } throw new IOException(badLine); } catch (InterruptedException e) { e.printStackTrace(); } }
/** * Test that {@link HFileOutputFormat2} RecordWriter writes tags such as ttl into * hfile. */ @Test public void test_WritingTagData() throws Exception { Configuration conf = new Configuration(this.util.getConfiguration()); final String HFILE_FORMAT_VERSION_CONF_KEY = "hfile.format.version"; conf.setInt(HFILE_FORMAT_VERSION_CONF_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); RecordWriter<ImmutableBytesWritable, Cell> writer = null; TaskAttemptContext context = null; Path dir = util.getDataTestDir("WritingTagData"); try { conf.set(HFileOutputFormat2.OUTPUT_TABLE_NAME_CONF_KEY, TABLE_NAMES[0].getNameAsString()); // turn locality off to eliminate getRegionLocation fail-and-retry time when writing kvs conf.setBoolean(HFileOutputFormat2.LOCALITY_SENSITIVE_CONF_KEY, false); Job job = new Job(conf); FileOutputFormat.setOutputPath(job, dir); context = createTestTaskAttemptContext(job); HFileOutputFormat2 hof = new HFileOutputFormat2(); writer = hof.getRecordWriter(context); final byte [] b = Bytes.toBytes("b"); List< Tag > tags = new ArrayList<>(); tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(978670))); KeyValue kv = new KeyValue(b, b, b, HConstants.LATEST_TIMESTAMP, b, tags); writer.write(new ImmutableBytesWritable(), kv); writer.close(context); writer = null; FileSystem fs = dir.getFileSystem(conf); RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(dir, true); while(iterator.hasNext()) { LocatedFileStatus keyFileStatus = iterator.next(); HFile.Reader reader = HFile.createReader(fs, keyFileStatus.getPath(), new CacheConfig(conf), true, conf); HFileScanner scanner = reader.getScanner(false, false, false); scanner.seekTo(); Cell cell = scanner.getCell(); List<Tag> tagsFromCell = PrivateCellUtil.getTags(cell); assertTrue(tagsFromCell.size() > 0); for (Tag tag : tagsFromCell) { assertTrue(tag.getType() == TagType.TTL_TAG_TYPE); } } } finally { if (writer != null && context != null) writer.close(context); dir.getFileSystem(conf).delete(dir, true); } }
