public void testAddIndexForTable() throws Exception { IndexManager im = IndexManager.getInstance(); assertNotNull("Index Manager should not be null.", im); List<IndexSpecification> indexList = new ArrayList<IndexSpecification>(1); IndexSpecification iSpec = new IndexSpecification("index_name"); iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 10); indexList.add(iSpec); im.addIndexForTable("index_name", indexList); indexList = im.getIndicesForTable("index_name"); assertEquals("Index name should be equal with actual value.", "index_name", indexList.get(0) .getName()); assertTrue("Column qualifier state mismatch.", indexList.get(0).getIndexColumns().contains(new ColumnQualifier("cf", "cq", null, 10))); }
public void testShouldNotThrowNPEIfValueTypeIsNull() throws Exception { IndexManager im = IndexManager.getInstance(); assertNotNull("Index Manager should not be null.", im); List<IndexSpecification> indexList = new ArrayList<IndexSpecification>(1); IndexSpecification iSpec = new IndexSpecification("index_name"); iSpec.addIndexColumn(new HColumnDescriptor("cf"), "cq", null, 5); indexList.add(iSpec); im.addIndexForTable("index_name", indexList); indexList = im.getIndicesForTable("index_name"); Set<ColumnQualifier> indexColumns = indexList.get(0).getIndexColumns(); for (ColumnQualifier columnQualifier : indexColumns) { assertNotNull(columnQualifier.getType()); } }
private static Multimap<Long, Cell> doGetAndGroupByTS(List<IndexSpecification> indexSpecs, HRegion userRegion, Cell deleteKV, List<IndexSpecification> indicesToConsider) throws IOException { Get get = new Get(deleteKV.getRow()); long maxTS = HConstants.LATEST_TIMESTAMP; if (deleteKV.getTimestamp() < maxTS) { // Add +1 to make the current get includes the timestamp maxTS = deleteKV.getTimestamp() + 1; } get.setTimeRange(0L, maxTS); for (IndexSpecification index : indexSpecs) { // Get all indices involves this family/qualifier if (index.contains(deleteKV.getFamily(), deleteKV.getQualifier())) { indicesToConsider.add(index); for (ColumnQualifier cq : index.getIndexColumns()) { get.addColumn(cq.getColumnFamily(), cq.getQualifier()); } } } if (((KeyValue) deleteKV).isDeleteType()) { get.setMaxVersions(1); } else if (((KeyValue) deleteKV).isDeleteColumnOrFamily()) { get.setMaxVersions(); } List<KeyValue> userKVs = userRegion.get(get).list(); // Group KV based on timestamp Multimap<Long, Cell> groupedKV = HashMultimap.create(); if (userKVs != null) { for (Cell userKV : userKVs) { groupedKV.put(userKV.getTimestamp(), userKV); } } return groupedKV; }
private boolean isIndexSuitable(IndexSpecification index, List<Column> cols, Map<Column, LeafFilterNode> leafNodes) { int matchedCols = 0; for (ColumnQualifier cq : index.getIndexColumns()) { Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); if (cols.contains(column)) { matchedCols++; // leafNodes.get(column) will never be null.. Don't worry if (leafNodes.get(column).getFilterColumnValueDetail() instanceof FilterColumnValueRange) { // When the condition on the column is a range condition, we need to ensure in this index // 1. The column is the last column // or // 2. There are no columns in this index which is part of the cols list if (matchedCols != cols.size()) { return false; } } } else { if (matchedCols != cols.size()) { return false; } } if (matchedCols == cols.size()) { return true; } } return false; }
public static Delete prepareIndexDelete(Delete userDelete, IndexSpecification index, byte[] indexRegionStartKey) throws IOException { ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegionStartKey, userDelete.getRow()); boolean update = false; for (ColumnQualifier cq : index.getIndexColumns()) { Cell kvFound = null; for (Entry<byte[], List<Cell>> entry : userDelete.getFamilyCellMap().entrySet()) { for (Cell cell : entry.getValue()) { Cell kv = KeyValueUtil.ensureKeyValue(cell); if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { kvFound = kv; update = true; break; } } } if (kvFound == null) { indexRow.position(indexRow.position() + cq.getMaxValueLength()); } else { IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); } } if (update) { // Append the actual row key at the end of the index row key. indexRow.put(userDelete.getRow()); Delete idxDelete = new Delete(indexRow.array()); idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, userDelete.getTimeStamp()); idxDelete.setDurability(Durability.SKIP_WAL); return idxDelete; } return null; }
public static void updateRowKeyForKV(ColumnQualifier indexCQ, Cell kv, ByteArrayBuilder indexRowKey) throws IOException { byte[] value = getValueFromKV(kv, indexCQ); int valuePadLength = indexCQ.getMaxValueLength() - value.length; if (valuePadLength < 0) { String errMsg = "The value length for the column " + indexCQ.getColumnFamilyString() + ":" + indexCQ.getQualifierString() + " is greater than the cofigured max value length : " + indexCQ.getMaxValueLength(); LOG.warn(errMsg); throw new IOException(errMsg); } indexRowKey.put(value); indexRowKey.position(indexRowKey.position() + valuePadLength); }
private static byte[] getValueFromKV(Cell kv, ColumnQualifier indexCQ) { ValuePartition vp = indexCQ.getValuePartition(); byte value[] = null; if (vp != null) { value = vp.getPartOfValue(kv.getValue()); if (value != null) { value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); } } else { LOG.trace("No offset or separator is mentioned. So just returning the value fetched from kv"); value = kv.getValue(); value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); } return value; }
@Test(timeout = 180000) public void testColumnQualifierSerialization() throws Exception { ByteArrayOutputStream bos = null; DataOutputStream dos = null; ByteArrayInputStream bis = null; DataInputStream dis = null; try { bos = new ByteArrayOutputStream(); dos = new DataOutputStream(bos); ColumnQualifier cq = new ColumnQualifier("cf", "cq", ValueType.String, 10, new SpatialPartition(0, 5)); cq.write(dos); dos.flush(); byte[] byteArray = bos.toByteArray(); bis = new ByteArrayInputStream(byteArray); dis = new DataInputStream(bis); ColumnQualifier c = new ColumnQualifier(); c.readFields(dis); assertTrue("ColumnQualifier state mismatch.", c.equals(cq)); } finally { if (null != bos) { bos.close(); } if (null != dos) { dos.close(); } if (null != bis) { bis.close(); } if (null != dis) { dis.close(); } } }
private void checkColumnsForValidityAndConsistency(HTableDescriptor desc, IndexSpecification iSpec, Map<Column, Pair<ValueType, Integer>> indexColDetails) throws IOException { for (ColumnQualifier cq : iSpec.getIndexColumns()) { if (null == desc.getFamily(cq.getColumnFamily())) { String message = "Column family " + cq.getColumnFamilyString() + " in index specification " + iSpec.getName() + " not in Column families of table " + desc.getNameAsString() + '.'; LOG.error(message); IllegalArgumentException ie = new IllegalArgumentException(message); throw new IOException(ie); } Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition()); ValueType type = cq.getType(); int maxlength = cq.getMaxValueLength(); Pair<ValueType, Integer> colDetail = indexColDetails.get(column); if (null != colDetail) { if (!colDetail.getFirst().equals(type) || colDetail.getSecond() != maxlength) { throw new IOException("ValueType/max value length of column " + column + " not consistent across the indices"); } } else { indexColDetails.put(column, new Pair<ValueType, Integer>(type, maxlength)); } } }
private static Multimap<Long, KeyValue> doGetAndGroupByTS(List<IndexSpecification> indexSpecs, HRegion userRegion, KeyValue deleteKV, List<IndexSpecification> indicesToConsider) throws IOException { Get get = new Get(deleteKV.getRow()); long maxTS = HConstants.LATEST_TIMESTAMP; if (deleteKV.getTimestamp() < maxTS) { // Add +1 to make the current get includes the timestamp maxTS = deleteKV.getTimestamp() + 1; } get.setTimeRange(HConstants.OLDEST_TIMESTAMP, maxTS); for (IndexSpecification index : indexSpecs) { // Get all indices involves this family/qualifier if (index.contains(deleteKV.getFamily(), deleteKV.getQualifier())) { indicesToConsider.add(index); for (ColumnQualifier cq : index.getIndexColumns()) { get.addColumn(cq.getColumnFamily(), cq.getQualifier()); } } } if (deleteKV.isDeleteType()) { get.setMaxVersions(1); } else if (deleteKV.isDeleteColumnOrFamily()) { get.setMaxVersions(); } List<KeyValue> userKVs = userRegion.get(get, 0).list(); // Group KV based on timestamp Multimap<Long, KeyValue> groupedKV = HashMultimap.create(); if (userKVs != null) { for (KeyValue userKV : userKVs) { groupedKV.put(userKV.getTimestamp(), userKV); } } return groupedKV; }
public static Delete prepareIndexDelete(Delete userDelete, IndexSpecification index, byte[] indexRegionStartKey) throws IOException { ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegionStartKey, userDelete.getRow()); boolean update = false; for (ColumnQualifier cq : index.getIndexColumns()) { KeyValue kvFound = null; for (Entry<byte[], List<KeyValue>> entry : userDelete.getFamilyMap().entrySet()) { for (KeyValue kv : entry.getValue()) { if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { kvFound = kv; update = true; break; } } } if (kvFound == null) { indexRow.position(indexRow.position() + cq.getMaxValueLength()); } else { IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); } } if (update) { // Append the actual row key at the end of the index row key. indexRow.put(userDelete.getRow()); Delete idxDelete = new Delete(indexRow.array()); idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, userDelete.getTimeStamp()); idxDelete.setWriteToWAL(false); return idxDelete; } return null; }
public static void updateRowKeyForKV(ColumnQualifier indexCQ, KeyValue kv, ByteArrayBuilder indexRowKey) throws IOException { byte[] value = getValueFromKV(kv, indexCQ); int valuePadLength = indexCQ.getMaxValueLength() - value.length; if (valuePadLength < 0) { String errMsg = "The value length for the column " + indexCQ.getColumnFamilyString() + ":" + indexCQ.getQualifierString() + " is greater than the cofigured max value length : " + indexCQ.getMaxValueLength(); LOG.warn(errMsg); throw new IOException(errMsg); } indexRowKey.put(value); indexRowKey.position(indexRowKey.position() + valuePadLength); }
private static byte[] getValueFromKV(KeyValue kv, ColumnQualifier indexCQ) { ValuePartition vp = indexCQ.getValuePartition(); byte value[] = null; if (vp != null) { value = vp.getPartOfValue(kv.getValue()); if (value != null) { value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); } } else { LOG.trace("No offset or separator is mentioned. So just returning the value fetched from kv"); value = kv.getValue(); value = IndexUtils.changeValueAccToDataType(value, indexCQ.getType()); } return value; }
private static Collection<Delete> getIndexDeletes(List<IndexSpecification> indexSpecs, HRegion userRegion, HRegion indexRegion, Cell deleteKV) throws IOException { Collection<Delete> indexDeletes = new LinkedHashSet<Delete>(); List<IndexSpecification> indicesToUpdate = new LinkedList<IndexSpecification>(); Multimap<Long, Cell> groupedKV = doGetAndGroupByTS(indexSpecs, userRegion, deleteKV, indicesToUpdate); // There can be multiple index kvs for each user kv // So, prepare all resultant index delete kvs for this user delete kv for (Entry<Long, Collection<Cell>> entry : groupedKV.asMap().entrySet()) { for (IndexSpecification index : indicesToUpdate) { ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegion.getStartKey(), deleteKV.getRow()); boolean update = false; for (ColumnQualifier cq : index.getIndexColumns()) { Cell kvFound = null; for (Cell kv : entry.getValue()) { if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { kvFound = kv; update = true; break; } } if (kvFound == null) { indexRow.position(indexRow.position() + cq.getMaxValueLength()); } else { IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); } } if (update) { // Append the actual row key at the end of the index row key. indexRow.put(deleteKV.getRow()); Delete idxDelete = new Delete(indexRow.array()); if (((KeyValue) deleteKV).isDeleteType()) { idxDelete .deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, entry.getKey()); } else { idxDelete.deleteFamily(Constants.IDX_COL_FAMILY, entry.getKey()); } indexDeletes.add(idxDelete); } } } return indexDeletes; }
private static Collection<Delete> getIndexDeletes(List<IndexSpecification> indexSpecs, HRegion userRegion, HRegion indexRegion, KeyValue deleteKV) throws IOException { Collection<Delete> indexDeletes = new LinkedHashSet<Delete>(); List<IndexSpecification> indicesToUpdate = new LinkedList<IndexSpecification>(); Multimap<Long, KeyValue> groupedKV = doGetAndGroupByTS(indexSpecs, userRegion, deleteKV, indicesToUpdate); // There can be multiple index kvs for each user kv // So, prepare all resultant index delete kvs for this user delete kv for (Entry<Long, Collection<KeyValue>> entry : groupedKV.asMap().entrySet()) { for (IndexSpecification index : indicesToUpdate) { ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegion.getStartKey(), deleteKV.getRow()); boolean update = false; for (ColumnQualifier cq : index.getIndexColumns()) { KeyValue kvFound = null; for (KeyValue kv : entry.getValue()) { if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { kvFound = kv; update = true; break; } } if (kvFound == null) { indexRow.position(indexRow.position() + cq.getMaxValueLength()); } else { IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); } } if (update) { // Append the actual row key at the end of the index row key. indexRow.put(deleteKV.getRow()); Delete idxDelete = new Delete(indexRow.array()); if (deleteKV.isDeleteType()) { idxDelete .deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, entry.getKey()); } else { idxDelete.deleteFamily(Constants.IDX_COL_FAMILY, entry.getKey()); } idxDelete.setWriteToWAL(false); indexDeletes.add(idxDelete); } } } return indexDeletes; }
