private void generateIRIndexCandidateRows(Scan scan) throws IOException { Set<ByteArray> indexHeap = new HashSet<ByteArray>(10000); printIndexTree(this.indexTree); indexHeap = readAndMergeIndex(indexTree, scan, indexHeap, Operator.MUST_PASS_ONE); if (!indexHeap.isEmpty()) { // here the values are already filtered! candidateIRIndexRows = new ArrayDeque<byte[]>(indexHeap.size() + 1); ByteArray[] heap = indexHeap.toArray(new ByteArray[indexHeap.size()]); long sortStartTime = System.currentTimeMillis(); // sort here! QuickSort.sort(heap, ByteArray.BAC); this.indexSortTime = System.currentTimeMillis() - sortStartTime; long mergeStartTime = System.currentTimeMillis(); byte[][] byteHeap = new byte[heap.length][]; for (int i = 0; i < heap.length; i++) { byteHeap[i] = heap[i].getByteArray(); } this.indexMergeTime = System.currentTimeMillis() - mergeStartTime; for (byte[] tmp : byteHeap) { candidateIRIndexRows.add(tmp); } } else { LOG.info("LCDBG, index heap is empty"); } }
@Test public void testAddFilter() throws Exception { Filter filter1 = new FirstKeyOnlyFilter(); Filter filter2 = new FirstKeyOnlyFilter(); FilterList filterList = new FilterList(filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); }
/** * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters * we expect to get the same result as the inclusive stop result. * @throws Exception */ public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new InclusiveStopFilter(r1)); flist.filterRowKey(r1, 0, r1.length); assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE); assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE); flist.reset(); flist.filterRowKey(r2, 0, r2.length); assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP); }
/** * Test serialization * @throws Exception */ @Test public void testSerialization() throws Exception { List<Filter> filters = new ArrayList<Filter>(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); // Decompose filterMPALL to bytes. byte[] buffer = filterMPALL.toByteArray(); // Recompose filterMPALL. FilterList newFilter = FilterList.parseFrom(buffer); // Run tests mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE()))); mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter()))); orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter()))); }
private FilterList getColumnValueFilters(Row row) { FilterList filterList = new FilterList(Operator.MUST_PASS_ALL); Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames()); for (Map.Entry<String, ColumnDef> column : columns.entrySet()) { if (!column.getValue().cf.equals("rowkey")) { if (filterColumnNames.contains(column.getKey())) { byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row); if (value != null) { SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter( Bytes.toBytes(column.getValue().cf), Bytes.toBytes(column.getValue().name), CompareFilter.CompareOp.EQUAL, value ); filterList.addFilter(columnValueFilter); } } } } return filterList; }
private void generateCandidateRows(Scan scan) throws IOException { Set<ByteArray> indexHeap = new HashSet<ByteArray>(10000); printIndexTree(this.indexTree); indexHeap = readAndMergeIndex(indexTree, scan, indexHeap, Operator.MUST_PASS_ONE); if (!indexHeap.isEmpty()) { // here the values are already filtered! candidateRows = new ArrayDeque<byte[]>(indexHeap.size() + 1); ByteArray[] heap = indexHeap.toArray(new ByteArray[indexHeap.size()]); long sortStartTime = System.currentTimeMillis(); // winter sort here! QuickSort.sort(heap, ByteArray.BAC); this.indexSortTime = System.currentTimeMillis() - sortStartTime; long mergeStartTime = System.currentTimeMillis(); byte[][] byteHeap = new byte[heap.length][]; for (int i = 0; i < heap.length; i++) { byteHeap[i] = heap[i].getByteArray(); } this.indexMergeTime = System.currentTimeMillis() - mergeStartTime; for (byte[] tmp : byteHeap) { candidateRows.add(tmp); } } }
/** * Test serialization * @throws Exception */ public void testSerialization() throws Exception { List<Filter> filters = new ArrayList<Filter>(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); // Decompose filterMPALL to bytes. ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(stream); filterMPALL.write(out); out.close(); byte[] buffer = stream.toByteArray(); // Recompose filterMPALL. DataInputStream in = new DataInputStream(new ByteArrayInputStream(buffer)); FilterList newFilter = new FilterList(); newFilter.readFields(in); // TODO: Run TESTS!!! }
private Filter handleFilterWithinAND(Filter filter) { if (filter instanceof FilterList) { FilterList fList = (FilterList) filter; if (fList.getOperator() == Operator.MUST_PASS_ONE) { return new FilterGroupingWorker().group(fList); } else { List<Filter> filters = fList.getFilters(); for (Filter subFilter : filters) { handleFilterWithinAND(subFilter); } } } else if (filter instanceof SingleColumnValueFilter) { handleScvf((SingleColumnValueFilter) filter); } // TODO when we expose SingleColumnRangeFilter to handle that also here. return null; }
FilterNode evalFilterForIndexSelection(Filter filter, List<IndexSpecification> indices) { if (filter instanceof FilterList) { FilterList fList = (FilterList) filter; GroupingCondition condition = (fList.getOperator() == Operator.MUST_PASS_ALL) ? GroupingCondition.AND : GroupingCondition.OR; NonLeafFilterNode nonLeafFilterNode = new NonLeafFilterNode(condition); List<Filter> filters = fList.getFilters(); for (Filter fltr : filters) { FilterNode node = evalFilterForIndexSelection(fltr, indices); nonLeafFilterNode.addFilterNode(node); } return handleNonLeafFilterNode(nonLeafFilterNode); } else if (filter instanceof SingleColumnValueFilter) { // Check for the availability of index return selectBestFitAndPossibleIndicesForSCVF(indices, (SingleColumnValueFilter) filter); } else if (filter instanceof SingleColumnRangeFilter) { return selectBestFitAndPossibleIndicesForSCRF(indices, (SingleColumnRangeFilter) filter); } return new NoIndexFilterNode(); }
@Test(timeout = 180000) public void testShouldRetrieveNegativeIntValue() throws Exception { Configuration conf = UTIL.getConfiguration(); String userTableName = "testShouldRetrieveNegativeIntValue"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); rangePutForIdx2WithInteger(table); FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); SingleColumnValueFilter scvf = new SingleColumnValueFilter("cf1".getBytes(), "c1".getBytes(), CompareOp.GREATER, new IntComparator(Bytes.toBytes(-6))); masterFilter.addFilter(scvf); Scan scan = new Scan(); scan.setFilter(masterFilter); ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); result = scanner.next(1); } assertTrue(testRes.size() == 5); }
private static Filter constructFilterForContain( HBaseColumnSchema hbaseColumnSchema, CompareOp compareOp, List<Object> list, Operator operator) { Util.checkNull(hbaseColumnSchema); Util.checkNull(compareOp); Util.checkNull(list); Util.checkNull(operator); List<Filter> filters = new ArrayList<Filter>(); for (Object obj : list) { filters.add(constructFilter(hbaseColumnSchema, compareOp, obj)); } FilterList filterList = new FilterList(operator, filters); return filterList; }
@Override public Filter visitNotbetweenconstant(NotbetweenconstantContext ctx) { CidContext cidContext = ctx.cid(); HBaseColumnSchema hbaseColumnSchema = ContextUtil .parseHBaseColumnSchema(hbaseTableConfig, cidContext); List<ConstantContext> constantContextList = ctx.constant(); List<Object> list = ContextUtil.parseConstantList(hbaseColumnSchema, constantContextList, runtimeSetting); Filter startFilter = constructFilter(hbaseColumnSchema, CompareOp.LESS, list.get(0)); Filter endFilter = constructFilter(hbaseColumnSchema, CompareOp.GREATER, list.get(1)); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, Arrays.asList(startFilter, endFilter)); return filterList; }
@Override public Filter visitNotbetweenvar(NotbetweenvarContext ctx) { CidContext cidContext = ctx.cid(); List<VarContext> varContextList = ctx.var(); HBaseColumnSchema hbaseColumnSchema = ContextUtil .parseHBaseColumnSchema(hbaseTableConfig, cidContext); List<Object> list = ContextUtil.parseParaList(varContextList, para); Filter startFilter = constructFilter(hbaseColumnSchema, CompareOp.LESS, list.get(0)); Filter endFilter = constructFilter(hbaseColumnSchema, CompareOp.GREATER, list.get(1)); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, Arrays.asList(startFilter, endFilter)); return filterList; }
@Override public Filter visitBetweenvar(BetweenvarContext ctx) { CidContext cidContext = ctx.cid(); List<VarContext> varContextList = ctx.var(); HBaseColumnSchema hbaseColumnSchema = ContextUtil .parseHBaseColumnSchema(hbaseTableConfig, cidContext); List<Object> list = ContextUtil.parseParaList(varContextList, para); Filter startFilter = constructFilter(hbaseColumnSchema, CompareOp.GREATER_OR_EQUAL, list.get(0)); Filter endFilter = constructFilter(hbaseColumnSchema, CompareOp.LESS_OR_EQUAL, list.get(1)); FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(startFilter, endFilter)); return filterList; }
@Override public Filter visitBetweenconstant(BetweenconstantContext ctx) { CidContext cidContext = ctx.cid(); HBaseColumnSchema hbaseColumnSchema = ContextUtil .parseHBaseColumnSchema(hbaseTableConfig, cidContext); List<ConstantContext> constantContextList = ctx.constant(); List<Object> list = ContextUtil.parseConstantList(hbaseColumnSchema, constantContextList, runtimeSetting); Filter startFilter = constructFilter(hbaseColumnSchema, CompareOp.GREATER_OR_EQUAL, list.get(0)); Filter endFilter = constructFilter(hbaseColumnSchema, CompareOp.LESS_OR_EQUAL, list.get(1)); FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(startFilter, endFilter)); return filterList; }
private void generateCandidateRows(Scan scan) throws IOException { Set<ByteArray> indexHeap = new HashSet<ByteArray>(10000); printIndexTree(this.indexTree); indexHeap = readAndMergeIndex(indexTree, scan, indexHeap, Operator.MUST_PASS_ONE); if (!indexHeap.isEmpty()) { candidateRows = new ArrayDeque<byte[]>(indexHeap.size() + 1); ByteArray[] heap = indexHeap.toArray(new ByteArray[indexHeap.size()]); long sortStartTime=System.currentTimeMillis(); QuickSort.sort(heap, ByteArray.BAC); this.indexSortTime=System.currentTimeMillis()-sortStartTime; long mergeStartTime=System.currentTimeMillis(); byte[][] byteHeap = new byte[heap.length][]; for (int i = 0; i < heap.length; i++) { byteHeap[i] = heap[i].getByteArray(); } this.indexMergeTime=System.currentTimeMillis()-mergeStartTime; for (byte[] tmp : byteHeap) { candidateRows.add(tmp); } } }
@Test public void testGetWithRowkeyFilter() throws IOException { commitColumnsWithDifferentTs(); Get get = createGetForDifferentTs(); get.setFilter(new PrefixFilter(ROW)); Result iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); get.setFilter(new PrefixFilter(ANOTHER_ROW)); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); FilterList filterList = new FilterList(); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); }
@Test public void testScanWithFilter() throws IOException { prepareScanData(TRANSACTION_COLUMNS); writeData(COLUMN, lastTs(prewriteTs), ANOTHER_VALUE); ValueFilter valueFilter = new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE)); PrefixFilter prefixFilter = new PrefixFilter(ANOTHER_ROW); FilterList filterList = new FilterList(); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); ThemisScanner scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkAndCloseScanner(scanner); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkScanRow(new ColumnCoordinate[]{COLUMN_WITH_ANOTHER_ROW}, scanner.next()); Assert.assertEquals(1, scanner.next().size()); checkAndCloseScanner(scanner); }
/** * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters * we expect to get the same result as the inclusive stop result. * @throws Exception */ @Test public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new InclusiveStopFilter(r1)); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1)); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1))); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2)); assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2))); }
/** * Test serialization * @throws Exception */ @Test public void testSerialization() throws Exception { List<Filter> filters = new ArrayList<>(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); // Decompose filterMPALL to bytes. byte[] buffer = filterMPALL.toByteArray(); // Recompose filterMPALL. FilterList newFilter = FilterList.parseFrom(buffer); // Run tests mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE()))); mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter()))); orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter()))); }
@Test public void testWithMultiVersionsInSameRow() throws Exception { FilterList filterList01 = new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0)); KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"), 1, Bytes.toBytes("value")); KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"), 2, Bytes.toBytes("value")); KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"), 3, Bytes.toBytes("value")); assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterCell(kv1)); assertEquals(ReturnCode.SKIP, filterList01.filterCell(kv2)); assertEquals(ReturnCode.SKIP, filterList01.filterCell(kv3)); FilterList filterList11 = new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1)); assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv1)); assertEquals(ReturnCode.SKIP, filterList11.filterCell(kv2)); assertEquals(ReturnCode.SKIP, filterList11.filterCell(kv3)); }
@Test public void testMPONEWithSeekNextUsingHint() throws Exception { byte[] col = Bytes.toBytes("c"); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col)); KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, Bytes.toBytes("value")); KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2, Bytes.toBytes("value")); KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3, Bytes.toBytes("value")); KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4, Bytes.toBytes("value")); assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1)); assertEquals(ReturnCode.SKIP, filterList.filterCell(kv2)); assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3)); assertEquals(ReturnCode.SKIP, filterList.filterCell(kv4)); }