public Iterable<Edge> getEdges(String key, Object value) { try { Scan scan = new Scan(Bytes.toBytes(this.graphName+"_"+key), this.getEndKey(Bytes.toBytes(this.graphName+"_"+key))); scan.addFamily(Bytes.toBytes("edge")); scan.setFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(String.valueOf(value))))); return new HbaseIterable<Edge>(scan, HbaseGraphUtils.PROPERTIESTABLENAME, hbaseConf) { @Override public Edge next(Result result) { for(Cell cell : result.listCells()) { String id = Bytes.toString(CellUtil.cloneQualifier(cell)); return new HbaseEdge(id, graphName, hbaseConf); } return null; } }; } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); return null; } }
public Iterable<Vertex> getVertices(String key, Object value) { try { Scan scan = new Scan(Bytes.toBytes(this.graphName+"_"+key), this.getEndKey(Bytes.toBytes(this.graphName+"_"+key))); scan.addFamily(Bytes.toBytes("vertex")); scan.setFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(String.valueOf(value))))); return new HbaseIterable<Vertex>(scan, HbaseGraphUtils.PROPERTIESTABLENAME, hbaseConf) { @Override public Vertex next(Result result) { for(Cell cell : result.listCells()) { String id = Bytes.toString(CellUtil.cloneQualifier(cell)); return new HbaseVertex(id, graphName, hbaseConf); } return null; } }; } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); return null; } }
@Test public void testScanWithFilter() throws IOException { prepareScanData(TRANSACTION_COLUMNS); writeData(COLUMN, lastTs(prewriteTs), ANOTHER_VALUE); ValueFilter valueFilter = new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE)); PrefixFilter prefixFilter = new PrefixFilter(ANOTHER_ROW); FilterList filterList = new FilterList(); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); ThemisScanner scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkAndCloseScanner(scanner); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkScanRow(new ColumnCoordinate[]{COLUMN_WITH_ANOTHER_ROW}, scanner.next()); Assert.assertEquals(1, scanner.next().size()); checkAndCloseScanner(scanner); }
@Override public RowFilter adapt(FilterAdapterContext context, ValueFilter filter) throws IOException { if (filter.getComparator() instanceof BinaryComparator) { return adaptBinaryComparator( filter.getOperator(), (BinaryComparator) filter.getComparator()); } else if (filter.getComparator() instanceof RegexStringComparator) { return adaptRegexStringComparator( filter.getOperator(), (RegexStringComparator) filter.getComparator()); } throw new IllegalStateException( String.format("Cannot adapt filter with comparator%s", filter.getComparator())); }
@Override public FilterSupportStatus isFilterSupported( FilterAdapterContext context, ValueFilter filter) { if (filter.getComparator() instanceof BinaryComparator || (filter.getComparator() instanceof RegexStringComparator && filter.getOperator() == CompareOp.EQUAL)) { return FilterSupportStatus.SUPPORTED; } return FilterSupportStatus.newNotSupported( String.format( "ValueFilter must have either a BinaryComparator with any compareOp " + "or a RegexStringComparator with an EQUAL compareOp. Found (%s, %s)", filter.getComparator().getClass().getSimpleName(), filter.getOperator())); }
private void assertAdaptedForm( ByteArrayComparable comparable, CompareFilter.CompareOp op, RowFilter expectedFilter) throws IOException { ValueFilter filter = new ValueFilter(op, comparable); RowFilter actualFilter = adapter.adapt(emptyScanContext, filter); Assert.assertEquals(expectedFilter, actualFilter); }
@Test public void testValueFilter() throws IOException { // Initialize int numGoodCols = 5; int numBadCols = 20; String goodValue = "includeThisValue"; Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testRow-"); Put put = new Put(rowKey); for (int i = 0; i < numBadCols; ++i) { put.addColumn(COLUMN_FAMILY, dataHelper.randomData(""), Bytes.toBytes("someval")); } for (int i = 0; i < numGoodCols; ++i) { put.addColumn(COLUMN_FAMILY, dataHelper.randomData(""), Bytes.toBytes(goodValue)); } table.put(put); // Filter for results Filter filter = new ValueFilter( CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(goodValue))); Get get = new Get(rowKey).setFilter(filter); Result result = table.get(get); Assert.assertEquals("Should only return good values", numGoodCols, result.size()); Cell[] cells = result.rawCells(); for (Cell cell : cells) { Assert.assertTrue("Should have good value", Bytes.toString(CellUtil.cloneValue(cell)).startsWith(goodValue)); } table.close(); }
private void loadCache(boolean reverse) { if (cachedRows != null && cachedRows.hasNext()) return; // There are more rows so return and the code will return // the next result from cache if (needToGetBatch) { if (batchListener != null) batchListener.beforeFetchingNextBatch(); Result[] result = null; byte[] family = Bytes.toBytes(indexTableName); List<Get> listGet = new ArrayList<Get>(); for (byte[] key : values) { Get get = new Get(rowKey); CompareFilter.CompareOp equal = CompareOp.EQUAL; if (key != null) { BinaryComparator valueComparator = createComparaor(key); ValueFilter filter = new ValueFilter(equal, valueComparator); get.setFilter(filter); get.addFamily(family); listGet.add(get); } } try { result = htable.get(listGet); } catch (IOException e) { e.printStackTrace(); } fillCache(result); needToGetBatch = false; if (reverse) { while (cachedRows.hasNext()) cachedRows.next(); } } }
@Test public void testFilterList() throws Exception { // Test getting a single row, single key using Row, Qualifier, and Value // regular expression and substring filters // Use must pass all List<Filter> filters = new ArrayList<Filter>(); filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One"))); Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); Scan s = new Scan(); s.addFamily(FAMILIES[0]); s.setFilter(f); KeyValue [] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) }; verifyScanFull(s, kvs); // Test getting everything with a MUST_PASS_ONE filter including row, qf, // val, regular expression and substring filters filters.clear(); filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+Two.+"))); filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One"))); f = new FilterList(Operator.MUST_PASS_ONE, filters); s = new Scan(); s.setFilter(f); verifyScanNoEarlyOut(s, numRows, colsPerRow); }
@Test public void testGetWithFilter() throws IOException { // rowkey filter pass while column filter not pass commitColumnsWithDifferentTs(); Get get = createGetForDifferentTs(); FilterList filterList = new FilterList(); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); get.setFilter(filterList); Result iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); // rowkey filter not pass while column filter pass filterList = new FilterList(); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(VALUE))); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(VALUE))); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); }
@Test public void testFilterList() throws Exception { // Test getting a single row, single key using Row, Qualifier, and Value // regular expression and substring filters // Use must pass all List<Filter> filters = new ArrayList<>(3); filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); Scan s = new Scan(); s.addFamily(FAMILIES[0]); s.setFilter(f); KeyValue [] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) }; verifyScanFull(s, kvs); // Test getting everything with a MUST_PASS_ONE filter including row, qf, // val, regular expression and substring filters filters.clear(); filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+"))); filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); f = new FilterList(Operator.MUST_PASS_ONE, filters); s = new Scan(); s.setFilter(f); verifyScanNoEarlyOut(s, numRows, colsPerRow); }
void addFilterByMapping( FilterList fl, CompareFilter.CompareOp comp, Class<?> comparatorClass, Object comparator, Mapping.TupleMapping tupleMapping ) throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException { switch ( tupleMapping ) { case KEY: { addFilter( RowFilter.class, fl, comp, comparatorClass, comparator ); return; } case FAMILY: { addFilter( FamilyFilter.class, fl, comp, comparatorClass, comparator ); return; } case COLUMN: { //TODO Check if ColumnPrefixFilter works faster and suit more addFilter( QualifierFilter.class, fl, comp, comparatorClass, comparator ); return; } case VALUE: { addFilter( ValueFilter.class, fl, comp, comparatorClass, comparator ); return; } case TIMESTAMP: { addFilter( TimestampsFilter.class, fl, comp, comparatorClass, comparator ); // Constructor<TimestampsFilter> columnFilterConstructor = // TimestampsFilter.class.getConstructor( CompareFilter.CompareOp.class, comparatorClass ); // TimestampsFilter scf = columnFilterConstructor.newInstance( comp, comparator ); // fl.addFilter( scf ); return; } } }
@Test public void testAddFilterByMapping() throws Exception { testAddFilterByMapping( Mapping.TupleMapping.KEY, RowFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.FAMILY, FamilyFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.COLUMN, QualifierFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.VALUE, ValueFilter.class ); }
@Test(timeOut = 60_000) public void testGetWithValueFilter(ITestContext context) throws Exception { testGet(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1))); }
@Test(timeOut = 60_000) public void testScanWithValueFilter(ITestContext context) throws Exception { testScan(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1))); }
protected CheckAndMutateRowRequest.Builder makeConditionalMutationRequestBuilder( byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, byte[] actionRow, List<com.google.bigtable.v1.Mutation> mutations) throws IOException { if (!Arrays.equals(actionRow, row)) { // The following odd exception message is for compat with HBase. throw new DoNotRetryIOException("Action's getRow must match the passed row"); } CheckAndMutateRowRequest.Builder requestBuilder = CheckAndMutateRowRequest.newBuilder(); metadataSetter.setMetadata(requestBuilder); requestBuilder.setRowKey(ByteString.copyFrom(row)); Scan scan = new Scan().addColumn(family, qualifier); scan.setMaxVersions(1); if (value == null) { // If we don't have a value and we are doing CompareOp.EQUAL, we want to mutate if there // is no cell with the qualifier. If we are doing CompareOp.NOT_EQUAL, we want to mutate // if there is any cell. We don't actually want an extra filter for either of these cases, // but we do need to invert the compare op. if (CompareFilter.CompareOp.EQUAL.equals(compareOp)) { requestBuilder.addAllFalseMutations(mutations); } else if (CompareFilter.CompareOp.NOT_EQUAL.equals(compareOp)) { requestBuilder.addAllTrueMutations(mutations); } } else { ValueFilter valueFilter = new ValueFilter(compareOp, new BinaryComparator(value)); scan.setFilter(valueFilter); requestBuilder.addAllTrueMutations(mutations); } requestBuilder.setPredicateFilter( new ScanAdapter(filterAdapter) .buildFilter(scan)); return requestBuilder; }
/** * Create a new FilterAdapter */ public static FilterAdapter buildAdapter() { FilterAdapter adapter = new FilterAdapter(); adapter.addFilterAdapter( ColumnPrefixFilter.class, new ColumnPrefixFilterAdapter()); adapter.addFilterAdapter( ColumnRangeFilter.class, new ColumnRangeFilterAdapter()); adapter.addFilterAdapter( KeyOnlyFilter.class, new KeyOnlyFilterAdapter()); adapter.addFilterAdapter( MultipleColumnPrefixFilter.class, new MultipleColumnPrefixFilterAdapter()); adapter.addFilterAdapter( TimestampsFilter.class, new TimestampsFilterAdapter()); ValueFilterAdapter valueFilterAdapter = new ValueFilterAdapter(); adapter.addFilterAdapter( ValueFilter.class, valueFilterAdapter); SingleColumnValueFilterAdapter scvfa = new SingleColumnValueFilterAdapter(valueFilterAdapter); adapter.addFilterAdapter( SingleColumnValueFilter.class, scvfa); adapter.addFilterAdapter( SingleColumnValueExcludeFilter.class, new SingleColumnValueExcludeFilterAdapter(scvfa)); adapter.addFilterAdapter( ColumnPaginationFilter.class, new ColumnPaginationFilterAdapter()); adapter.addFilterAdapter( FirstKeyOnlyFilter.class, new FirstKeyOnlyFilterAdapter()); adapter.addFilterAdapter( ColumnCountGetFilter.class, new ColumnCountGetFilterAdapter()); adapter.addFilterAdapter( RandomRowFilter.class, new RandomRowFilterAdapter()); adapter.addFilterAdapter( PrefixFilter.class, new PrefixFilterAdapter()); adapter.addFilterAdapter( QualifierFilter.class, new QualifierFilterAdapter()); // Passing the FilterAdapter in to the FilterListAdapter is a bit // unfortunate, but makes adapting the FilterList's subfilters simpler. FilterListAdapter filterListAdapter = new FilterListAdapter(adapter); // FilterList implements UnsupportedStatusCollector so it should // be used when possible (third parameter to addFilterAdapter()). adapter.addFilterAdapter( FilterList.class, filterListAdapter, filterListAdapter); return adapter; }
/** * Construct a ValueFilter for a SingleColumnValueFilter. */ private ValueFilter createValueFilter(SingleColumnValueFilter filter) { return new ValueFilter(filter.getOperator(), filter.getComparator()); }
/** * Emit a filter that will match against a single value. */ private RowFilter createValueMatchFilter( FilterAdapterContext context, SingleColumnValueFilter filter) throws IOException { ValueFilter valueFilter = createValueFilter(filter); return delegateAdapter.adapt(context, valueFilter); }
FilterList makeFilterList(Operator filterOperator) { return new FilterList( filterOperator, new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("value"))), new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("value2")))); }
private void loadCache(boolean reverse) { if (cachedRows != null && cachedRows.hasNext()) return; // There are more rows so return and the code will return // the next result from cache if (needToGetBatch) { if (batchListener != null) batchListener.beforeFetchingNextBatch(); byte[] family = Bytes.toBytes(indexTableName); Scan scan = new Scan(rowKey, rowKey); scan.addFamily(family); FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL); if (from != null) { ValueFilter fromfilter = createFilterFrom(); list.addFilter(fromfilter); } if (to != null) { ValueFilter toFilter = createFilterTo(); list.addFilter(toFilter); } if (!list.getFilters().isEmpty()) scan.setFilter(list); if (batchSize != null) scan.setBatch(batchSize); // set this if there could be many columns returned ResultScanner rs; List<KeyValue> finalRes = new ArrayList<KeyValue>(); try { rs = hTable.getScanner(scan); for (Result r = rs.next(); r != null; r = rs.next()) { for (KeyValue kv : r.raw()) { finalRes.add(kv); } } rs.close(); if (batchListener != null) batchListener.afterFetchingNextBatch(2); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } fillinCache(finalRes); needToGetBatch = false; if (reverse) { while (cachedRows.hasNext()) cachedRows.next(); } } }
@Test public void testGetWithNoRowkeyFilter() throws IOException { // test ValueFilter commitColumnsWithDifferentTs(); Get get = createGetForDifferentTs(); get.setFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); Result iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); // test SingleColumnValueFilter get.setFilter(new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.EQUAL, ANOTHER_VALUE)); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); writeData(COLUMN, prewriteTs - 200, ANOTHER_VALUE); writePutColumn(COLUMN, prewriteTs - 200, commitTs - 200); get.setFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.list().size() == 1 && !ThemisCpUtil.isLockResult(iResult)); Assert.assertEquals(prewriteTs - 200, iResult.list().get(0).getTimestamp()); get.setFilter(new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.EQUAL, ANOTHER_VALUE)); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); // test FilterList with MUST_PASS_ALL FilterList filterList = new FilterList(); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); filterList.addFilter(new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.EQUAL, ANOTHER_VALUE)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.list().size() == 1 && !ThemisCpUtil.isLockResult(iResult)); Assert.assertEquals(prewriteTs - 200, iResult.list().get(0).getTimestamp()); // test FilterList with MUST_PASS_ONE filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE))); filterList.addFilter(new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.EQUAL, ANOTHER_VALUE)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); // test transfer filter get.setFilter(new KeyOnlyFilter()); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); for (KeyValue kv : iResult.list()) { Assert.assertEquals(0, kv.getValueLength()); } }
@Test public void testGetWithFilter() throws IOException, InterruptedException { byte[] row1 = Bytes.toBytes("row1"); byte[] fam1 = Bytes.toBytes("fam1"); byte[] col1 = Bytes.toBytes("col1"); byte[] value1 = Bytes.toBytes("value1"); byte[] value2 = Bytes.toBytes("value2"); final int maxVersions = 3; HColumnDescriptor hcd = new HColumnDescriptor(fam1); hcd.setMaxVersions(maxVersions); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testFilterAndColumnTracker")); htd.addFamily(hcd); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); final WAL wal = HBaseTestingUtility.createWal(TEST_UTIL.getConfiguration(), logDir, info); this.region = TEST_UTIL.createLocalHRegion(info, htd, wal); try { // Put 4 version to memstore long ts = 0; Put put = new Put(row1, ts); put.addColumn(fam1, col1, value1); region.put(put); put = new Put(row1, ts + 1); put.addColumn(fam1, col1, Bytes.toBytes("filter1")); region.put(put); put = new Put(row1, ts + 2); put.addColumn(fam1, col1, Bytes.toBytes("filter2")); region.put(put); put = new Put(row1, ts + 3); put.addColumn(fam1, col1, value2); region.put(put); Get get = new Get(row1); get.setMaxVersions(); Result res = region.get(get); // Get 3 versions, the oldest version has gone from user view assertEquals(maxVersions, res.size()); get.setFilter(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("value"))); res = region.get(get); // When use value filter, the oldest version should still gone from user view and it // should only return one key vaule assertEquals(1, res.size()); assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); region.flush(true); region.compact(true); Thread.sleep(1000); res = region.get(get); // After flush and compact, the result should be consistent with previous result assertEquals(1, res.size()); assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); } finally { HBaseTestingUtility.closeRegionAndWAL(this.region); this.region = null; } }
