private ResultScanner buildScanner(String keyPrefix, String value, Table ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
private Scan getVertexIndexScanWithLimit(String label, boolean isUnique, String key, Object from, int limit, boolean reversed) { byte[] prefix = serializeForRead(label, isUnique, key, null); byte[] startRow = from != null ? serializeForRead(label, isUnique, key, from) : prefix; byte[] stopRow = HConstants.EMPTY_END_ROW; if (graph.configuration().getInstanceType() == HBaseGraphConfiguration.InstanceType.BIGTABLE) { if (reversed) { throw new UnsupportedOperationException("Reverse scans not supported by Bigtable"); } else { // PrefixFilter in Bigtable does not automatically stop // See https://github.com/GoogleCloudPlatform/cloud-bigtable-client/issues/1087 stopRow = HBaseGraphUtils.incrementBytes(prefix); } } if (reversed) startRow = HBaseGraphUtils.incrementBytes(startRow); Scan scan = new Scan(startRow, stopRow); FilterList filterList = new FilterList(); filterList.addFilter(new PrefixFilter(prefix)); filterList.addFilter(new PageFilter(limit)); scan.setFilter(filterList); scan.setReversed(reversed); return scan; }
private ResultScanner buildScanner(String keyPrefix, String value, HTable ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
@Test public void testPrefixAddedAsRowRegex() throws IOException { PrefixFilterAdapter adapter = new PrefixFilterAdapter(); String prefix = "Foobar"; PrefixFilter filter = new PrefixFilter(Bytes.toBytes(prefix)); Scan emptyScan = new Scan(); FilterAdapterContext context = new FilterAdapterContext(emptyScan); byte[] prefixRegex = Bytes.toBytes(prefix + "\\C*"); Assert.assertEquals( RowFilter.newBuilder() .setRowKeyRegexFilter( ByteString.copyFrom(prefixRegex)) .build(), adapter.adapt(context, filter)); }
@Test public void testPrefixFilter() throws IOException { String prefix = "testPrefixFilter"; int rowCount = 10; byte[][] rowKeys = dataHelper.randomData(prefix, rowCount); List<Put> puts = new ArrayList<>(); for (byte[] rowKey : rowKeys) { puts.add( new Put(rowKey) .addColumn(COLUMN_FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("val1"))); } Table table = getConnection().getTable(TABLE_NAME); table.put(puts); PrefixFilter filter = new PrefixFilter(Bytes.toBytes(prefix)); Scan scan = new Scan().addFamily(COLUMN_FAMILY).setFilter(filter); ResultScanner scanner = table.getScanner(scan); Result[] results = scanner.next(rowCount + 2); Assert.assertEquals(rowCount, results.length); Arrays.sort(rowKeys, Bytes.BYTES_COMPARATOR); // Both results[] and rowKeys[] should be in the same order now. Iterate over both // and verify rowkeys. for (int i = 0; i < rowCount; i++) { Assert.assertArrayEquals(rowKeys[i], results[i].getRow()); } }
@Test public void testConstructDataRowGet() throws IOException { ThemisGet get = new ThemisGet(ROW); get.addFamily(FAMILY); get.addColumn(ANOTHER_FAMILY, QUALIFIER); get.setFilter(new PrefixFilter(ROW)); get.setCacheBlocks(true); ThemisGet actual = IndexScanner.constructDataRowGet(ANOTHER_ROW, get); Assert.assertArrayEquals(ANOTHER_ROW, actual.getRow()); Assert .assertArrayEquals(QUALIFIER, actual.getFamilyMap().get(ANOTHER_FAMILY).iterator().next()); Assert.assertTrue(actual.getFamilyMap().containsKey(FAMILY)); Assert.assertNull(actual.getFamilyMap().get(FAMILY)); Assert.assertTrue(actual.getCacheBlocks()); }
@Test public void testGetWithRowkeyFilter() throws IOException { commitColumnsWithDifferentTs(); Get get = createGetForDifferentTs(); get.setFilter(new PrefixFilter(ROW)); Result iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); get.setFilter(new PrefixFilter(ANOTHER_ROW)); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); FilterList filterList = new FilterList(); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); Assert.assertTrue(iResult.isEmpty()); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(new PrefixFilter(ROW)); filterList.addFilter(new PrefixFilter(ANOTHER_ROW)); get.setFilter(filterList); iResult = cpClient.themisGet(TABLENAME, get, prewriteTs); checkGetResultForDifferentTs(iResult); }
@Test public void testScanWithFilter() throws IOException { prepareScanData(TRANSACTION_COLUMNS); writeData(COLUMN, lastTs(prewriteTs), ANOTHER_VALUE); ValueFilter valueFilter = new ValueFilter(CompareOp.EQUAL, new BinaryComparator(ANOTHER_VALUE)); PrefixFilter prefixFilter = new PrefixFilter(ANOTHER_ROW); FilterList filterList = new FilterList(); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); ThemisScanner scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkAndCloseScanner(scanner); filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.addFilter(valueFilter); filterList.addFilter(prefixFilter); scanner = prepareScanner(TRANSACTION_COLUMNS, filterList); checkScanRow(new ColumnCoordinate[]{COLUMN_WITH_ANOTHER_ROW}, scanner.next()); Assert.assertEquals(1, scanner.next().size()); checkAndCloseScanner(scanner); }
private ResultScanner buildScanner(String keyPrefix, String value, Table ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns) throws IOError, TException { try { HTable table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if(columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan)); } catch (IOException e) { throw new IOError(e.getMessage()); } }
@Test public void testAddColumnFilterToScanPrefixFilter() throws Exception { ColumnFilter cf = new ColumnFilter( "Family" ); cf.setConstant( "123" ); cf.setSignedComparison( true ); VariableSpace space = mockVariableSpace(); connectionSpy.m_sourceScan = new Scan(); HBaseValueMeta meta = new HBaseValueMeta( "colFamly,colname,Family", 1, 20, 1 ); meta.setKey( true ); meta.setIsLongOrDouble( true ); doReturn( null ).when( connectionSpy ).getCompareOpByComparisonType( any( ColumnFilter.ComparisonType.class ) ); connectionSpy.addColumnFilterToScan( cf, meta, space, true ); FilterList filter = (FilterList) connectionSpy.m_sourceScan.getFilter(); assertFalse( filter.getFilters().isEmpty() ); Assert.assertEquals( filter.getFilters().size(), 1 ); Assert.assertEquals( PrefixFilter.class, filter.getFilters().get( 0 ).getClass() ); }
/** * Returns the {@link Flow} instance matching the application ID and run ID. * * @param cluster the cluster identifier * @param user the user running the jobs * @param appId the application description * @param runId the specific run ID for the flow * @param populateTasks whether or not to populate the task details for each * job * @return */ public Flow getFlow(String cluster, String user, String appId, long runId, boolean populateTasks) throws IOException { Flow flow = null; byte[] startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(cluster), Bytes.toBytes(user), Bytes.toBytes(appId), Bytes.toBytes(FlowKey.encodeRunId(runId)), Constants.EMPTY_BYTES); LOG.info( "Reading job_history rows start at " + Bytes.toStringBinary(startRow)); Scan scan = new Scan(); // start scanning history at cluster!user!app!run! scan.setStartRow(startRow); // require that all results match this flow prefix scan.setFilter(new WhileMatchFilter(new PrefixFilter(startRow))); List<Flow> flows = createFromResults(scan, populateTasks, 1); if (flows.size() > 0) { flow = flows.get(0); } return flow; }
/** * Returns the {@link Flow} instance containing the given job ID. * * @param cluster the cluster identifier * @param jobId the job identifier * @return */ public Flow getFlowByJobID(String cluster, String jobId, boolean populateTasks) throws IOException { Flow flow = null; JobKey key = idService.getJobKeyById(new QualifiedJobId(cluster, jobId)); if (key != null) { byte[] startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(key.getCluster()), Bytes.toBytes(key.getUserName()), Bytes.toBytes(key.getAppId()), Bytes.toBytes(key.getEncodedRunId()), Constants.EMPTY_BYTES); LOG.info("Reading job_history rows start at " + Bytes.toStringBinary(startRow)); Scan scan = new Scan(); // start scanning history at cluster!user!app!run! scan.setStartRow(startRow); // require that all results match this flow prefix scan.setFilter(new WhileMatchFilter(new PrefixFilter(startRow))); List<Flow> flows = createFromResults(scan, populateTasks, 1); if (flows.size() > 0) { flow = flows.get(0); } } return flow; }
/** * creates a scan for flow data * @param rowPrefix - start row prefix * @param limit - limit on scanned results * @param version - version to match * @return Scan */ private Scan createFlowScan(byte[] rowPrefix, int limit, String version) { Scan scan = new Scan(); scan.setStartRow(rowPrefix); // using a large scanner caching value with a small limit can mean we scan a // lot more data than necessary, so lower the caching for low limits scan.setCaching(Math.min(limit, defaultScannerCaching)); // require that all rows match the prefix we're looking for Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix)); // if version is passed, restrict the rows returned to that version if (version != null && version.length() > 0) { FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); filters.addFilter(prefixFilter); filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(version))); scan.setFilter(filters); } else { scan.setFilter(prefixFilter); } return scan; }
@Override public Set<Integer> findKeysByScope(String scope) { Set<Integer> keys = Sets.newHashSet(); // note HTableStore isn't capable of ad hoc scans try (Table table = connection.getTable(lookupTableName)) { Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setCaching(HBASE_CLIENT_CACHING); scan.setFilter(new PrefixFilter(Bytes.toBytes(scope))); ResultScanner results = table.getScanner(scan); for (Result result : results) { byte[] rawKey = result.getValue(Columns.CF, Bytes.toBytes(Columns.LOOKUP_KEY_COLUMN)); if (rawKey != null) { keys.add(Bytes.toInt(rawKey)); } } } catch (IOException e) { throw new ServiceUnavailableException("Could not read from HBase", e); } return keys; }
/** * 根据Row的前缀获得value * * @param tableName * @param rowPrefix * @param familyName * @param qualifierName * @return * @throws Exception */ public List<String> getValueByRowPrefix(String tableName, String rowPrefix, String familyName, String qualifierName) throws Exception { HTable htable = getHtable(tableName); List<String> values = new ArrayList<>(); Scan scan = new Scan(); scan.setFilter(new PrefixFilter(Bytes.toBytes(rowPrefix))); htable.getScanner(scan).forEach((result) -> { Cell cell = result.getColumnLatestCell(Bytes.toBytes(familyName), Bytes.toBytes(qualifierName)); if (cell != null) { values.add(Bytes.toString(CellUtil.cloneValue(cell))); } }); return values; }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { Table table = null; try { table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw new IOError(Throwables.getStackTraceAsString(e)); } finally{ closeTable(table); } }
