/** * For core use. DO NOT USE. */ public SingleRowQueryCriteria toOTSParameter(String tableName) { PrimaryKey primaryKey = OTSUtil.toPrimaryKey(getRow(), OTSConstants.PRIMARY_KEY_NAME); SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(tableName, primaryKey); criteria.setMaxVersions(getMaxVersions()); criteria.setTimeRange(OTSUtil.toTimeRange(getTimeRange())); for (byte[] col : columnsToGet) { criteria.addColumnsToGet(Bytes.toString(col)); } if (filter != null) { if (filter instanceof OColumnPaginationFilter) { OColumnPaginationFilter oFilter = (OColumnPaginationFilter)filter; com.alicloud.openservices.tablestore.model.filter.ColumnPaginationFilter columnPaginationFilter = new com.alicloud.openservices.tablestore.model.filter.ColumnPaginationFilter(oFilter.getLimit()); if (oFilter.getColumnOffset() == null) { columnPaginationFilter.setOffset(oFilter.getOffset()); } else { criteria.setStartColumn(Bytes.toString(oFilter.getColumnOffset())); } criteria.setFilter(columnPaginationFilter); } else { criteria.setFilter(OTSUtil.toFilter(getFilter())); } } return criteria; }
@Override public RowFilter adapt(FilterAdapterContext context, ColumnPaginationFilter filter) throws IOException { if (filter.getColumnOffset() != null) { byte[] family = context.getScan().getFamilies()[0]; // Include all cells starting at the qualifier scan.getColumnOffset() // up to limit cells. return createChain( filter, RowFilter.newBuilder() .setColumnRangeFilter( ColumnRange.newBuilder() .setFamilyName(Bytes.toString(family)) .setStartQualifierInclusive( ByteString.copyFrom(filter.getColumnOffset())))); } else if (filter.getOffset() > 0) { // Include starting at an integer offset up to limit cells. return createChain( filter, RowFilter.newBuilder() .setCellsPerRowOffsetFilter(filter.getOffset())); } else { // No meaningful offset supplied. return createChain(filter, null); } }
@Test public void integerLimitsAreApplied() throws IOException { ColumnPaginationFilter filter = new ColumnPaginationFilter(10, 20); RowFilter adaptedFilter = adapter.adapt( new FilterAdapterContext(new Scan()), filter); Assert.assertEquals( RowFilter.newBuilder() .setChain( Chain.newBuilder() .addFilters(RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1)) .addFilters(RowFilter.newBuilder() .setCellsPerRowOffsetFilter(20)) .addFilters(RowFilter.newBuilder() .setCellsPerRowLimitFilter(10))) .build(), adaptedFilter); }
@Test public void zeroOffsetLimitIsSupported() throws IOException { ColumnPaginationFilter filter = new ColumnPaginationFilter(10, 0); RowFilter adaptedFilter = adapter.adapt( new FilterAdapterContext(new Scan()), filter); Assert.assertEquals( RowFilter.newBuilder() .setChain( Chain.newBuilder() .addFilters(RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1)) .addFilters(RowFilter.newBuilder() .setCellsPerRowLimitFilter(10))) .build(), adaptedFilter); }
/** * Requirement 9.2 */ @Test @Category(KnownGap.class) public void testColumnPaginationFilter_StartingAtParticularQualifier() throws Exception { Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testrow-"); Put put = new Put(rowKey); byte[] value = Bytes.toBytes("someval"); put.addColumn(COLUMN_FAMILY, Bytes.toBytes("A"), value); put.addColumn(COLUMN_FAMILY, Bytes.toBytes("AA"), value); put.addColumn(COLUMN_FAMILY, Bytes.toBytes("B"), value); put.addColumn(COLUMN_FAMILY, Bytes.toBytes("BB"), value); put.addColumn(COLUMN_FAMILY, Bytes.toBytes("C"), value); table.put(put); // Filter and test Filter filter = new ColumnPaginationFilter(3, Bytes.toBytes("AA")); Get get = new Get(rowKey).setFilter(filter).setMaxVersions(10); Result result = table.get(get); Assert.assertEquals("Should have filtered to N columns", 3, result.size()); Assert.assertEquals("AA", Bytes.toString(CellUtil.cloneQualifier(result.rawCells()[0]))); Assert.assertEquals("B", Bytes.toString(CellUtil.cloneQualifier(result.rawCells()[1]))); Assert.assertEquals("BB", Bytes.toString(CellUtil.cloneQualifier(result.rawCells()[2]))); table.close(); }
public void testGet() throws IOException { long st = System.currentTimeMillis(); Get get = new Get( Bytes.toBytes("{1F591795-74DE-EB70-0245-0E4465C72CFA}")); get.addColumn(Bytes.toBytes("bhvr"), Bytes.toBytes("vvmid")); get.setMaxVersions(100); // get.setTimeRange(1354010844711L - 12000L, 1354010844711L); // get.setTimeStamp(1354700700000L); Filter filter = new ColumnPaginationFilter(1, 10); get.setFilter(filter); Result dbResult = table.get(get); System.out.println("result=" + dbResult.toString()); long en2 = System.currentTimeMillis(); System.out.println("Total Time: " + (en2 - st) + " ms"); }
@Test public void testGet() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Get get = new Get(Bytes.toBytes(rowPrefix)); Filter filter = new ColumnPaginationFilter(1, 2); get.setFilter(filter); Result result = table.get(get); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(1, cells.size()); assertEquals("col_3_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(0, cells.size()); } }
@Test public void testScan() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Scan scan = new Scan(); Filter filter = new ColumnPaginationFilter(1, 2); scan.setFilter(filter); scan.setStartRow(Bytes.toBytes(rowPrefix)); scan.setStopRow(Bytes.toBytes(rowPrefix + 5)); ResultScanner scanResult = table.getScanner(scan); Result result = scanResult.next(); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(1, cells.size()); assertEquals("col_3_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(0, cells.size()); } }
@Test public void testResultLessThanLimit() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Get get = new Get(Bytes.toBytes(rowPrefix)); Filter filter = new ColumnPaginationFilter(5, 3); get.setFilter(filter); Result result = table.get(get); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(1, cells.size()); assertEquals("col_4_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); } }
@Test public void testStartRow() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Get get = new Get(Bytes.toBytes(rowPrefix)); Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_2")); get.setFilter(filter); Result result = table.get(get); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(1, cells.size()); assertEquals("col_2_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(0, cells.size()); } }
@Test public void testStartRowByScan() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Scan scan = new Scan(); Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_2")); scan.setFilter(filter); scan.setStartRow(Bytes.toBytes(rowPrefix)); scan.setStopRow(Bytes.toBytes(rowPrefix + 5)); ResultScanner scanResult = table.getScanner(scan); Result result = scanResult.next(); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(1, cells.size()); assertEquals("col_2_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(0, cells.size()); } }
@Test public void testStartRowIsNotExist() throws IOException { clean(); { Put put = new Put(Bytes.toBytes(rowPrefix)); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var")); put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var")); table.put(put); } { Get get = new Get(Bytes.toBytes(rowPrefix)); Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_3_not_exist")); get.setFilter(filter); Result result = table.get(get); List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3")); assertEquals(0, cells.size()); cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4")); assertEquals(1, cells.size()); assertEquals("col_4_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength())); } }
/** * Create a filter chain that allows the latest values for each * qualifier, those cells that pass an option intermediate filter * and are less than the limit per row. */ private RowFilter createChain( ColumnPaginationFilter filter, RowFilter.Builder intermediate) { Chain.Builder builder = Chain.newBuilder(); builder.addFilters( RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1)); if (intermediate != null) { builder.addFilters(intermediate); } builder.addFilters( RowFilter.newBuilder() .setCellsPerRowLimitFilter(filter.getLimit())); return RowFilter.newBuilder().setChain(builder).build(); }
@Override public FilterSupportStatus isFilterSupported( FilterAdapterContext context, ColumnPaginationFilter filter) { // We require a single column family to be specified: int familyCount = context.getScan().numFamilies(); if (filter.getColumnOffset() != null && familyCount != 1) { return UNSUPPORTED_STATUS; } return FilterSupportStatus.SUPPORTED; }
@Test public void qualifierOffsetIsPartiallySupported() throws IOException { ColumnPaginationFilter filter = new ColumnPaginationFilter(10, Bytes.toBytes("q1")); RowFilter adaptedFilter = adapter.adapt( new FilterAdapterContext( new Scan().addFamily(Bytes.toBytes("f1"))), filter); Assert.assertEquals( RowFilter.newBuilder() .setChain( Chain.newBuilder() .addFilters( RowFilter.newBuilder() .setCellsPerColumnLimitFilter(1)) .addFilters( RowFilter.newBuilder() .setColumnRangeFilter( ColumnRange.newBuilder() .setFamilyName("f1") .setStartQualifierInclusive( ByteString.copyFromUtf8("q1")))) .addFilters( RowFilter.newBuilder() .setCellsPerRowLimitFilter(10))) .build(), adaptedFilter); }
/** * Requirement 9.2 - ColumnPaginationFilter - same as ColumnCountGetFilter, but with an offset * too; offset can be a # of cols, or can be a particular qualifier byte[] value (inclusive) */ @Test public void testColumnPaginationFilter() throws Exception { // Initialize data int numColumns = 20; int numColumnsToFilter = 8; int offset = 5; Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testrow-"); byte[][] quals = new byte[numColumns][]; byte[][][] values = new byte[2][][]; values[0] = dataHelper.randomData("testvalue-", numColumns); values[1] = dataHelper.randomData("testvalue-", numColumns); Put put = new Put(rowKey); for (int i = 0; i < numColumns; ++i) { quals[i] = Bytes.toBytes(i); // Add two timestamps to test that filter only grabs the latest version put.addColumn(COLUMN_FAMILY, quals[i], 1L, values[0][i]); put.addColumn(COLUMN_FAMILY, quals[i], 2L, values[1][i]); } table.put(put); // Filter and test Filter filter = new ColumnPaginationFilter(numColumnsToFilter, offset); Get get = new Get(rowKey).setFilter(filter).setMaxVersions(10); Result result = table.get(get); Assert.assertEquals("Should have filtered to N columns", numColumnsToFilter, result.size()); for (int i = offset ; i < (numColumnsToFilter + offset); ++i) { Assert.assertTrue("Should contain qual " + Bytes.toInt(quals[i]), result.containsColumn(COLUMN_FAMILY, quals[i])); List<Cell> cells = result.getColumnCells(COLUMN_FAMILY, quals[i]); Assert.assertEquals("Should have only the latest version", 1, cells.size()); Assert.assertArrayEquals("Value v2 should be first and match", values[1][i], CellUtil.cloneValue(cells.get(0))); } table.close(); }
public void testScanGet() throws IOException { long st = System.currentTimeMillis(); Get get = new Get( Bytes.toBytes("{1F591795-74DE-EB70-0245-0E4465C72CFA}")); get.addColumn(Bytes.toBytes("bhvr"), Bytes.toBytes("vvmid")); Filter filter = new ColumnPaginationFilter(1, 10); Scan scanner = new Scan(get); scanner.setFilter(filter); scanner.setMaxVersions(100); // scanner.setCaching(100); // scanner.setBatch(10); ResultScanner rsScanner = table.getScanner(scanner); for (Result result : rsScanner) { System.out.println(result); } rsScanner.close(); int version = 0; // int count = 0; // for (Result res : rsScanner) { // System.out.println("count: " + ++count); // final List<KeyValue> list = res.list(); // // for (final KeyValue kv : list) // System.out.println(kv.toString()); // // version = list.size(); // } // rsScanner.close(); long en2 = System.currentTimeMillis(); System.out.println("Total rows: " + version); System.out.println("Total Time: " + (en2 - st) + " ms"); }
/** * Create a new FilterAdapter */ public static FilterAdapter buildAdapter() { FilterAdapter adapter = new FilterAdapter(); adapter.addFilterAdapter( ColumnPrefixFilter.class, new ColumnPrefixFilterAdapter()); adapter.addFilterAdapter( ColumnRangeFilter.class, new ColumnRangeFilterAdapter()); adapter.addFilterAdapter( KeyOnlyFilter.class, new KeyOnlyFilterAdapter()); adapter.addFilterAdapter( MultipleColumnPrefixFilter.class, new MultipleColumnPrefixFilterAdapter()); adapter.addFilterAdapter( TimestampsFilter.class, new TimestampsFilterAdapter()); ValueFilterAdapter valueFilterAdapter = new ValueFilterAdapter(); adapter.addFilterAdapter( ValueFilter.class, valueFilterAdapter); SingleColumnValueFilterAdapter scvfa = new SingleColumnValueFilterAdapter(valueFilterAdapter); adapter.addFilterAdapter( SingleColumnValueFilter.class, scvfa); adapter.addFilterAdapter( SingleColumnValueExcludeFilter.class, new SingleColumnValueExcludeFilterAdapter(scvfa)); adapter.addFilterAdapter( ColumnPaginationFilter.class, new ColumnPaginationFilterAdapter()); adapter.addFilterAdapter( FirstKeyOnlyFilter.class, new FirstKeyOnlyFilterAdapter()); adapter.addFilterAdapter( ColumnCountGetFilter.class, new ColumnCountGetFilterAdapter()); adapter.addFilterAdapter( RandomRowFilter.class, new RandomRowFilterAdapter()); adapter.addFilterAdapter( PrefixFilter.class, new PrefixFilterAdapter()); adapter.addFilterAdapter( QualifierFilter.class, new QualifierFilterAdapter()); // Passing the FilterAdapter in to the FilterListAdapter is a bit // unfortunate, but makes adapting the FilterList's subfilters simpler. FilterListAdapter filterListAdapter = new FilterListAdapter(adapter); // FilterList implements UnsupportedStatusCollector so it should // be used when possible (third parameter to addFilterAdapter()). adapter.addFilterAdapter( FilterList.class, filterListAdapter, filterListAdapter); return adapter; }