public long deleteViews(String cp, String recommender) { FilterList filters = new FilterList(); // filter column-family & recommender name filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(EVENTLOG_COLUMN_USERVIEW))); filters .addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys.getUserViewKey(recommender)))); Scan scan = new Scan().addFamily(EVENTLOG_COLUMN_USERVIEW).setFilter(filters); // TODO put this in a map-reduce delete.. that executes in background.. // we only need to pass the table & a scan object. should be quite easy long count = deleteAll(scan, cp, EVENTLOG_COLUMN_RECOMMENDED); return count; }
/** * @param cp */ public void deleteStatisticsTasks(String cp) { FilterList filters = new FilterList(); // column family filter: filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); // filter on task type: SingleColumnValueFilter tasktype = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.TASK_TYPE), CompareOp.EQUAL, new BinaryComparator( Bytes.toBytes(TaskConfig.TASK_TYPE_STATISTICS))); tasktype.setFilterIfMissing(true); filters.addFilter(tasktype); // content provider filter: SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filters.addFilter(cpfilter); Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filters); deleteAll(scan, cp); }
public static void main(String[] args) throws IOException { Configuration configuration = getConfiguration(); HBaseHelper helper = HBaseHelper.getHelper(configuration); Connection connection = helper.getConnection(); /*helper.dropTable(TEST_TABLE_NAME); helper.createTable(TEST_TABLE_NAME, "colfam1", "colfam2", "colfam3", "colfam4"); log.info("Adding rows to table..."); helper.fillTable(TEST_TABLE_NAME, 1, 10, 2, "colfam1", "colfam2", "colfam3", "colfam4");*/ Table table = connection.getTable(TableName.valueOf(TEST_TABLE_NAME)); Filter filter1 = new FamilyFilter(CompareOp.LESS, new BinaryComparator(Bytes.toBytes("colfam3"))); Scan scan = new Scan(); scan.setFilter(filter1); ResultScanner scanner = table.getScanner(scan); log.info("Scanning table... "); for (Result result : scanner) { log.info(result.toString()); } scanner.close(); Get get1 = new Get(Bytes.toBytes("row-5")); get1.setFilter(filter1); Result result1 = table.get(get1); log.info("Result of get(): {}", result1); Filter filter2 = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("colfam3"))); Get get2 = new Get(Bytes.toBytes("row-5")); get2.addFamily(Bytes.toBytes("colfam1")); get2.setFilter(filter2); Result result2 = table.get(get2); log.info("Result of get(): {}", result2); }
@Test public void testDescriptorsAndPrefix() throws Exception { Filter filter = getHBaseStorageFilter("cf1:a cf1:b cf2:foo*"); List<Filter> childFilters = assertFilterList(filter, FilterList.Operator.MUST_PASS_ALL, 1); List<Filter> groupFilters = assertFilterList(childFilters.get(0), FilterList.Operator.MUST_PASS_ONE, 2); List<Filter> firstFilters = assertFilterList(groupFilters.get(0), FilterList.Operator.MUST_PASS_ALL, 2); FamilyFilter firstFamilyFilter = assertFamilyFilter(firstFilters.get(0), CompareFilter.CompareOp.EQUAL); List<Filter> secondFilters = assertFilterList(groupFilters.get(1), FilterList.Operator.MUST_PASS_ALL, 2); FamilyFilter secondFamilyFilter = assertFamilyFilter(secondFilters.get(0), CompareFilter.CompareOp.EQUAL); // one of the above will be the cf1 filters, one will be the cf2. Order is unknown Filter cf1ColumnList; Filter cf2ColumnList; if (Bytes.toString(firstFamilyFilter.getComparator().getValue()).equals("cf1")) { assertEquals("cf2", Bytes.toString(secondFamilyFilter.getComparator().getValue())); cf1ColumnList = firstFilters.get(1); cf2ColumnList = secondFilters.get(1); } else { assertEquals("cf1", Bytes.toString(secondFamilyFilter.getComparator().getValue())); assertEquals("cf2", Bytes.toString(firstFamilyFilter.getComparator().getValue())); cf1ColumnList = secondFilters.get(1); cf2ColumnList = firstFilters.get(1); } List<Filter> c1ColumnFilters = assertFilterList(cf1ColumnList, FilterList.Operator.MUST_PASS_ONE, 2); assertQualifierFilter(c1ColumnFilters.get(0), CompareFilter.CompareOp.EQUAL, "a"); assertQualifierFilter(c1ColumnFilters.get(1), CompareFilter.CompareOp.EQUAL, "b"); List<Filter> c2ColumnFilters = assertFilterList(cf2ColumnList, FilterList.Operator.MUST_PASS_ONE, 1); assertPrefixFilter(c2ColumnFilters.get(0), "foo"); }
private FamilyFilter assertFamilyFilter(Filter filter, CompareFilter.CompareOp compareOp) { assertTrue("Filter is not a FamilyFilter: " + filter.getClass().getSimpleName(), filter instanceof FamilyFilter); FamilyFilter familyFilter = (FamilyFilter)filter; assertEquals("Unexpected compareOp", compareOp, familyFilter.getOperator()); return familyFilter; }
void addFilterByMapping( FilterList fl, CompareFilter.CompareOp comp, Class<?> comparatorClass, Object comparator, Mapping.TupleMapping tupleMapping ) throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException { switch ( tupleMapping ) { case KEY: { addFilter( RowFilter.class, fl, comp, comparatorClass, comparator ); return; } case FAMILY: { addFilter( FamilyFilter.class, fl, comp, comparatorClass, comparator ); return; } case COLUMN: { //TODO Check if ColumnPrefixFilter works faster and suit more addFilter( QualifierFilter.class, fl, comp, comparatorClass, comparator ); return; } case VALUE: { addFilter( ValueFilter.class, fl, comp, comparatorClass, comparator ); return; } case TIMESTAMP: { addFilter( TimestampsFilter.class, fl, comp, comparatorClass, comparator ); // Constructor<TimestampsFilter> columnFilterConstructor = // TimestampsFilter.class.getConstructor( CompareFilter.CompareOp.class, comparatorClass ); // TimestampsFilter scf = columnFilterConstructor.newInstance( comp, comparator ); // fl.addFilter( scf ); return; } } }
@Test public void testAddFilterByMapping() throws Exception { testAddFilterByMapping( Mapping.TupleMapping.KEY, RowFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.FAMILY, FamilyFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.COLUMN, QualifierFilter.class ); testAddFilterByMapping( Mapping.TupleMapping.VALUE, ValueFilter.class ); }
public long deleteViews(String cp, String recommender, long user) { FilterList filters = new FilterList(); filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(EVENTLOG_COLUMN_USERVIEW))); filters.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys.getUserViewKey(recommender, user)))); Scan scan = new Scan().addFamily(EVENTLOG_COLUMN_USERVIEW).setFilter(filters); return deleteAll(scan, cp, EVENTLOG_COLUMN_USERVIEW); }
public long deleteView(String cp, String recommender, long user, Long startDate, Long endDate) { FilterList filters = new FilterList(); filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(EVENTLOG_COLUMN_USERVIEW))); filters.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys.getUserViewKey(recommender, user)))); Scan scan = new Scan().addFamily(EVENTLOG_COLUMN_USERVIEW).setFilter(filters); return deleteAll(scan, cp, EVENTLOG_COLUMN_USERVIEW); }
public long deleteRecommend(String cp, String recommender) { FilterList filters = new FilterList(); filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(EVENTLOG_COLUMN_RECOMMENDED))); filters.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys .getRecommendedItemKey(recommender)))); Scan scan = new Scan().addFamily(EVENTLOG_COLUMN_RECOMMENDED).setFilter(filters); return deleteAll(scan, cp, EVENTLOG_COLUMN_RECOMMENDED); }
public long deleteRecommend(String cp, String recommender, long user) { FilterList filters = new FilterList(); filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(EVENTLOG_COLUMN_RECOMMENDED))); filters.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys.getRecommendedItemKey( recommender, user)))); Scan scan = new Scan().addFamily(EVENTLOG_COLUMN_RECOMMENDED).setFilter(filters); return deleteAll(scan, cp, EVENTLOG_COLUMN_RECOMMENDED); }
/** * delete all ratings for a given collection * @param cp * @param collection */ public void deleteRatings(String cp, String collection) { FilterList filters = new FilterList(); // create a filter: filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_RATING))); filters.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryPrefixComparator(RowKeys.getRatingKey(collection)))); Scan scan = new Scan().addFamily(COLUMN_RATING).setFilter(filters); deleteAll(scan, cp, COLUMN_RATING); // TODO hm... this will fuck up the rating count of a user... & needs a fix - track the users... do a batch // decrease on them or something }
/** * Removes all tasks related to collection import * @param cp content provider * @param collection collection name */ public void deleteCollectionTasks(String cp, String collection) { FilterList filter = new FilterList(); // column family filter: filter.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); // filter on task type: SingleColumnValueFilter tasktype = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.TASK_TYPE), CompareOp.EQUAL, new BinaryComparator( Bytes.toBytes(TaskConfig.TASK_TYPE_COLLECTION))); tasktype.setFilterIfMissing(true); filter.addFilter(tasktype); // filter on content provider: SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filter.addFilter(cpfilter); // filter on collection: (this is not really needed for now, because there's only one collection per cp) if (collection != null) { SingleColumnValueFilter coll = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.COLLECTION), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(collection))); cpfilter.setFilterIfMissing(true); filter.addFilter(coll); } Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filter); deleteAll(scan, cp); }
public void deleteRecommenderTasks(String cp, String recommender) { FilterList filters = new FilterList(); // column family filter: filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); // filter on task type: SingleColumnValueFilter tasktype = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.TASK_TYPE), CompareOp.EQUAL, new BinaryComparator( Bytes.toBytes(TaskConfig.TASK_TYPE_RECOMMENDER))); tasktype.setFilterIfMissing(true); filters.addFilter(tasktype); // content provider filter: SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filters.addFilter(cpfilter); // filter on recommender: if (recommender != null) { SingleColumnValueFilter recFilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.RECOMMENDER), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(recommender))); recFilter.setFilterIfMissing(true); filters.addFilter(recFilter); } Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filters); deleteAll(scan, cp); }
/** * If we have a qualifier with a prefix and a wildcard (i.e. cf:foo*), we need a filter on every * possible column to be returned as shown below. This will become very inneficient for long * lists of columns mixed with a prefixed wildcard. * * FilterList - must pass ALL of * - FamilyFilter * - AND a must pass ONE FilterList of * - either Qualifier * - or ColumnPrefixFilter * * If we have only column family filters (i.e. cf:*) or explicit column descriptors * (i.e., cf:foo) or a mix of both then we don't need filters, since the scan will take * care of that. */ private void addFiltersWithColumnPrefix(List<ColumnInfo> columnInfos) { // we need to apply a CF AND column list filter for each family FilterList allColumnFilters = null; Map<String, List<ColumnInfo>> groupedMap = groupByFamily(columnInfos); for (String cfString : groupedMap.keySet()) { List<ColumnInfo> columnInfoList = groupedMap.get(cfString); byte[] cf = Bytes.toBytes(cfString); // all filters roll up to one parent OR filter if (allColumnFilters == null) { allColumnFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); } // each group contains a column family filter AND (all) and an OR (one of) of // the column filters FilterList thisColumnGroupFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL); thisColumnGroupFilter.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(cf))); FilterList columnFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); for (ColumnInfo colInfo : columnInfoList) { if (colInfo.isColumnMap()) { if (LOG.isDebugEnabled()) { LOG.debug("Adding family:prefix filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnPrefix())); } // add a PrefixFilter to the list of column filters if (colInfo.getColumnPrefix() != null) { columnFilters.addFilter(new ColumnPrefixFilter( colInfo.getColumnPrefix())); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Adding family:descriptor filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnName())); } // add a QualifierFilter to the list of column filters columnFilters.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(colInfo.getColumnName()))); } } thisColumnGroupFilter.addFilter(columnFilters); allColumnFilters.addFilter(thisColumnGroupFilter); } if (allColumnFilters != null) { addFilter(allColumnFilters); } }
private FamilyFilter assertFamilyFilter(Filter filter, CompareFilter.CompareOp compareOp, String value) { FamilyFilter familyFilter = assertFamilyFilter(filter, compareOp); assertEquals("Unexpected value", value, Bytes.toString(familyFilter.getComparator().getValue())); return familyFilter; }
private void initScan() { scan = new Scan(); // Map-reduce jobs should not run with cacheBlocks scan.setCacheBlocks(false); // Set filters, if any. if (configuredOptions_.hasOption("gt")) { gt_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("gt"))); addRowFilter(CompareOp.GREATER, gt_); } if (configuredOptions_.hasOption("lt")) { lt_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("lt"))); addRowFilter(CompareOp.LESS, lt_); } if (configuredOptions_.hasOption("gte")) { gte_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("gte"))); addRowFilter(CompareOp.GREATER_OR_EQUAL, gte_); } if (configuredOptions_.hasOption("lte")) { lte_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("lte"))); addRowFilter(CompareOp.LESS_OR_EQUAL, lte_); } // apply any column filters FilterList allColumnFilters = null; for (ColumnInfo colInfo : columnInfo_) { // all column family filters roll up to one parent OR filter if (allColumnFilters == null) { allColumnFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); } // and each filter contains a column family filter FilterList thisColumnFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL); thisColumnFilter.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(colInfo.getColumnFamily()))); if (colInfo.isColumnMap()) { if (LOG.isInfoEnabled()) { LOG.info("Adding family:prefix filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnPrefix())); } // each column map filter consists of a FamilyFilter AND // optionally a PrefixFilter if (colInfo.getColumnPrefix() != null) { thisColumnFilter.addFilter(new ColumnPrefixFilter( colInfo.getColumnPrefix())); } } else { if (LOG.isInfoEnabled()) { LOG.info("Adding family:descriptor filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnName())); } // each column value filter consists of a FamilyFilter AND // a QualifierFilter thisColumnFilter.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(colInfo.getColumnName()))); } allColumnFilters.addFilter(thisColumnFilter); } if (allColumnFilters != null) { addFilter(allColumnFilters); } }
public TaskConfig searchLatestTask(String cp, String type, String value) { // create a filter, that looks for successfully completed tasks: FilterList filters = new FilterList(); // we are looking only in this family: filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); // for the given cp: SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filters.addFilter(cpfilter); filterTaskType(type, value, filters); Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filters); try { HTableInterface tableInterface = table.getTable(); ResultScanner scanner = tableInterface.getScanner(scan); try { Result rr = scanner.next(); if (rr != null && !rr.isEmpty()) { NavigableMap<byte[], byte[]> map = rr.getFamilyMap(COLUMN_ZIEOOK_TASK); Map<String, String> taskmap = new HashMap<String, String>(4); for (Map.Entry<byte[], byte[]> entry : map.entrySet()) { taskmap.put(Bytes.toString(entry.getKey()), Bytes.toString(entry.getValue())); } return new TaskConfig(taskmap); } } finally { scanner.close(); table.putTable(tableInterface); } } catch (IOException e) { LOG.error("failed to SCAN table '" + table.getTableName(null) + "' for recommenders", e); } return null; }
/** * return the oldest task that is successfully completed * @param cp * @param recommender * @return */ public TaskConfig searchCompletedTask(String cp, String type, String value) { // create a filter, that looks for successfully completed tasks: FilterList filters = new FilterList(); // we are looking only in this family: filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); // for the given cp: SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filters.addFilter(cpfilter); filterTaskType(type, value, filters); // And the task should be completed: SingleColumnValueFilter taskokfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.TASK_RESULT), CompareOp.EQUAL, new BinaryComparator( Bytes.toBytes(TaskConfig.TASK_RESULT_OK))); taskokfilter.setFilterIfMissing(true); filters.addFilter(taskokfilter); LOG.debug("scanning for task with: <cp,type,value> = <{},{}>", cp + "," + type, value); Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filters); try { HTableInterface tableInterface = table.getTable(); ResultScanner scanner = tableInterface.getScanner(scan); try { Result rr = scanner.next(); if (rr != null && !rr.isEmpty()) { NavigableMap<byte[], byte[]> map = rr.getFamilyMap(COLUMN_ZIEOOK_TASK); Map<String, String> taskmap = new HashMap<String, String>(4); for (Map.Entry<byte[], byte[]> entry : map.entrySet()) { taskmap.put(Bytes.toString(entry.getKey()), Bytes.toString(entry.getValue())); } return new TaskConfig(taskmap); } } finally { scanner.close(); table.putTable(tableInterface); } } catch (IOException e) { LOG.error("failed to SCAN table '" + table.getTableName(null) + "' for recommenders", e); } return null; }
public List<Long> searchFutureTasks(String cp, String type, String value, long startTime, int size) { // we need a specific filter for TaskConfig.CP and TaskConfig.recommender (key, value) // and are only interested in the taskconfig column: FilterList filters = new FilterList(); filters.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); SingleColumnValueFilter cpfilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CP), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(cp))); cpfilter.setFilterIfMissing(true); filters.addFilter(cpfilter); filterTaskType(type, value, filters); // handle open start: if (startTime != -1) { SingleColumnValueFilter filter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.NEXT), CompareOp.GREATER, new BinaryComparator( Bytes.toBytes(getLongAsString(startTime)))); filter.setFilterIfMissing(true); filters.addFilter(filter); } // assign the filter: Scan scan = new Scan().addFamily(COLUMN_ZIEOOK_TASK).setFilter(filters); final byte[] TASK_ID = Bytes.toBytes(TaskConfig.TASK_ID); // scan!!! List<Long> result = new ArrayList<Long>(); try { HTableInterface tableInterface = table.getTable(); ResultScanner scanner = tableInterface.getScanner(scan); try { int count = 0; // get the rows from the table: for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { KeyValue kv = rr.getColumnLatest(COLUMN_ZIEOOK_TASK, TASK_ID); if (kv != null) { // the task id is stored as a string (like the whole taskconfig) - but we only need this particular // value: - if the parselong fails, there's something really wrong - a coding bug - result.add(Long.parseLong(Bytes.toString(kv.getValue()))); count++; } if (size > 0 && count >= size) { break; } } } finally { scanner.close(); table.putTable(tableInterface); } } catch (IOException e) { LOG.error("failed to SCAN table '" + table.getTableName(null) + "' for recommenders", e); } return result; }
public List<Long> searchTaskByClassName(String classname) { // we need to be able to get the task id from the task: final byte[] TASK_ID = Bytes.toBytes(TaskConfig.TASK_ID); List<Filter> filterlist = new ArrayList<Filter>(4); filterlist.add(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK))); SingleColumnValueFilter cpFilter = new SingleColumnValueFilter(COLUMN_ZIEOOK_TASK, Bytes.toBytes(TaskConfig.CLASSNAME), CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(classname))); cpFilter.setFilterIfMissing(true); filterlist.add(cpFilter); FilterList filter = new FilterList(filterlist); // assign the filter: Scan scan = new Scan().setFilter(filter); // scan!!! List<Long> result = new ArrayList<Long>(); try { HTableInterface tableInterface = table.getTable(); ResultScanner scanner = tableInterface.getScanner(scan); try { // get the rows from the table: for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { KeyValue kv = rr.getColumnLatest(COLUMN_ZIEOOK_TASK, TASK_ID); if (kv != null) { // the task id is stored as a string (like the whole taskconfig) - but we only need this particular // value: - if the parselong fails, there's something really wrong - a coding bug - result.add(Long.parseLong(Bytes.toString(kv.getValue()))); } } } finally { scanner.close(); table.putTable(tableInterface); } } catch (IOException e) { LOG.error("failed to SCAN table '" + table.getTableName(null) + "' for recommenders", e); } return result; }
/** * returns all task id's in batches of size * @param start * @param size * @return */ public List<Long> getAllTasks(long start, long size) { // we need to be able to get the task id from the task: final byte[] TASK_ID = Bytes.toBytes(TaskConfig.TASK_ID); // Scan scan = new Scan(RowKeys.getTaskRow(start)); Filter filter = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(COLUMN_ZIEOOK_TASK)); scan.setFilter(filter); // scan!!! List<Long> result = new ArrayList<Long>(); try { HTableInterface tableInterface = table.getTable(); int i = 0; ResultScanner scanner = tableInterface.getScanner(scan); try { // get the rows from the table: for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { KeyValue kv = rr.getColumnLatest(COLUMN_ZIEOOK_TASK, TASK_ID); if (kv != null) { // the task id is stored as a string (like the whole taskconfig) - but we only need this particular // value: - if the parselong fails, there's something really wrong - a coding bug - result.add(Long.parseLong(Bytes.toString(kv.getValue()))); } i++; if (i == size) { break; } } } finally { scanner.close(); table.putTable(tableInterface); } } catch (IOException e) { LOG.error("failed to SCAN table '" + table.getTableName(null) + "' for recommenders", e); } return result; }
private void initScan() { scan = new Scan(); // Set filters, if any. if (configuredOptions_.hasOption("gt")) { gt_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("gt"))); addRowFilter(CompareOp.GREATER, gt_); } if (configuredOptions_.hasOption("lt")) { lt_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("lt"))); addRowFilter(CompareOp.LESS, lt_); } if (configuredOptions_.hasOption("gte")) { gte_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("gte"))); addRowFilter(CompareOp.GREATER_OR_EQUAL, gte_); } if (configuredOptions_.hasOption("lte")) { lte_ = Bytes.toBytesBinary(Utils.slashisize(configuredOptions_.getOptionValue("lte"))); addRowFilter(CompareOp.LESS_OR_EQUAL, lte_); } // apply any column filters FilterList allColumnFilters = null; for (ColumnInfo colInfo : columnInfo_) { // all column family filters roll up to one parent OR filter if (allColumnFilters == null) { allColumnFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); } // and each filter contains a column family filter FilterList thisColumnFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL); thisColumnFilter.addFilter(new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(colInfo.getColumnFamily()))); if (colInfo.isColumnMap()) { if (LOG.isInfoEnabled()) { LOG.info("Adding family:prefix filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnPrefix())); } // each column map filter consists of a FamilyFilter AND // optionally a PrefixFilter if (colInfo.getColumnPrefix() != null) { thisColumnFilter.addFilter(new ColumnPrefixFilter( colInfo.getColumnPrefix())); } } else { if (LOG.isInfoEnabled()) { LOG.info("Adding family:descriptor filters with values " + Bytes.toString(colInfo.getColumnFamily()) + COLON + Bytes.toString(colInfo.getColumnName())); } // each column value filter consists of a FamilyFilter AND // a QualifierFilter thisColumnFilter.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(colInfo.getColumnName()))); } allColumnFilters.addFilter(thisColumnFilter); } if (allColumnFilters != null) { addFilter(allColumnFilters); } }