protected List<List<Cell>> getExistingLabelsWithAuths() throws IOException { Scan scan = new Scan(); RegionScanner scanner = labelsRegion.getScanner(scan); List<List<Cell>> existingLabels = new ArrayList<List<Cell>>(); try { while (true) { List<Cell> cells = new ArrayList<Cell>(); scanner.next(cells); if (cells.isEmpty()) { break; } existingLabels.add(cells); } } finally { scanner.close(); } return existingLabels; }
@Test public void testHBASE12817() throws IOException { for (int i = 0; i < 100; i++) { region .put(new Put(Bytes.toBytes("obj" + (2900 + i))).addColumn(fam, qual1, Bytes.toBytes(i))); } region.put(new Put(Bytes.toBytes("obj299")).addColumn(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj29")).addColumn(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj2")).addColumn(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj3")).addColumn(fam, qual1, Bytes.toBytes("whatever"))); region.flush(true); Scan scan = new Scan(Bytes.toBytes("obj29995")); RegionScanner scanner = region.getScanner(scan); List<Cell> cells = new ArrayList<Cell>(); assertFalse(scanner.next(cells)); assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow()); }
@Override public RegionScanner postScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { byte[] byteTransaction = scan.getAttribute(CellUtils.TRANSACTION_ATTRIBUTE); if (byteTransaction == null) { return s; } TSOProto.Transaction transaction = TSOProto.Transaction.parseFrom(byteTransaction); long id = transaction.getTimestamp(); long readTs = transaction.getReadTimestamp(); long epoch = transaction.getEpoch(); VisibilityLevel visibilityLevel = VisibilityLevel.fromInteger(transaction.getVisibilityLevel()); HBaseTransaction hbaseTransaction = new HBaseTransaction(id, readTs, visibilityLevel, epoch, new HashSet<HBaseCellId>(), null); RegionAccessWrapper regionAccessWrapper = new RegionAccessWrapper(HBaseShims.getRegionCoprocessorRegion(e.getEnvironment())); snapshotFilter.setTableAccessWrapper(regionAccessWrapper); return new OmidRegionScanner(snapshotFilter, s, hbaseTransaction, 1); }
@Test public void testHBASE12817() throws IOException { for (int i = 0; i < 100; i++) { region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).add(fam, qual1, Bytes.toBytes(i))); } region.put(new Put(Bytes.toBytes("obj299")).add(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj29")).add(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj2")).add(fam, qual1, Bytes.toBytes("whatever"))); region.put(new Put(Bytes.toBytes("obj3")).add(fam, qual1, Bytes.toBytes("whatever"))); region.flushcache(); Scan scan = new Scan(Bytes.toBytes("obj29995")); RegionScanner scanner = region.getScanner(scan); List<Cell> cells = new ArrayList<Cell>(); assertFalse(scanner.next(cells)); assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow()); }
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { HRegion region = e.getEnvironment().getRegion(); Authorizations authorizations = null; // If a super user issues a scan, he should be able to scan the cells // irrespective of the Visibility labels if (checkIfScanOrGetFromSuperUser()) { return s; } try { authorizations = scan.getAuthorizations(); } catch (DeserializationException de) { throw new IOException(de); } Filter visibilityLabelFilter = createVisibilityLabelFilter(region, authorizations); if (visibilityLabelFilter != null) { Filter filter = scan.getFilter(); if (filter != null) { scan.setFilter(new FilterList(filter, visibilityLabelFilter)); } else { scan.setFilter(visibilityLabelFilter); } } return s; }
private List<List<Cell>> getExistingLabelsWithAuths() throws IOException { Scan scan = new Scan(); RegionScanner scanner = this.regionEnv.getRegion().getScanner(scan); List<List<Cell>> existingLabels = new ArrayList<List<Cell>>(); try { while (true) { List<Cell> cells = new ArrayList<Cell>(); scanner.next(cells); if (cells.isEmpty()) { break; } existingLabels.add(cells); } } finally { scanner.close(); } return existingLabels; }
private List<String> getUserAuthsFromLabelsTable(byte[] user) throws IOException { Scan s = new Scan(); s.addColumn(LABELS_TABLE_FAMILY, user); Filter filter = createVisibilityLabelFilter(this.regionEnv.getRegion(), new Authorizations( SYSTEM_LABEL)); s.setFilter(filter); List<String> auths = new ArrayList<String>(); // We do ACL check here as we create scanner directly on region. It will not make calls to // AccessController CP methods. performACLCheck(); RegionScanner scanner = this.regionEnv.getRegion().getScanner(s); List<Cell> results = new ArrayList<Cell>(1); while (true) { scanner.next(results); if (results.isEmpty()) break; Cell cell = results.get(0); int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); String label = this.visibilityManager.getLabel(ordinal); if (label != null) { auths.add(label); } results.clear(); } return auths; }
@Override public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e, final Scan scan, final RegionScanner s) throws IOException { try { Long themisStartTs = getStartTsFromAttribute(scan); if (themisStartTs != null) { ThemisCpUtil.prepareScan(scan, e.getEnvironment().getRegion().getTableDesc().getFamilies()); checkFamily(e.getEnvironment().getRegion(), scan); ThemisProtocolImpl.checkReadTTL(System.currentTimeMillis(), themisStartTs, PRE_SCANNER_OPEN_FEEK_ROW); Scan internalScan = ThemisCpUtil.constructLockAndWriteScan(scan, themisStartTs); ThemisServerScanner pScanner = new ThemisServerScanner(e.getEnvironment().getRegion() .getScanner(internalScan), internalScan, themisStartTs, scan); e.bypass(); return pScanner; } return s; } catch (Throwable ex) { throw new DoNotRetryIOException("themis exception in preScannerOpen", ex); } }
protected List<List<Cell>> getExistingLabelsWithAuths() throws IOException { Scan scan = new Scan(); RegionScanner scanner = labelsRegion.getScanner(scan); List<List<Cell>> existingLabels = new ArrayList<>(); try { while (true) { List<Cell> cells = new ArrayList<>(); scanner.next(cells); if (cells.isEmpty()) { break; } existingLabels.add(cells); } } finally { scanner.close(); } return existingLabels; }
private void initiateScan(HRegion region) throws IOException { Scan scan = new Scan(); scan.setCaching(1); RegionScanner resScanner = null; try { resScanner = region.getScanner(scan); List<Cell> results = new ArrayList<>(); boolean next = resScanner.next(results); try { counter.incrementAndGet(); latch.await(); } catch (InterruptedException e) { } while (next) { next = resScanner.next(results); } } finally { scanCompletedCounter.incrementAndGet(); resScanner.close(); } }
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, // Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException { if (context.isCoprocessorEnabled() == false) { return table.getScanner(scan); } CoprocessorRowType type = CoprocessorRowType.fromCuboid(segment, cuboid); CoprocessorFilter filter = CoprocessorFilter.fromFilter(segment, tupleFiler); CoprocessorProjector projector = CoprocessorProjector.makeForObserver(segment, cuboid, groupBy); ObserverAggregators aggrs = ObserverAggregators.fromValueDecoders(rowValueDecoders); if (DEBUG_LOCAL_COPROCESSOR) { RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan)); AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner); return new ResultScannerAdapter(aggrScanner); } else { scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 }); scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type)); scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector)); scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs)); scan.setAttribute(AggregateRegionObserver.FILTER, CoprocessorFilter.serialize(filter)); return table.getScanner(scan); } }
@Override public final RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException { boolean copAbortOnError = ctxt.getEnvironment().getConfiguration().getBoolean(RegionCoprocessorHost.ABORT_ON_ERROR_KEY, RegionCoprocessorHost.DEFAULT_ABORT_ON_ERROR); // never throw out exception that could abort region server if (copAbortOnError) { try { return doPostScannerObserver(ctxt, scan, innerScanner); } catch (Throwable e) { LOG.error("Kylin Coprocessor Error", e); return innerScanner; } } else { return doPostScannerObserver(ctxt, scan, innerScanner); } }
/** * Test region scanner. * * @throws IOException Signals that an I/O exception has occurred. */ public void testRegionScanner() throws IOException { LOG.info("Test Region scanner"); Scan scan = new Scan(); scan.setStartRow(region.getStartKey()); scan.setStopRow(region.getEndKey()); RegionScanner scanner = region.getScanner(scan); //Store store = region.getStore(CF); //StoreScanner scanner = new StoreScanner(store, store.getScanInfo(), scan, null); long start = System.currentTimeMillis(); int total = 0; List<Cell> result = new ArrayList<Cell>(); while(scanner.next(result)){ total++; result.clear(); } LOG.info("Test Region scanner finished. Found "+total +" in "+(System.currentTimeMillis() - start)+"ms"); LOG.info("cache hits ="+cache.getStats().getHitCount()+" miss="+cache.getStats().getMissCount()); }
/** * refresh underlying RegionScanner we call this when new store file gets * created by MemStore flushes or current scanner fails due to compaction */ public void updateScanner() throws IOException { if (LOG.isDebugEnabled()) { SpliceLogUtils.debug(LOG, "updateScanner with hregionInfo=%s, tableName=%s, rootDir=%s, scan=%s", hri, htd.getNameAsString(), rootDir, scan); } if (flushed) { if (LOG.isDebugEnabled()) SpliceLogUtils.debug(LOG, "Flush occurred"); if (this.topCell != null) { if (LOG.isDebugEnabled()) SpliceLogUtils.debug(LOG, "setting start row to %s", topCell); //noinspection deprecation scan.setStartRow(Bytes.add(topCell.getRow(), new byte[]{0})); } } memScannerList.add(getMemStoreScanner()); this.region = openHRegion(); RegionScanner regionScanner = new CountingRegionScanner(BaseHRegionUtil.getScanner(region, scan, memScannerList), region, scan); if (flushed) { if (scanner != null) scanner.close(); } scanner = regionScanner; }
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { if (!initialized) { throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"); } // Nothing to do if authorization is not enabled if (!authorizationEnabled) { return s; } Region region = e.getEnvironment().getRegion(); Authorizations authorizations = null; try { authorizations = scan.getAuthorizations(); } catch (DeserializationException de) { throw new IOException(de); } if (authorizations == null) { // No Authorizations present for this scan/Get! // In case of system tables other than "labels" just scan with out visibility check and // filtering. Checking visibility labels for META and NAMESPACE table is not needed. TableName table = region.getRegionInfo().getTable(); if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) { return s; } } Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region, authorizations); if (visibilityLabelFilter != null) { Filter filter = scan.getFilter(); if (filter != null) { scan.setFilter(new FilterList(filter, visibilityLabelFilter)); } else { scan.setFilter(visibilityLabelFilter); } } return s; }
@Override public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException { User user = VisibilityUtils.getActiveUser(); if (user != null && user.getShortName() != null) { scannerOwners.put(s, user.getShortName()); } return s; }
@Override public List<String> getUserAuths(byte[] user, boolean systemCall) throws IOException { assert (labelsRegion != null || systemCall); if (systemCall || labelsRegion == null) { return this.labelsCache.getUserAuths(Bytes.toString(user)); } Scan s = new Scan(); if (user != null && user.length > 0) { s.addColumn(LABELS_TABLE_FAMILY, user); } Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion, new Authorizations(SYSTEM_LABEL)); s.setFilter(filter); ArrayList<String> auths = new ArrayList<String>(); RegionScanner scanner = this.labelsRegion.getScanner(s); try { List<Cell> results = new ArrayList<Cell>(1); while (true) { scanner.next(results); if (results.isEmpty()) break; Cell cell = results.get(0); int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); String label = this.labelsCache.getLabel(ordinal); if (label != null) { auths.add(label); } results.clear(); } } finally { scanner.close(); } return auths; }
@Override public List<String> getGroupAuths(String[] groups, boolean systemCall) throws IOException { assert (labelsRegion != null || systemCall); if (systemCall || labelsRegion == null) { return this.labelsCache.getGroupAuths(groups); } Scan s = new Scan(); if (groups != null && groups.length > 0) { for (String group : groups) { s.addColumn(LABELS_TABLE_FAMILY, Bytes.toBytes(AuthUtil.toGroupEntry(group))); } } Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion, new Authorizations(SYSTEM_LABEL)); s.setFilter(filter); Set<String> auths = new HashSet<String>(); RegionScanner scanner = this.labelsRegion.getScanner(s); try { List<Cell> results = new ArrayList<Cell>(1); while (true) { scanner.next(results); if (results.isEmpty()) break; Cell cell = results.get(0); int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); String label = this.labelsCache.getLabel(ordinal); if (label != null) { auths.add(label); } results.clear(); } } finally { scanner.close(); } return new ArrayList<String>(auths); }
@Override public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException { User user = getActiveUser(); if (user != null && user.getShortName() != null) { // store reference to scanner owner for later checks scannerOwners.put(s, user.getShortName()); } return s; }
public LCIndexMemStoreScanner2(RegionScanner regionScanner, IndexTableRelation relation, ScanRange primaryRange) throws IOException { super(); long start = System.currentTimeMillis(); IndexTableRelation copyRelation = new IndexTableRelation(relation.getTableName(), IndexType.LCIndex); copyRelation.addIndexColumn(primaryRange.getFamily(), primaryRange.getQualifier()); copyRelation.setColumns(relation.getFamilyMap()); IndexPutParser parser = IndexPutParser.getParser(IndexType.LCIndex, copyRelation); dataList = init(regionScanner, primaryRange, parser); currentIndexPoint = 0; System.out.println("LCDBG, LCIndexScanner cost " + (System.currentTimeMillis() - start) / 1000.0 + " seconds to build LCIndexMemstoreScanner2 from memstore, the size of this scanner is: " + dataList.size()); }
private ArrayList<Cell> init(RegionScanner regionScanner, ScanRange primaryRange, IndexPutParser parse) throws IOException { ArrayList<Cell> ret = new ArrayList<>(); while (true) { List<Cell> oneRow = new ArrayList<>(); if (!regionScanner.nextRaw(oneRow)) break; processOneRow(ret, oneRow, parse); // processOneRow(ret, oneRow, primaryRange, relation); } Collections.sort(ret, KeyValue.COMPARATOR); return ret; }
@Override public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException { ctPreScannerOpen.incrementAndGet(); return null; }
@Override public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException { ctPostScannerOpen.incrementAndGet(); return s; }
private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException { String cf = "f"; Scan scan = new Scan(); scan.addFamily(cf.getBytes()); scan.setFilter(filter); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes()); HRegion first = regions.get(0); first.getScanner(scan); RegionScanner scanner = first.getScanner(scan); List<Cell> results = new ArrayList<Cell>(); // Result result; long timeBeforeScan = System.currentTimeMillis(); int found = 0; while (scanner.next(results)) { found += results.size(); results.clear(); } found += results.size(); long scanTime = System.currentTimeMillis() - timeBeforeScan; scanner.close(); LOG.info("\nscan time = " + scanTime + "ms"); LOG.info("found " + found + " results\n"); assertEquals(expectedSize, found); }
@Test public void testReverseScanWithoutPadding() throws Exception { byte[] row1 = Bytes.toBytes("a"); byte[] row2 = Bytes.toBytes("ab"); byte[] row3 = Bytes.toBytes("b"); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<Cell>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(Bytes.toString(res.get(0).getRowArray(), res.get(0).getRowOffset(), res.get(0) .getRowLength()), "b"); assertEquals(Bytes.toString(res.get(1).getRowArray(), res.get(1).getRowOffset(), res.get(1) .getRowLength()), "ab"); assertEquals(Bytes.toString(res.get(2).getRowArray(), res.get(2).getRowOffset(), res.get(2) .getRowLength()), "a"); assertEquals(3, count); }
@Test public void testReverseScanWithPadding() throws Exception { byte[] terminator = new byte[] { -1 }; byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator); byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator); byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<Cell>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(3, count); }
@Override public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e, final Scan scan, final RegionScanner s) throws IOException { if (countOfOpen.incrementAndGet() == 2) { //slowdown openScanner randomly slowdownCode(e); } return s; }
public OmidRegionScanner(SnapshotFilterImpl snapshotFilter, RegionScanner s, HBaseTransaction transaction, int maxVersions) { this.snapshotFilter = snapshotFilter; this.scanner = s; this.transaction = transaction; this.maxVersions = maxVersions; this.familyDeletionCache = new HashMap<String, List<Cell>>(); }
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { Transaction tx = getFromOperation(scan); if (tx != null) { projectFamilyDeletes(scan); scan.setMaxVersions(); scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData), TxUtils.getMaxVisibleTimestamp(tx)); Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter()); scan.setFilter(newFilter); } return s; }
@SuppressWarnings("StatementWithEmptyBody") private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception { try (RegionScanner regionScanner = region.getScanner(scan)) { List<Cell> results = Lists.newArrayList(); while (regionScanner.next(results)) { } assertEquals(expected, results); } }
