/** * Run a Get against passed in <code>store</code> on passed <code>row</code>, etc. * * @param store * @param row * @param family * @param tr * @return Get result. * @throws IOException */ private List<Cell> doGet(final Store store, final byte[] row, final Map.Entry<byte[], List<Cell>> family, final TimeRange tr) throws IOException { // Sort the cells so that they match the order that they // appear in the Get results. Otherwise, we won't be able to // find the existing values if the cells are not specified // in order by the client since cells are in an array list. Collections.sort(family.getValue(), store.getComparator()); // Get previous values for all columns in this family Get get = new Get(row); for (Cell cell : family.getValue()) { get.addColumn(family.getKey(), CellUtil.cloneQualifier(cell)); } if (tr != null) get.setTimeRange(tr.getMin(), tr.getMax()); return get(get, false); }
/** * Method may need modification if Get attributes are added or removed in future HBase releases. * * @param originalGet * @return convertedGet * @throws IOException */ Get cloneGetWithoutFamilyMap(Get originalGet) throws IOException { Get convertedGet = new Get(originalGet.getRow()); // from Query convertedGet.setFilter(originalGet.getFilter()); convertedGet.setReplicaId(originalGet.getReplicaId()); convertedGet.setConsistency(originalGet.getConsistency()); // from Get convertedGet.setCacheBlocks(originalGet.getCacheBlocks()); convertedGet.setMaxVersions(originalGet.getMaxVersions()); convertedGet.setMaxResultsPerColumnFamily(originalGet.getMaxResultsPerColumnFamily()); convertedGet.setRowOffsetPerColumnFamily(originalGet.getRowOffsetPerColumnFamily()); convertedGet.setCheckExistenceOnly(originalGet.isCheckExistenceOnly()); convertedGet.setClosestRowBefore(originalGet.isClosestRowBefore()); for (Map.Entry<String, byte[]> attr : originalGet.getAttributesMap().entrySet()) { convertedGet.setAttribute(attr.getKey(), attr.getValue()); } for (Map.Entry<byte[], TimeRange> entry : originalGet.getColumnFamilyTimeRange().entrySet()) { TimeRange tr = entry.getValue(); convertedGet.setColumnFamilyTimeRange(entry.getKey(), tr.getMin(), tr.getMax()); } return convertedGet; }
private RowFilter createTimeRangeFilter(TimeRange timeRange) { TimestampRange.Builder rangeBuilder = TimestampRange.newBuilder(); long lowerBound = BigtableConstants.BIGTABLE_TIMEUNIT.convert( timeRange.getMin(), BigtableConstants.HBASE_TIMEUNIT); rangeBuilder.setStartTimestampMicros(lowerBound); if (timeRange.getMax() != Long.MAX_VALUE) { long upperBound = BigtableConstants.BIGTABLE_TIMEUNIT.convert( timeRange.getMax(), BigtableConstants.HBASE_TIMEUNIT); rangeBuilder.setEndTimestampMicros(upperBound); } return RowFilter.newBuilder() .setTimestampRangeFilter(rangeBuilder) .build(); }
protected UserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns, boolean hasNullColumn, long oldestUnexpiredTS, long now) { super(createStartKey(scan, scanInfo), scanInfo, columns, oldestUnexpiredTS, now); this.hasNullColumn = hasNullColumn; this.filter = scan.getFilter(); if (this.filter != null) { this.versionsAfterFilter = scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), scanInfo.getMaxVersions()); } else { this.versionsAfterFilter = 0; } this.stopRow = scan.getStopRow(); TimeRange timeRange = scan.getColumnFamilyTimeRange().get(scanInfo.getFamily()); if (timeRange == null) { this.tr = scan.getTimeRange(); } else { this.tr = timeRange; } }
/** * Creates the get. * * @param row the row * @param familyMap the family map * @param tr the tr * @param f the f * @return the gets the * @throws IOException Signals that an I/O exception has occurred. */ protected static Get createGet(byte[] row, Map<byte[], NavigableSet<byte[]>> familyMap, TimeRange tr, Filter f ) throws IOException { Get get = new Get(row); if(tr != null){ get.setTimeRange(tr.getMin(), tr.getMax()); } if(f != null) get.setFilter(f); if(familyMap != null){ for(byte[] fam: familyMap.keySet()) { NavigableSet<byte[]> cols = familyMap.get(fam); if( cols == null || cols.size() == 0){ get.addFamily(fam); } else{ for(byte[] col: cols) { get.addColumn(fam, col); } } } } return get; }
/** * Creates the increment. * * @param row the row * @param familyMap the family map * @param tr the tr * @param value the value * @return the increment * @throws IOException Signals that an I/O exception has occurred. */ protected static Increment createIncrement(byte[] row, Map<byte[], NavigableSet<byte[]>> familyMap, TimeRange tr, long value) throws IOException { Increment incr = new Increment(row); if(tr != null){ incr.setTimeRange(tr.getMin(), tr.getMax()); } if(familyMap != null){ for(byte[] fam: familyMap.keySet()) { NavigableSet<byte[]> cols = familyMap.get(fam); for(byte[] col: cols) { incr.addColumn(fam, col, value); } } } return incr; }
/** * Creates the get. * * @param row the row * @param familyMap the family map * @param tr the tr * @param f the f * @return the gets the * @throws IOException Signals that an I/O exception has occurred. */ protected Get createGet(byte[] row, Map<byte[], NavigableSet<byte[]>> familyMap, TimeRange tr, Filter f ) throws IOException { Get get = new Get(row); if(tr != null){ get.setTimeRange(tr.getMin(), tr.getMax()); } if(f != null) get.setFilter(f); if(familyMap != null){ for(byte[] fam: familyMap.keySet()) { NavigableSet<byte[]> cols = familyMap.get(fam); if( cols == null || cols.size() == 0){ get.addFamily(fam); } else{ for(byte[] col: cols) { get.addColumn(fam, col); } } } } return get; }
/** * Creates the increment. * * @param row the row * @param familyMap the family map * @param tr the tr * @param value the value * @return the increment * @throws IOException Signals that an I/O exception has occurred. */ protected Increment createIncrement(byte[] row, Map<byte[], NavigableSet<byte[]>> familyMap, TimeRange tr, long value) throws IOException { Increment incr = new Increment(row); if(tr != null){ incr.setTimeRange(tr.getMin(), tr.getMax()); } if(familyMap != null){ for(byte[] fam: familyMap.keySet()) { NavigableSet<byte[]> cols = familyMap.get(fam); for(byte[] col: cols) { incr.addColumn(fam, col, value); } } } return incr; }
/** * Creates a {@link OGet} (OTS) from a {@link Get} (HBase). * * * @param in the <code>Get</code> to convert * * @return <code>OGet</code> object * @throws IOException */ static OGet toOtsGet(Get in, ColumnMapping columnMapping) throws IOException,UnsupportedOperationException { OGet out = new OGet(in.getRow()); validateMultiFamilySupport(in.getFamilyMap().keySet(), columnMapping, true); if (!in.getFamilyMap().isEmpty()) { Map<byte[], NavigableSet<byte[]>> familyMap = in.getFamilyMap(); for (Map.Entry<byte[], NavigableSet<byte[]>> familyEntry : familyMap .entrySet()) { byte[] family = familyEntry.getKey(); if (familyEntry.getValue() == null || familyEntry.getValue().isEmpty()) { // this means get whole row in OTS } else { for (byte[] qualifier : familyEntry.getValue()) { out.addColumn(columnMapping.getTablestoreColumn(family, qualifier)); } } } } if (in.getMaxVersions() > 1) { out.setMaxVersions(in.getMaxVersions()); } if (!in.getTimeRange().isAllTime()) { TimeRange tr = in.getTimeRange(); out.setTimeRange(tr.getMin(), tr.getMax()); } if (in.getFilter() != null) { out.setFilter(toOtsFilter(in.getFilter(),columnMapping)); } checkGetRowSupport(in); return out; }
@Override public boolean shouldUseScanner(Scan scan, Store store, long oldestUnexpiredTS) { // if the file has no entries, no need to validate or create a scanner. byte[] cf = store.getFamily().getName(); TimeRange timeRange = scan.getColumnFamilyTimeRange().get(cf); if (timeRange == null) { timeRange = scan.getTimeRange(); } return reader.passesTimerangeFilter(timeRange, oldestUnexpiredTS) && reader .passesKeyRangeFilter(scan) && reader.passesBloomFilter(scan, scan.getFamilyMap().get(cf)); }
/** * Check if this memstore may contain the required keys * @param scan scan * @param store holds reference to cf * @param oldestUnexpiredTS * @return False if the key definitely does not exist in this Memstore */ public boolean shouldSeek(Scan scan, Store store, long oldestUnexpiredTS) { byte[] cf = store.getFamily().getName(); TimeRange timeRange = scan.getColumnFamilyTimeRange().get(cf); if (timeRange == null) { timeRange = scan.getTimeRange(); } return (timeRangeTracker.includesTimeRange(timeRange) || snapshotTimeRangeTracker.includesTimeRange(timeRange)) && (Math.max(timeRangeTracker.getMaximumTimestamp(), snapshotTimeRangeTracker.getMaximumTimestamp()) >= oldestUnexpiredTS); }
/** * Do a specific Get on passed <code>columnFamily</code> and column qualifiers from * <code>incrementCoordinates</code> only. * * @param increment * @param columnFamily * @param increments * @return Return the Cells to Increment * @throws IOException */ private List<Cell> getIncrementCurrentValue(final Increment increment, byte[] columnFamily, final List<Cell> increments, final IsolationLevel isolation) throws IOException { Get get = new Get(increment.getRow()); if (isolation != null) get.setIsolationLevel(isolation); for (Cell cell : increments) { get.addColumn(columnFamily, CellUtil.cloneQualifier(cell)); } TimeRange tr = increment.getTimeRange(); if (tr != null) { get.setTimeRange(tr.getMin(), tr.getMax()); } return get(get, false); }
@Test public void testSimpleInRange() { TimeRangeTracker trr = new TimeRangeTracker(); trr.includeTimestamp(0); trr.includeTimestamp(2); assertTrue(trr.includesTimeRange(new TimeRange(1))); }
@Test public void testGet() throws Exception{ byte[] row = "row".getBytes(); byte[] fam = "fam".getBytes(); byte[] qf1 = "qf1".getBytes(); long ts = System.currentTimeMillis(); int maxVersions = 2; Get get = new Get(row); get.addColumn(fam, qf1); get.setTimeRange(ts, ts+1); get.setMaxVersions(maxVersions); ClientProtos.Get getProto = ProtobufUtil.toGet(get); Get desGet = ProtobufUtil.toGet(getProto); assertTrue(Bytes.equals(get.getRow(), desGet.getRow())); Set<byte[]> set = null; Set<byte[]> desSet = null; for(Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()){ assertTrue(desGet.getFamilyMap().containsKey(entry.getKey())); set = entry.getValue(); desSet = desGet.getFamilyMap().get(entry.getKey()); for(byte [] qualifier : set){ assertTrue(desSet.contains(qualifier)); } } assertEquals(get.getMaxVersions(), desGet.getMaxVersions()); TimeRange tr = get.getTimeRange(); TimeRange desTr = desGet.getTimeRange(); assertEquals(tr.getMax(), desTr.getMax()); assertEquals(tr.getMin(), desTr.getMin()); }
private static HBaseProtos.TimeRange.Builder timeRangeToProto(TimeRange timeRange) { HBaseProtos.TimeRange.Builder timeRangeBuilder = HBaseProtos.TimeRange.newBuilder(); timeRangeBuilder.setFrom(timeRange.getMin()); timeRangeBuilder.setTo(timeRange.getMax()); return timeRangeBuilder; }
private static TimeRange protoToTimeRange(HBaseProtos.TimeRange timeRange) throws IOException { long minStamp = 0; long maxStamp = Long.MAX_VALUE; if (timeRange.hasFrom()) { minStamp = timeRange.getFrom(); } if (timeRange.hasTo()) { maxStamp = timeRange.getTo(); } return new TimeRange(minStamp, maxStamp); }
/** * Get versions of columns with the specified timestamp. Note, default maximum * versions to return is 1. If your time range spans more than one version * and you want all versions returned, up the number of versions beyond the * defaut. * @param timestamp version timestamp * @see #setMaxVersions() * @see #setMaxVersions(int) * @return this */ public Scan setTimeStamp(long timestamp) throws IOException { try { tr = new TimeRange(timestamp, timestamp+1); } catch(Exception e) { // This should never happen, unless integer overflow or something extremely wrong... LOG.error("TimeRange failed, likely caused by integer overflow. ", e); throw e; } return this; }
/** * Get versions of columns with the specified timestamp. * @param timestamp version timestamp * @return this for invocation chaining */ public Get setTimeStamp(long timestamp) throws IOException { try { tr = new TimeRange(timestamp, timestamp+1); } catch(Exception e) { // This should never happen, unless integer overflow or something extremely wrong... LOG.error("TimeRange failed, likely caused by integer overflow. ", e); throw e; } return this; }
/** * Creates a new instance of this class while copying all values. * @param scan The scan instance to copy from. * @throws IOException When copying the values fails. */ public Scan(Scan scan) throws IOException { startRow = scan.getStartRow(); stopRow = scan.getStopRow(); maxVersions = scan.getMaxVersions(); batch = scan.getBatch(); caching = scan.getCaching(); cacheBlocks = scan.getCacheBlocks(); filter = scan.getFilter(); // clone? TimeRange ctr = scan.getTimeRange(); tr = new TimeRange(ctr.getMin(), ctr.getMax()); Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap(); for (Map.Entry<byte[], NavigableSet<byte[]>> entry : fams.entrySet()) { byte[] fam = entry.getKey(); NavigableSet<byte[]> cols = entry.getValue(); if (cols != null && cols.size() > 0) { for (byte[] col : cols) { addColumn(fam, col); } } else { addFamily(fam); } } for (Map.Entry<String, byte[]> attr : scan.getAttributesMap().entrySet()) { setAttribute(attr.getKey(), attr.getValue()); } }
public void readFields(final DataInput in) throws IOException { int version = in.readByte(); if (version > (int) SCAN_VERSION) { throw new IOException("version not supported"); } this.startRow = Bytes.readByteArray(in); this.stopRow = Bytes.readByteArray(in); this.maxVersions = in.readInt(); this.batch = in.readInt(); this.caching = in.readInt(); this.cacheBlocks = in.readBoolean(); if (in.readBoolean()) { this.filter = Classes.createWritableForName(Bytes.toString(Bytes.readByteArray(in))); this.filter.readFields(in); } this.tr = new TimeRange(); tr.readFields(in); int numFamilies = in.readInt(); this.familyMap = new TreeMap<byte[], NavigableSet<byte[]>>(Bytes.BYTES_COMPARATOR); for (int i = 0; i < numFamilies; i++) { byte[] family = Bytes.readByteArray(in); int numColumns = in.readInt(); TreeSet<byte[]> set = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); for (int j = 0; j < numColumns; j++) { byte[] qualifier = Bytes.readByteArray(in); set.add(qualifier); } this.familyMap.put(family, set); } if (version > 1) { readAttributes(in); } }
/** * Get versions of columns with the specified timestamp. * @param timestamp version timestamp * @return this for invocation chaining */ public Get setTimeStamp(long timestamp) { try { tr = new TimeRange(timestamp, timestamp+1); } catch(IOException e) { // Will never happen } return this; }
public void readFields(final DataInput in) throws IOException { int version = in.readByte(); if (version > GET_VERSION) { throw new IOException("unsupported version"); } this.row = Bytes.readByteArray(in); this.lockId = in.readLong(); this.maxVersions = in.readInt(); boolean hasFilter = in.readBoolean(); if (hasFilter) { this.filter = Classes.createWritableForName( Bytes.toString(Bytes.readByteArray(in))); this.filter.readFields(in); } this.cacheBlocks = in.readBoolean(); this.tr = new TimeRange(); tr.readFields(in); int numFamilies = in.readInt(); this.familyMap = new TreeMap<byte [],NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR); for(int i=0; i<numFamilies; i++) { byte [] family = Bytes.readByteArray(in); boolean hasColumns = in.readBoolean(); NavigableSet<byte []> set = null; if(hasColumns) { int numColumns = in.readInt(); set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR); for(int j=0; j<numColumns; j++) { byte [] qualifier = Bytes.readByteArray(in); set.add(qualifier); } } this.familyMap.put(family, set); } readAttributes(in); }
public void readFields(final DataInput in) throws IOException { int version = in.readByte(); if (version > INCREMENT_VERSION) { throw new IOException("unsupported version"); } this.row = Bytes.readByteArray(in); this.tr = new TimeRange(); tr.readFields(in); this.lockId = in.readLong(); int numFamilies = in.readInt(); if (numFamilies == 0) { throw new IOException("At least one column required"); } this.familyMap = new TreeMap<byte [],NavigableMap<byte [], Long>>(Bytes.BYTES_COMPARATOR); for(int i=0; i<numFamilies; i++) { byte [] family = Bytes.readByteArray(in); boolean hasColumns = in.readBoolean(); NavigableMap<byte [], Long> set = null; if(hasColumns) { int numColumns = in.readInt(); set = new TreeMap<byte [], Long>(Bytes.BYTES_COMPARATOR); for(int j=0; j<numColumns; j++) { byte [] qualifier = Bytes.readByteArray(in); set.put(qualifier, in.readLong()); } } else { throw new IOException("At least one column required per family"); } this.familyMap.put(family, set); } if (version > 1) { this.writeToWAL = in.readBoolean(); } }
@Test public void testGet() throws Exception{ byte[] row = "row".getBytes(); byte[] fam = "fam".getBytes(); byte[] qf1 = "qf1".getBytes(); long ts = System.currentTimeMillis(); int maxVersions = 2; long lockid = 5; RowLock rowLock = new RowLock(lockid); Get get = new Get(row, rowLock); get.addColumn(fam, qf1); get.setTimeRange(ts, ts+1); get.setMaxVersions(maxVersions); byte[] sb = Writables.getBytes(get); Get desGet = (Get)Writables.getWritable(sb, new Get()); assertTrue(Bytes.equals(get.getRow(), desGet.getRow())); Set<byte[]> set = null; Set<byte[]> desSet = null; for(Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()){ assertTrue(desGet.getFamilyMap().containsKey(entry.getKey())); set = entry.getValue(); desSet = desGet.getFamilyMap().get(entry.getKey()); for(byte [] qualifier : set){ assertTrue(desSet.contains(qualifier)); } } assertEquals(get.getLockId(), desGet.getLockId()); assertEquals(get.getMaxVersions(), desGet.getMaxVersions()); TimeRange tr = get.getTimeRange(); TimeRange desTr = desGet.getTimeRange(); assertEquals(tr.getMax(), desTr.getMax()); assertEquals(tr.getMin(), desTr.getMin()); }
@Test public void testTimeRange() throws Exception{ TimeRange tr = new TimeRange(0,5); byte [] mb = Writables.getBytes(tr); TimeRange deserializedTr = (TimeRange)Writables.getWritable(mb, new TimeRange()); assertEquals(tr.getMax(), deserializedTr.getMax()); assertEquals(tr.getMin(), deserializedTr.getMin()); }
/** * Creates a new instance of this class while copying all values. * * @param scan The scan instance to copy from. * @throws IOException When copying the values fails. */ public Scan(Scan scan) throws IOException { startRow = scan.getStartRow(); stopRow = scan.getStopRow(); maxVersions = scan.getMaxVersions(); batch = scan.getBatch(); storeLimit = scan.getMaxResultsPerColumnFamily(); storeOffset = scan.getRowOffsetPerColumnFamily(); caching = scan.getCaching(); maxResultSize = scan.getMaxResultSize(); cacheBlocks = scan.getCacheBlocks(); getScan = scan.isGetScan(); filter = scan.getFilter(); // clone? loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue(); consistency = scan.getConsistency(); reversed = scan.isReversed(); small = scan.isSmall(); TimeRange ctr = scan.getTimeRange(); tr = new TimeRange(ctr.getMin(), ctr.getMax()); Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap(); for (Map.Entry<byte[],NavigableSet<byte[]>> entry : fams.entrySet()) { byte [] fam = entry.getKey(); NavigableSet<byte[]> cols = entry.getValue(); if (cols != null && cols.size() > 0) { for (byte[] col : cols) { addColumn(fam, col); } } else { addFamily(fam); } } for (Map.Entry<String, byte[]> attr : scan.getAttributesMap().entrySet()) { setAttribute(attr.getKey(), attr.getValue()); } }
/** * Get versions of columns with the specified timestamp. Note, default maximum * versions to return is 1. If your time range spans more than one version * and you want all versions returned, up the number of versions beyond the * defaut. * @param timestamp version timestamp * @see #setMaxVersions() * @see #setMaxVersions(int) * @return this */ public Scan setTimeStamp(long timestamp) throws IOException { try { tr = new TimeRange(timestamp, timestamp+1); } catch(IOException e) { // This should never happen, unless integer overflow or something extremely wrong... LOG.error("TimeRange failed, likely caused by integer overflow. ", e); throw e; } return this; }
/** * Get versions of columns with the specified timestamp. * @param timestamp version timestamp * @return this for invocation chaining */ public Get setTimeStamp(long timestamp) throws IOException { try { tr = new TimeRange(timestamp, timestamp+1); } catch(IOException e) { // This should never happen, unless integer overflow or something extremely wrong... LOG.error("TimeRange failed, likely caused by integer overflow. ", e); throw e; } return this; }
/** * Creates a new instance of this class while copying all values. * * @param scan The scan instance to copy from. * @throws IOException When copying the values fails. */ public Scan(Scan scan) throws IOException { startRow = scan.getStartRow(); stopRow = scan.getStopRow(); maxVersions = scan.getMaxVersions(); batch = scan.getBatch(); storeLimit = scan.getMaxResultsPerColumnFamily(); storeOffset = scan.getRowOffsetPerColumnFamily(); caching = scan.getCaching(); maxResultSize = scan.getMaxResultSize(); cacheBlocks = scan.getCacheBlocks(); getScan = scan.isGetScan(); filter = scan.getFilter(); // clone? loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue(); TimeRange ctr = scan.getTimeRange(); tr = new TimeRange(ctr.getMin(), ctr.getMax()); Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap(); for (Map.Entry<byte[],NavigableSet<byte[]>> entry : fams.entrySet()) { byte [] fam = entry.getKey(); NavigableSet<byte[]> cols = entry.getValue(); if (cols != null && cols.size() > 0) { for (byte[] col : cols) { addColumn(fam, col); } } else { addFamily(fam); } } for (Map.Entry<String, byte[]> attr : scan.getAttributesMap().entrySet()) { setAttribute(attr.getKey(), attr.getValue()); } }
public void readFields(final DataInput in) throws IOException { int version = in.readByte(); if (version > (int)SCAN_VERSION) { throw new IOException("version not supported"); } this.startRow = Bytes.readByteArray(in); this.stopRow = Bytes.readByteArray(in); this.maxVersions = in.readInt(); this.batch = in.readInt(); this.caching = in.readInt(); this.cacheBlocks = in.readBoolean(); if(in.readBoolean()) { this.filter = Classes.createWritableForName( Bytes.toString(Bytes.readByteArray(in))); this.filter.readFields(in); } this.tr = new TimeRange(); tr.readFields(in); int numFamilies = in.readInt(); this.familyMap = new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR); for(int i=0; i<numFamilies; i++) { byte [] family = Bytes.readByteArray(in); int numColumns = in.readInt(); TreeSet<byte []> set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR); for(int j=0; j<numColumns; j++) { byte [] qualifier = Bytes.readByteArray(in); set.add(qualifier); } this.familyMap.put(family, set); } if (version > 1) { readAttributes(in); } }