/** * Check whether this version should be retained. * There are 4 variables considered: * If this version is past max versions -> skip it * If this kv has expired or was deleted, check min versions * to decide whther to skip it or not. * * Increase the version counter unless this is a delete */ private MatchCode checkVersion(byte type, long timestamp) { if (!CellUtil.isDelete(type)) { currentCount++; } if (currentCount > maxVersions) { return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; // skip to next col } // keep the KV if required by minversions or it is not expired, yet if (currentCount <= minVersions || !isExpired(timestamp)) { setTSAndType(timestamp, type); return ScanQueryMatcher.MatchCode.INCLUDE; } else { return MatchCode.SEEK_NEXT_COL; } }
public void testMatch_ExplicitColumns() throws IOException { //Moving up from the Tracker by using Gets and List<KeyValue> instead //of just byte [] //Expected result List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); expected.add(ScanQueryMatcher.MatchCode.DONE); _testMatch_ExplicitColumns(scan, expected); }
private void runTest(int maxVersions, TreeSet<byte[]> trackColumns, List<byte[]> scannerColumns, List<MatchCode> expected) throws IOException { ColumnTracker exp = new ExplicitColumnTracker( trackColumns, 0, maxVersions, Long.MIN_VALUE); //Initialize result List<ScanQueryMatcher.MatchCode> result = new ArrayList<ScanQueryMatcher.MatchCode>(); long timestamp = 0; //"Match" for(byte [] col : scannerColumns){ result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp, KeyValue.Type.Put.getCode(), false)); } assertEquals(expected.size(), result.size()); for(int i=0; i< expected.size(); i++){ assertEquals(expected.get(i), result.get(i)); } }
@Test public void testGet_SingleVersion() throws IOException{ //Create tracker TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); //Looking for every other columns.add(col2); columns.add(col4); List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col1 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col3 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); // col5 int maxVersions = 1; //Create "Scanner" List<byte[]> scanner = new ArrayList<byte[]>(); scanner.add(col1); scanner.add(col2); scanner.add(col3); scanner.add(col4); scanner.add(col5); runTest(maxVersions, columns, scanner, expected); }
/** * Check whether this version should be retained. There are 4 variables considered: If this * version is past max versions -> skip it If this kv has expired or was deleted, check min * versions to decide whther to skip it or not. Increase the version counter unless this is a * delete */ private MatchCode checkVersion(byte type, long timestamp) { if (!KeyValue.isDelete(type)) { currentCount++; } if (currentCount > maxVersions) { return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; // skip to next col } // keep the KV if required by minversions or it is not expired, yet if (currentCount <= minVersions || !isExpired(timestamp)) { setTSAndType(timestamp, type); return ScanQueryMatcher.MatchCode.INCLUDE; } else { return MatchCode.SEEK_NEXT_COL; } }
public void testMatch_ExplicitColumnsWithLookAhead() throws IOException { //Moving up from the Tracker by using Gets and List<KeyValue> instead //of just byte [] //Expected result List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.SKIP); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.SKIP); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); expected.add(ScanQueryMatcher.MatchCode.DONE); Scan s = new Scan(scan); s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2)); _testMatch_ExplicitColumns(s, expected); }
private void testDropDeletes( byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTime(); // Set time to purge deletes to negative value to avoid it ever happening. ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator); NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, now, from, to, null); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(rows.length); byte[] prevRow = null; for (byte[] row : rows) { if (prevRow == null || !Bytes.equals(prevRow, row)) { qm.setRow(row, 0, (short)row.length); prevRow = row; } actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete))); } assertEquals(expected.length, actual.size()); for (int i = 0; i < expected.length; i++) { if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i)); assertEquals(expected[i], actual.get(i)); } }
private void runTest(int maxVersions, TreeSet<byte[]> trackColumns, List<byte[]> scannerColumns, List<MatchCode> expected, int lookAhead) throws IOException { ColumnTracker exp = new ExplicitColumnTracker( trackColumns, 0, maxVersions, Long.MIN_VALUE, lookAhead); //Initialize result List<ScanQueryMatcher.MatchCode> result = new ArrayList<ScanQueryMatcher.MatchCode>(); long timestamp = 0; //"Match" for(byte [] col : scannerColumns){ result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp, KeyValue.Type.Put.getCode(), false)); } assertEquals(expected.size(), result.size()); for(int i=0; i< expected.size(); i++){ assertEquals(expected.get(i), result.get(i)); } }
@Test public void testGet_SingleVersion() throws IOException{ //Create tracker TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); //Looking for every other columns.add(col2); columns.add(col4); List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col1 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); // col3 expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4 expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW); // col5 int maxVersions = 1; //Create "Scanner" List<byte[]> scanner = new ArrayList<byte[]>(); scanner.add(col1); scanner.add(col2); scanner.add(col3); scanner.add(col4); scanner.add(col5); runTest(maxVersions, columns, scanner, expected, 0); }
/** * Check whether this version should be retained. * There are 4 variables considered: * If this version is past max versions -> skip it * If this kv has expired or was deleted, check min versions * to decide whther to skip it or not. * * Increase the version counter unless this is a delete */ private MatchCode checkVersion(byte type, long timestamp) { if (!KeyValue.isDelete(type)) { currentCount++; } if (currentCount > maxVersions) { return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; // skip to next col } // keep the KV if required by minversions or it is not expired, yet if (currentCount <= minVersions || !isExpired(timestamp)) { setTSAndType(timestamp, type); return ScanQueryMatcher.MatchCode.INCLUDE; } else { return MatchCode.SEEK_NEXT_COL; } }
private void testDropDeletes( byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTimeMillis(); // Set time to purge deletes to negative value to avoid it ever happening. ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator); NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to, null); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(rows.length); byte[] prevRow = null; for (byte[] row : rows) { if (prevRow == null || !Bytes.equals(prevRow, row)) { qm.setRow(row, 0, (short)row.length); prevRow = row; } actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete))); } assertEquals(expected.length, actual.size()); for (int i = 0; i < expected.length; i++) { if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i)); assertEquals(expected[i], actual.get(i)); } }
private MatchCode checkVersionForTwoVersion(long timestamp) { // keep the KV if required by minversions or it is not expired, yet if (timestamp > queryTimestamp) { return ScanQueryMatcher.MatchCode.SKIP; // skip to next col } else if(timestamp > cubeTimestamp){ if(hasGetQueryVersion) return MatchCode.SKIP; else{ hasGetQueryVersion = true; return ScanQueryMatcher.MatchCode.INCLUDE; } }else if(hasGetQueryVersion){ return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL; }else{ return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; } }
private void testDropDeletes( byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTimeMillis(); // Set time to purge deletes to negative value to avoid it ever happening. ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator); NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(rows.length); byte[] prevRow = null; for (byte[] row : rows) { if (prevRow == null || !Bytes.equals(prevRow, row)) { qm.setRow(row, 0, (short)row.length); prevRow = row; } actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete))); } assertEquals(expected.length, actual.size()); for (int i = 0; i < expected.length; i++) { if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i)); assertEquals(expected[i], actual.get(i)); } }
@Override public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type, boolean ignoreCount) throws IOException { assert !CellUtil.isDelete(type); if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; // Check if it is a duplicate timestamp if (sameAsPreviousTS(timestamp)) { // If duplicate, skip this Key return ScanQueryMatcher.MatchCode.SKIP; } int count = this.column.increment(); if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) { // Done with versions for this column ++this.index; resetTS(); if (done()) { // We have served all the requested columns. this.column = null; return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; } // We are done with current column; advance to next column // of interest. this.column = this.columns[this.index]; return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL; } setTS(timestamp); return ScanQueryMatcher.MatchCode.INCLUDE; }
public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset, int qualLength) { doneWithColumn(bytes, offset,qualLength); if (getColumnHint() == null) { return MatchCode.SEEK_NEXT_ROW; } else { return MatchCode.SEEK_NEXT_COL; } }
private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException { long now = EnvironmentEdgeManager.currentTime(); // 2,4,5 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2), now - ttl, now); List<KeyValue> memstore = new ArrayList<KeyValue>(); memstore.add(new KeyValue(row1, fam2, col1, 1, data)); memstore.add(new KeyValue(row1, fam2, col2, 1, data)); memstore.add(new KeyValue(row1, fam2, col3, 1, data)); memstore.add(new KeyValue(row1, fam2, col4, 1, data)); memstore.add(new KeyValue(row1, fam2, col5, 1, data)); memstore.add(new KeyValue(row2, fam1, col1, data)); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(); KeyValue k = memstore.get(0); qm.setRow(k.getRowArray(), k.getRowOffset(), k.getRowLength()); for (KeyValue kv : memstore){ actual.add(qm.match(kv)); } assertEquals(expected.size(), actual.size()); for(int i=0; i< expected.size(); i++){ assertEquals(expected.get(i), actual.get(i)); if(PRINT){ System.out.println("expected "+expected.get(i)+ ", actual " +actual.get(i)); } } }
private void testDropDeletes( byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTime(); // Set time to purge deletes to negative value to avoid it ever happening. ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator); NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, now, from, to, null); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(rows.length); byte[] prevRow = null; for (byte[] row : rows) { if (prevRow == null || !Bytes.equals(prevRow, row)) { qm.setRow(row, 0, (short)row.length); prevRow = row; } actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete))); } assertEquals(expected.length, actual.size()); for (int i = 0; i < expected.length; i++) { if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i)); assertEquals(expected[i], actual.get(i)); } }
public void testCheckColumn_Ok() throws IOException { ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(0, VERSIONS, Long.MIN_VALUE); //Create list of qualifiers List<byte[]> qualifiers = new ArrayList<byte[]>(); qualifiers.add(Bytes.toBytes("qualifier1")); qualifiers.add(Bytes.toBytes("qualifier2")); qualifiers.add(Bytes.toBytes("qualifier3")); qualifiers.add(Bytes.toBytes("qualifier4")); //Setting up expected result List<MatchCode> expected = new ArrayList<MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); List<ScanQueryMatcher.MatchCode> actual = new ArrayList<MatchCode>(); for(byte [] qualifier : qualifiers) { ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1, KeyValue.Type.Put.getCode(), false); actual.add(mc); } //Compare actual with expected for(int i=0; i<expected.size(); i++) { assertEquals(expected.get(i), actual.get(i)); } }
public void testCheckColumn_EnforceVersions() throws IOException { ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(0, VERSIONS, Long.MIN_VALUE); //Create list of qualifiers List<byte[]> qualifiers = new ArrayList<byte[]>(); qualifiers.add(Bytes.toBytes("qualifier1")); qualifiers.add(Bytes.toBytes("qualifier1")); qualifiers.add(Bytes.toBytes("qualifier1")); qualifiers.add(Bytes.toBytes("qualifier2")); //Setting up expected result List<ScanQueryMatcher.MatchCode> expected = new ArrayList<MatchCode>(); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL); expected.add(ScanQueryMatcher.MatchCode.INCLUDE); List<MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>(); long timestamp = 0; for(byte [] qualifier : qualifiers) { MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, ++timestamp, KeyValue.Type.Put.getCode(), false); actual.add(mc); } //Compare actual with expected for(int i=0; i<expected.size(); i++) { assertEquals(expected.get(i), actual.get(i)); } }
/** * Regression test for HBASE-2545 */ @Test public void testInfiniteLoop() throws IOException { TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); columns.addAll(Arrays.asList(new byte[][] { col2, col3, col5 })); List<byte[]> scanner = Arrays.<byte[]>asList( new byte[][] { col1, col4 }); List<ScanQueryMatcher.MatchCode> expected = Arrays.<ScanQueryMatcher.MatchCode>asList( new ScanQueryMatcher.MatchCode[] { ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.SEEK_NEXT_COL }); runTest(1, columns, scanner, expected); }
@Override public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type, boolean ignoreCount) { assert !KeyValue.isDelete(type); if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; // Check if it is a duplicate timestamp if (sameAsPreviousTS(timestamp)) { // If duplicate, skip this Key return ScanQueryMatcher.MatchCode.SKIP; } int count = this.column.increment(); if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) { // Done with versions for this column ++this.index; resetTS(); if (done()) { // We have served all the requested columns. this.column = null; return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW; } // We are done with current column; advance to next column // of interest. this.column = this.columns.get(this.index); return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL; } setTS(timestamp); return ScanQueryMatcher.MatchCode.INCLUDE; }