@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows)); scan.setCaching(opts.caching); FilterList list = new FilterList(); if (opts.addColumns) { scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); } else { scan.addFamily(FAMILY_NAME); } if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); ResultScanner s = this.table.getScanner(scan); for (Result rr; (rr = s.next()) != null;) { updateValueSize(rr); } s.close(); }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows)); FilterList list = new FilterList(); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); ResultScanner s = this.table.getScanner(scan); for (Result rr; (rr = s.next()) != null;) { updateValueSize(rr); } s.close(); }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows)) .setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType); FilterList list = new FilterList(); if (opts.addColumns) { scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); } else { scan.addFamily(FAMILY_NAME); } if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); ResultScanner s = this.table.getScanner(scan); for (Result rr; (rr = s.next()) != null;) { updateValueSize(rr); } s.close(); }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns) throws IOError, TException { try { HTable table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if(columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan)); } catch (IOException e) { throw new IOError(e.getMessage()); } }
/** * Returns the {@link Flow} instance matching the application ID and run ID. * * @param cluster the cluster identifier * @param user the user running the jobs * @param appId the application description * @param runId the specific run ID for the flow * @param populateTasks whether or not to populate the task details for each * job * @return */ public Flow getFlow(String cluster, String user, String appId, long runId, boolean populateTasks) throws IOException { Flow flow = null; byte[] startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(cluster), Bytes.toBytes(user), Bytes.toBytes(appId), Bytes.toBytes(FlowKey.encodeRunId(runId)), Constants.EMPTY_BYTES); LOG.info( "Reading job_history rows start at " + Bytes.toStringBinary(startRow)); Scan scan = new Scan(); // start scanning history at cluster!user!app!run! scan.setStartRow(startRow); // require that all results match this flow prefix scan.setFilter(new WhileMatchFilter(new PrefixFilter(startRow))); List<Flow> flows = createFromResults(scan, populateTasks, 1); if (flows.size() > 0) { flow = flows.get(0); } return flow; }
/** * Returns the {@link Flow} instance containing the given job ID. * * @param cluster the cluster identifier * @param jobId the job identifier * @return */ public Flow getFlowByJobID(String cluster, String jobId, boolean populateTasks) throws IOException { Flow flow = null; JobKey key = idService.getJobKeyById(new QualifiedJobId(cluster, jobId)); if (key != null) { byte[] startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(key.getCluster()), Bytes.toBytes(key.getUserName()), Bytes.toBytes(key.getAppId()), Bytes.toBytes(key.getEncodedRunId()), Constants.EMPTY_BYTES); LOG.info("Reading job_history rows start at " + Bytes.toStringBinary(startRow)); Scan scan = new Scan(); // start scanning history at cluster!user!app!run! scan.setStartRow(startRow); // require that all results match this flow prefix scan.setFilter(new WhileMatchFilter(new PrefixFilter(startRow))); List<Flow> flows = createFromResults(scan, populateTasks, 1); if (flows.size() > 0) { flow = flows.get(0); } } return flow; }
/** * creates a scan for flow data * @param rowPrefix - start row prefix * @param limit - limit on scanned results * @param version - version to match * @return Scan */ private Scan createFlowScan(byte[] rowPrefix, int limit, String version) { Scan scan = new Scan(); scan.setStartRow(rowPrefix); // using a large scanner caching value with a small limit can mean we scan a // lot more data than necessary, so lower the caching for low limits scan.setCaching(Math.min(limit, defaultScannerCaching)); // require that all rows match the prefix we're looking for Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix)); // if version is passed, restrict the rows returned to that version if (version != null && version.length() > 0) { FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); filters.addFilter(prefixFilter); filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(version))); scan.setFilter(filters); } else { scan.setFilter(prefixFilter); } return scan; }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { Table table = null; try { table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw new IOError(Throwables.getStackTraceAsString(e)); } finally{ closeTable(table); } }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, this.totalRows)); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); scan.setFilter(new WhileMatchFilter(new PageFilter(120))); ResultScanner s = this.table.getScanner(scan); s.close(); }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { try { HTable table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw new IOError(e.getMessage()); } }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, this.totalRows)); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); scan.setFilter(new WhileMatchFilter(new PageFilter(120))); ResultScanner s = this.table.getScanner(scan); for (Result rr; (rr = s.next()) != null;) ; s.close(); }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, this.totalRows)); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); scan.setFilter(new WhileMatchFilter(new PageFilter(120))); ResultScanner s = this.table.getScanner(scan); //int count = 0; for (Result rr = null; (rr = s.next()) != null;) { // LOG.info("" + count++ + " " + rr.toString()); } s.close(); }
@Override void testRow(final int i) throws IOException { Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows)); FilterList list = new FilterList(); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); ResultScanner s = this.table.getScanner(scan); for (Result rr; (rr = s.next()) != null;) ; s.close(); }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { Table table = null; try { table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = CellUtil.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw getIOError(e); } finally{ closeTable(table); } }
/** * Returns a Scan instance to retrieve all the task rows for a given job from * the job_history_task table. * @param jobKey the job key to match for all task rows * @return a {@code Scan} instance for the job_history_task table */ private Scan getTaskScan(JobKey jobKey) { byte[] startKey = Bytes.add(jobKeyConv.toBytes(jobKey), Constants.SEP_BYTES); Scan scan = new Scan(); scan.setStartRow(startKey); // only return tasks for this job scan.setFilter(new WhileMatchFilter(new PrefixFilter(startKey))); // expect a lot of tasks on average scan.setCaching(500); return scan; }
/** * Retrieves all the event rows matching a single * {@link com.twitter.hraven.Flow}. * @param flowKey * @return */ public List<FlowEvent> getFlowEvents(FlowKey flowKey) throws IOException { byte[] startKey = Bytes.add(flowKeyConverter.toBytes(flowKey), Constants.SEP_BYTES); Scan scan = new Scan(startKey); scan.setFilter(new WhileMatchFilter(new PrefixFilter(startKey))); List<FlowEvent> results = new ArrayList<FlowEvent>(); ResultScanner scanner = null; Table eventTable = null; try { eventTable = hbaseConnection .getTable(TableName.valueOf(Constants.FLOW_EVENT_TABLE)); scanner = eventTable.getScanner(scan); for (Result r : scanner) { FlowEvent event = createEventFromResult(r); if (event != null) { results.add(event); } } } finally { try { if (scanner != null) { scanner.close(); } } finally { if (eventTable != null) { eventTable.close(); } } } return results; }
/** * Retrieves all events added after the given event key (with sequence numbers * greater than the given key). If no new events are found returns an empty * list. * @param lastSeen * @return */ public List<FlowEvent> getFlowEventsSince(FlowEventKey lastSeen) throws IOException { // rows must match the FlowKey portion + SEP byte[] keyPrefix = Bytes.add(flowKeyConverter.toBytes(lastSeen), Constants.SEP_BYTES); // start at the next following sequence number FlowEventKey nextEvent = new FlowEventKey(lastSeen.getCluster(), lastSeen.getUserName(), lastSeen.getAppId(), lastSeen.getRunId(), lastSeen.getSequence() + 1); byte[] startKey = keyConverter.toBytes(nextEvent); Scan scan = new Scan(startKey); scan.setFilter(new WhileMatchFilter(new PrefixFilter(keyPrefix))); List<FlowEvent> results = new ArrayList<FlowEvent>(); ResultScanner scanner = null; Table eventTable = null; try { eventTable = hbaseConnection .getTable(TableName.valueOf(Constants.FLOW_EVENT_TABLE)); scanner = eventTable.getScanner(scan); for (Result r : scanner) { FlowEvent event = createEventFromResult(r); if (event != null) { results.add(event); } } } finally { try { if (scanner != null) { scanner.close(); } } finally { if (eventTable != null) { eventTable.close(); } } } return results; }
/** * Gets hdfs stats about all dirs on the given cluster * @param cluster * @param pathPrefix * @param limit * @param runId * @return list of hdfs stats * @throws IOException */ public List<HdfsStats> getAllDirs(String cluster, String pathPrefix, int limit, long runId) throws IOException { long encodedRunId = getEncodedRunId(runId); String rowPrefixStr = Long.toString(encodedRunId) + HdfsConstants.SEP + cluster; if (StringUtils.isNotEmpty(pathPrefix)) { // path expected to be cleansed at collection/storage time as well rowPrefixStr += HdfsConstants.SEP + StringUtil.cleanseToken(pathPrefix); } LOG.info(" Getting all dirs for cluster " + cluster + " with pathPrefix: " + pathPrefix + " for runId " + runId + " encodedRunId: " + encodedRunId + " limit: " + limit + " row prefix : " + rowPrefixStr); byte[] rowPrefix = Bytes.toBytes(rowPrefixStr); Scan scan = createScanWithAllColumns(); scan.setStartRow(rowPrefix); // require that all rows match the prefix we're looking for Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix)); scan.setFilter(prefixFilter); // using a large scanner caching value with a small limit can mean we scan a // lot more data than // necessary, so lower the caching for low limits scan.setCaching(Math.min(limit, defaultScannerCaching)); // we need only the latest cell version scan.setMaxVersions(1); return createFromScanResults(cluster, null, scan, limit, Boolean.FALSE, 0l, 0l); }
@Override public int scannerOpenWithPrefix(ByteBuffer tableName, ByteBuffer startAndPrefix, List<ByteBuffer> columns, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { try { HTable table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && columns.size() != 0) { for(ByteBuffer column : columns) { byte [][] famQf = KeyValue.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan)); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw new IOError(e.getMessage()); } }
