protected String resultToString(Result result) { StringBuilder sb = new StringBuilder(); sb.append("{").append(keyToString(result.getRow())).append(":"); for (Cell cell : result.listCells()) { byte[] f = CellUtil.cloneFamily(cell); byte[] q = CellUtil.cloneQualifier(cell); RangeDescription range = rangeMap.get(Bytes.add(f, q)); sb.append("[").append(Bytes.toString(f)).append(":").append(Bytes.toString(q)).append("->"); if (notPrintingSet.contains(q)) sb.append("skipped random value"); else sb.append(DataType.byteToString(range.dataType, CellUtil.cloneValue(cell))); sb.append("]"); } sb.append("}"); return sb.toString(); }
public LCStatInfo2(byte[] family, byte[] qualifier, DataType type, int parts, String min, String max) throws IOException { this.family = family; this.qualifier = qualifier; this.type = type; this.isSet = false; switch (type) { case INT: parseInt(parts, min, max); break; case LONG: parseLong(parts, min, max); break; case DOUBLE: parseDouble(parts, min, max); break; default: throw new IOException("LCDBG, StatInfo ranges not support type: " + type); } }
/** * create map based on statDesc * 1. for set, "family qualifier DataType set [v1] [v2] [...]" * 2. for array, "family qualifier DataType min max parts" */ public static Map<TableName, LCStatInfo2> parseStatString(IndexTableRelation indexTableRelation, String statDesc) throws IOException { Map<TableName, LCStatInfo2> map = new HashMap<>(); String[] lines = statDesc.split(LC_TABLE_DESC_RANGE_DELIMITER); for (String line : lines) { String[] parts = line.split("\t"); byte[] family = Bytes.toBytes(parts[0]); byte[] qualifier = Bytes.toBytes(parts[1]); TableName tableName = indexTableRelation.getIndexTableName(family, qualifier); LCStatInfo2 statInfo; try { if ("set".equalsIgnoreCase(parts[3])) { statInfo = new LCStatInfo2(family, qualifier, DataType.valueOf(parts[2]), parts, 4); } else { statInfo = new LCStatInfo2(family, qualifier, DataType.valueOf(parts[2]), Integer.valueOf(parts[5]), parts[3], parts[4]); } } catch (IOException e) { throw new IOException("exception for parsing line: " + line, e); } map.put(tableName, statInfo); } return map; }
public static String getStringOfValueAndType(final DataType type, final byte[] data) { if (data == null) return "null"; if (type == DataType.SHORT || type == DataType.INT) { return String.valueOf(Bytes.toInt(data)); } if (type == DataType.DOUBLE) { return String.valueOf(Bytes.toDouble(data)); } if (type == DataType.LONG) { return String.valueOf(Bytes.toLong(data)); } if (type == DataType.STRING) { return Bytes.toString(data); } return "mWinterGetStringOfValueAndType type not supported!"; }
public static int compareValues(byte[] b1, byte[] b2, DataType type) { switch (type) { case INT: // jdk 1.7 // return Integer.compare(Bytes.toInt(b1), Bytes.toInt(b2)); // jdk 1.6 return compareInt(Bytes.toInt(b1), Bytes.toInt(b2)); case LONG: return compareLong(Bytes.toLong(b1), Bytes.toLong(b2)); case DOUBLE: return Double.compare(Bytes.toDouble(b1), Bytes.toDouble(b2)); case STRING: return Bytes.toString(b1).compareTo(Bytes.toString(b2)); default: break; } new Exception("winter compareWithQualifier not supportted type: " + type).printStackTrace(); return 0; }
public static byte[] parsingStringToBytesWithType(DataType type, String s) { switch (type) { case INT: return Bytes.toBytes(Integer.valueOf(s)); case DOUBLE: return Bytes.toBytes(Double.valueOf(s)); case LONG: return Bytes.toBytes(Long.valueOf(s)); case SHORT: return Bytes.toBytes(Short.valueOf(s)); case BOOLEAN: return Bytes.toBytes(Boolean.valueOf(s)); case STRING: return Bytes.toBytes(s); } return null; }
@Override public void readFields(DataInput in) throws IOException { family = WritableUtils.readCompressedByteArray(in); qualifier = WritableUtils.readCompressedByteArray(in); dataType = DataType.valueOf(WritableUtils.readString(in)); isIndex = WritableUtils.readVInt(in) == 1; hashCode = calHashCode(); }
/** * Get each column's data type of this table. * * @return * @throws IOException */ public Map<byte[], DataType> getColumnInfoMap(TableName tableName) throws IOException { String tempInfo = admin.getTableDescriptor(tableName).getValue("DATA_FORMAT"); Map<byte[], DataType> columnTypeMap = null; if (tempInfo != null) { columnTypeMap = new TreeMap<byte[], DataType>(Bytes.BYTES_COMPARATOR); String[] temp = tempInfo.split(","); for (int i = 0; i < temp.length; i++) { int loc = temp[i].lastIndexOf(':'); if (loc != -1) { columnTypeMap.put(Bytes.toBytes(temp[i].substring(0, loc)), DataType.valueOf(temp[i].substring(loc + 1))); } else { LOG.warn("Failed to read column type!" + temp[i]); } } } return columnTypeMap; }
@Override public void readFields(DataInput in) throws IOException { family = Bytes.readByteArray(in); qualifier = Bytes.readByteArray(in); boolean startNull = Bytes.toBoolean(Bytes.readByteArray(in)); if (startNull) { start = null; startOp = CompareFilter.CompareOp.NO_OP; } else { start = Bytes.readByteArray(in); startOp = CompareFilter.CompareOp.valueOf(Bytes.toString(Bytes.readByteArray(in))); } boolean stopNull = Bytes.toBoolean(Bytes.readByteArray(in)); if (stopNull) { stop = null; stopOp = CompareFilter.CompareOp.NO_OP; } else { stop = Bytes.readByteArray(in); stopOp = CompareFilter.CompareOp.valueOf(Bytes.toString(Bytes.readByteArray(in))); } startTs = in.readLong(); stopTs = in.readLong(); dataType = DataType.valueOf(Bytes.toString(Bytes.readByteArray(in))); }
/** * Get each column's data type of this table. * @return * @throws IOException */ public Map<byte[], DataType> getColumnInfoMap(byte[] tableName) throws IOException { String tempInfo = admin.getTableDescriptor(tableName).getValue("DATA_FORMAT"); Map<byte[], DataType> columnTypeMap = null; if (tempInfo != null) { columnTypeMap = new TreeMap<byte[], DataType>(Bytes.BYTES_COMPARATOR); String[] temp = tempInfo.split(","); for (int i = 0; i < temp.length; i++) { int loc = temp[i].lastIndexOf(':'); if (loc != -1) { columnTypeMap.put(Bytes.toBytes(temp[i].substring(0, loc)), DataType.valueOf(temp[i].substring(loc + 1))); } else { LOG.warn("Failed to read column type!" + temp[i]); } } } return columnTypeMap; }
private byte[] mWinterCalRangeKey(byte[] qualifier, byte[] key) throws IOException { if (key == null) { return null; } // winter the column indicated the target column-family to scan! DataType type = LCCIndexConstant.getType(lccIndexQualifierType, qualifier); byte[] lccValue = null; if (type == DataType.DOUBLE) { lccValue = Bytes.toBytes(LCCIndexConstant.paddedStringDouble(Bytes.toDouble(key))); } else if (type == DataType.INT) { lccValue = Bytes.toBytes(LCCIndexConstant.paddedStringInt(Bytes.toInt(key))); } else if (type == DataType.LONG) { lccValue = Bytes.toBytes(LCCIndexConstant.paddedStringLong(Bytes.toLong(key))); } else if (type == DataType.STRING) { // lccValue = Bytes.toBytes(LCCIndexConstant.paddedStringString(Bytes.toString(key))); lccValue = key; } else { throw new IOException("winter range generating new endkey not implemented yet: " + Bytes.toString(qualifier)); } return lccValue; }
private boolean mWinterValueCoincident(CompareOp op, byte[] v1, byte[] v2, DataType type) { int ret = LCCIndexConstant.compareValues(v1, v2, type); switch (op) { case GREATER: return ret > 0; case GREATER_OR_EQUAL: return ret >= 0; case LESS: return ret < 0; case LESS_OR_EQUAL: return ret <= 0; case NOT_EQUAL: return ret != 0; case EQUAL: return ret == 0; case NO_OP: default: System.out.println("winter what does this op mean?"); break; } return true; }
public LCStatInfo(String name, DataType type, int parts, String min, String max) throws IOException { this.name = name; this.type = type; this.isSet = false; switch (type) { case INT: parseInt(parts, min, max); break; case LONG: parseLong(parts, min, max); break; case DOUBLE: parseDouble(parts, min, max); break; default: new Exception("winter StatInfo ranges not supportted type: " + type).printStackTrace(); throw new IOException("winter StatInfo ranges not supportted type: " + type); } }
public static List<LCStatInfo> parseStatString(String str, List<String> includedQualifiers) throws IOException { List<LCStatInfo> list = new ArrayList<LCStatInfo>(); String[] lines = str.split(LCCIndexConstant.LCC_TABLE_DESC_RANGE_DELIMITER); for (String line : lines) { String[] parts = line.split("\t"); if (includedQualifiers != null && !includedQualifiers.contains(parts[0])) { continue; } if ("set".equalsIgnoreCase(parts[2])) { list.add(new LCStatInfo(parts[0], DataType.valueOf(parts[1]), parts, 3)); } else { list.add(new LCStatInfo(parts[0], DataType.valueOf(parts[1]), Integer.valueOf(parts[2]), parts[3], parts[4])); } } return list; }
public LCCIndexMemStoreScanner(KeyValueScanner scanner, TreeMap<byte[], DataType> lccIndexQualifier, byte[] target) throws IOException { super(); dataList = new LinkedList<KeyValue>(); this.lccIndexQualifier = new TreeMap<byte[], DataType>(Bytes.BYTES_COMPARATOR); if (target == null || !lccIndexQualifier.containsKey(target)) { throw new IOException("winter index column " + Bytes.toString(target) + " is uknown type"); } this.lccIndexQualifier.put(target, lccIndexQualifier.get(target)); // this.lccIndexQualifier = lccIndexQualifier; long start = System.currentTimeMillis(); init(scanner); System.out.println("winter LCCIndexMemStoreScanner cost " + (System.currentTimeMillis() - start) / 1000.0 + " seconds to build lcc memstore scanner from memstore, the size of this scanner is: " + dataList.size()); }
public LCCIndexGenerator(TreeMap<byte[], DataType> lccIndexQualifier, String descStr) { lccRowkeyMap = new TreeMap<byte[], ArrayList<byte[]>>(Bytes.BYTES_COMPARATOR); lccIndexStatistic = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); lccResults = new ArrayList<KeyValue>(); this.lccIndexQualifierType = lccIndexQualifier; lccQueue = new LinkedList<KeyValue>(); if (descStr != null) { try { statList = LCStatInfo.parseStatString(descStr); } catch (IOException e) { System.out.println("winter can not parse string to LCStatInfo, str: " + descStr); System.out.println("winter can not parse string to LCStatInfo, exception msg: " + e); } } else { System.out.println("winter will not create statList because desc is null"); } }
private void writeFile() throws IOException { Random random = new Random(); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName))); StringBuilder sb = null; // rowkey \t qualifier:value1 \t qualifier: value2 ... for (int i = start; i < stop; ++i) { sb = new StringBuilder(); sb.append(String.format(GENERATED_FORMAT, i)); for (CF_INFO ci : PutTestConstants.getCFInfo()) { if (ci.type == DataType.STRING || ci.isIndex == false) { continue; } sb.append("\t"); sb.append(ci.qualifier); sb.append(":"); sb.append(String.valueOf(random.nextInt(recordNumber * ci.scale))); } sb.append("\n"); writer.write(sb.toString()); } writer.close(); }
@Override public void printString(Result result) { StringBuilder sb = new StringBuilder(); List<KeyValue> kv = null; SimpleIndexKeyGenerator kg = new SimpleIndexKeyGenerator(); sb.append("row=" + LCCIndexConstant.getStringOfValueAndType(mainIndexType, result.getRow())); byte[][] bytes = kg.parseIndexRowKey(result.getRow()); sb.append(", key=" + LCCIndexConstant.getStringOfValueAndType(DataType.STRING, bytes[0])); sb.append(", value=" + LCCIndexConstant.getStringOfValueAndType(mainIndexType, bytes[1])); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { if (ci.qualifier.equals(mainIndexColumn)) { continue; } kv = result.getColumn(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes(ci.qualifier)); if (kv.size() != 0 && ci.type != DataType.STRING) { sb.append(", [" + FAMILY_NAME + ":" + ci.qualifier + "]=" + LCCIndexConstant.getStringOfValueAndType(ci.type, (kv.get(0).getValue()))); } } System.out.println(sb.toString()); }
@Override public void printString(Result result) { StringBuilder sb = new StringBuilder(); List<KeyValue> kv = null; sb.append("row=" + Bytes.toString(result.getRow())); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { kv = result.getColumn(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes(ci.qualifier)); if (kv.size() != 0 && ci.type != DataType.STRING) { sb.append(", [" + FAMILY_NAME + ":" + ci.qualifier + "]=" + LCCIndexConstant.getStringOfValueAndType(ci.type, (kv.get(0).getValue()))); } } System.out.println(sb.toString()); }
private void innerGenerateRandomData() { int counter = 0; for (int i = 0; i < RECORD_NUMBER; ++i, ++counter) { Put put = new Put(Bytes.toBytes(String.format(ROWKEY_FORMAT, i))); for (CF_INFO ci : PutTestConstants.getCFInfo()) { if (ci.type == DataType.STRING && ci.isIndex == false) { put.add(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes(ci.qualifier), Bytes.toBytes(RandomStringUtils.random(PutTestConstants.GENERATED_STRING_LENGTH))); // int } else { put.add( Bytes.toBytes(FAMILY_NAME), Bytes.toBytes(ci.qualifier), LCCIndexConstant.parsingStringToBytesWithType(ci.type, String.valueOf(random.nextInt(RECORD_NUMBER)))); // int } } if (counter == PRINT_INTERVAL) { counter = 0; System.out.println("coffey generate data " + i + " class: " + this.getClass().getName()); } queue.add(put); } }
private void work(int recordNumber, String dataFileName) throws IOException { Random random = new Random(); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFileName))); StringBuilder sb = null; // rowkey \t qualifier:value1 \t qualifier: value2 ... for (int i = 0; i < recordNumber; ++i) { sb = new StringBuilder(); sb.append(String.format("1%05d", i)); for (CF_INFO ci : PutTestConstants.getCFInfo()) { if (ci.type == DataType.STRING && ci.isIndex == false) { continue; } sb.append("\t"); sb.append(ci.qualifier); sb.append(":"); sb.append(String.valueOf(random.nextInt(recordNumber))); } sb.append("\n"); writer.write(sb.toString()); } writer.close(); }
private void initIRIndex(HBaseAdmin admin) throws IOException { System.out.println("start init IRIndex"); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("c3"), DataType.DOUBLE); IndexDescriptor index2 = new IndexDescriptor(Bytes.toBytes("c4"), DataType.STRING); IndexDescriptor index3 = new IndexDescriptor(Bytes.toBytes("c5"), DataType.STRING); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); family.addIndex(index2); family.addIndex(index3); tableDesc.addFamily(family); admin.createTable(tableDesc); }
private void doIndexScan() throws IOException { ScanRange.ScanRangeList rangeList = new ScanRange.ScanRangeList(); FilterList filterList = new FilterList(); CompareFilter.CompareOp startOp = CompareFilter.CompareOp.GREATER_OR_EQUAL; CompareFilter.CompareOp stopOp = CompareFilter.CompareOp.LESS_OR_EQUAL; for (int i = 0; i < indexColumnNames.length && i < scanValues.length; i++) { rangeList.addScanRange(new ScanRange(familyName, Bytes.toBytes(indexColumnNames[i]), Bytes.toBytes(scanValues[i][0]), Bytes.toBytes(scanValues[i][1]), startOp, stopOp, DataType.INT)); filterList.addFilter( new SingleColumnValueFilter(familyName, Bytes.toBytes(indexColumnNames[i]), startOp, Bytes.toBytes(scanValues[i][0]))); filterList.addFilter( new SingleColumnValueFilter(familyName, Bytes.toBytes(indexColumnNames[i]), stopOp, Bytes.toBytes(scanValues[i][1]))); } Scan scan = new Scan(); scan.setFilter(filterList); if (rangeList.getRanges().size() > 0) { scan.setAttribute(ScanRange.SCAN_RANGE_ATTRIBUTE_STR, rangeList.toBytesAttribute()); } scan.setId("LMD-scan"); scan.setCaching(1); ResultScanner scanner = BaseIndexScanner.getIndexScanner(conn, relation, scan); Result result; int count = 0; while ((result = scanner.next()) != null) { count++; if (PRINT_RESULT) printResult(result); } scanner.close(); System.out.println("LMDIndex scan has " + count + " records"); }
public RangeDescription(byte[] family, byte[] qualifier, DataType dataType, String[] parts, int startIndex) { this(family, qualifier, dataType); isSet = true; list = new ArrayList<>(); for (int i = startIndex; i < parts.length; i++) { list.add(parts[i]); } }
@Override public List<String> statDescriptions() { List<String> list = new ArrayList(); list.add(String .format("a\t%s\t%d\t%d\t%d", DataType.INT.toString(), minAValue, maxAValue, nbLCStatRange)); list.add(String .format("b\t%s\t%d\t%d\t%d", DataType.INT.toString(), minBValue, minBValue, nbLCStatRange)); return list; }
@Override public List<String> statDescriptions() { List<String> list = new ArrayList<>(); list.add(String .format("t\t%s\t%d\t%d\t%d", DataType.INT.toString(), minTotalPrice, maxTotalPrice, nbLCStatRange)); list.add(String .format("d\t%s\t%d\t%d\t%d", DataType.INT.toString(), minDate, maxDate, nbLCStatRange)); list.add(String.format("p\t%s\t%d\t%d\t%d", DataType.INT.toString(), minPriority, maxPriority, nbLCStatRange)); return list; }
@Override public List<String> statDescriptions() { List<String> list = new ArrayList(); list.add( String.format("a\t%s\t%d\t%d\t%d", DataType.INT.toString(), minA, maxA, nbLCStatRange)); list.add( String.format("b\t%s\t%d\t%d\t%d", DataType.INT.toString(), minB, maxB, nbLCStatRange)); list.add( String.format("c\t%s\t%d\t%d\t%d", DataType.INT.toString(), minC, maxC, nbLCStatRange)); return list; }
public LCStatInfo2(byte[] family, byte[] qualifier, DataType type, String[] allParts, int startIndex) throws IOException { this.family = family; this.qualifier = qualifier; this.type = type; this.isSet = true; this.setMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (int i = startIndex; i < allParts.length; ++i) { setMap.put(DataType.stringToBytes(type, allParts[i]), (long) 0); } }
public ColumnInfo(byte[] f, byte[] q, DataType type, boolean isIndex) { family = f; qualifier = q; dataType = type; this.isIndex = isIndex; hashCode = calHashCode(); }
/** * Get each column's data type of this table. * * @return */ public Map<byte[], DataType> getColumnInfoMap() { if (this.columnTypeMap == null || this.columnTypeMap.isEmpty()) { return null; } else { return Collections.unmodifiableMap(this.columnTypeMap); } }
/** * Set each column's data type of this table. * * @param columnTypes * @throws IOException */ public void setColumnInfoMap(TableName tableName, Map<byte[], DataType> columnTypes) throws IOException { HTableDescriptor desc = admin.getTableDescriptor(tableName); if (isTableEnabled(tableName)) { throw new IOException("Table " + tableName + " is enabled! Disable it first!"); } StringBuilder sb = new StringBuilder(); if (columnTypes != null && !columnTypes.isEmpty()) { int i = 0; for (Map.Entry<byte[], DataType> entry : columnTypes.entrySet()) { sb.append(Bytes.toString(entry.getKey())); sb.append(":"); sb.append(entry.getValue().toString()); if (i != columnTypes.size() - 1) { sb.append(","); } i++; } } desc.setValue("DATA_FORMAT", sb.toString()); admin.modifyTable(tableName, desc); // TODO maybe need to enable and disable, check add indexes }
private static void printLine(Result r, Map<byte[], DataType> columnInfoMap) { StringBuilder sb = new StringBuilder(); sb.append("row=" + Bytes.toString(r.getRow())); byte[] tmp = null; for (KeyValue kv : r.list()) { tmp = KeyValue.makeColumn(kv.getFamily(), kv.getQualifier()); sb.append("," + Bytes.toString(tmp) + "="); if (columnInfoMap == null || columnInfoMap.isEmpty()) { sb.append(Bytes.toString(kv.getValue())); } else { switch (columnInfoMap.get(tmp)) { case DOUBLE: sb.append(Bytes.toDouble(kv.getValue())); break; case LONG: sb.append(Bytes.toLong(kv.getValue())); break; case INT: sb.append(Bytes.toInt(kv.getValue())); break; case STRING: sb.append(Bytes.toString(kv.getValue())); break; default: sb.append(Bytes.toString(kv.getValue())); break; } } } System.out.println(sb.toString()); }
public ScanRange(byte[] family, byte[] qualifier, byte[] start, byte[] stop, CompareFilter.CompareOp startOp, CompareFilter.CompareOp stopOp, long startTs, long stopTs, DataType dataType) { this.family = family; this.qualifier = qualifier; this.start = start; this.stop = stop; this.startOp = startOp; this.stopOp = stopOp; this.startTs = startTs; this.stopTs = stopTs; this.dataType = dataType; }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("[").append(Bytes.toString(family)).append(":").append(Bytes.toString(qualifier)) .append("]"); sb.append(DataType.byteToString(dataType, start)).append("-").append(startOp).append(", "); sb.append(DataType.byteToString(dataType, stop)).append("-").append(stopOp); return sb.toString(); }
public static Scan getScan(String fileName) throws IOException { BufferedReader br = new BufferedReader(new FileReader(fileName)); String line; Scan scan = new Scan(); System.out.println("winter for scan ******"); FilterList filterList = new FilterList(); while ((line = br.readLine()) != null) { System.out.println("winter for scan : " + line); if (line.startsWith("#")) continue; // family, qualifier, type, >=, 10, <=, 1000 // family, qualifier, type, >=, 10 String[] splits = line.split("\t"); byte[] family = Bytes.toBytes(splits[0]); byte[] qualifier = Bytes.toBytes(splits[1]); DataType type = DataType.valueOf(splits[2].toUpperCase()); CompareFilter.CompareOp firstOp = parseOp(splits[3]); byte[] firstValue = DataType.stringToBytes(type, splits[4]); filterList.addFilter(new SingleColumnValueFilter(family, qualifier, firstOp, firstValue)); if (splits.length >= 6) { CompareFilter.CompareOp secondOp = parseOp(splits[5].toUpperCase()); byte[] secondValue = DataType.stringToBytes(type, splits[6]); filterList .addFilter(new SingleColumnValueFilter(family, qualifier, secondOp, secondValue)); } } scan.setFilter(filterList); ScanRangeList scanRangeList = ScanRangeList.getScanRangeList(fileName); if (scanRangeList.getRanges().size() > 0) { scan.setAttribute(ScanRange.SCAN_RANGE_ATTRIBUTE_STR, scanRangeList.toBytesAttribute()); } return scan; }
/** * Get each column's data type of this table. * @return */ public Map<byte[], DataType> getColumnInfoMap() { if (this.columnTypeMap == null || this.columnTypeMap.isEmpty()) { return null; } else { return Collections.unmodifiableMap(this.columnTypeMap); } }