/** * Prints a TBase. * * @param t The object to print. * @param depth The print nesting level. * @return The pretty-printed version of the TBase. */ private static String printTbase(TBase t, int depth) { List<String> fields = Lists.newArrayList(); for (Map.Entry<? extends TFieldIdEnum, FieldMetaData> entry : FieldMetaData.getStructMetaDataMap(t.getClass()).entrySet()) { @SuppressWarnings("unchecked") boolean fieldSet = t.isSet(entry.getKey()); String strValue; if (fieldSet) { @SuppressWarnings("unchecked") Object value = t.getFieldValue(entry.getKey()); strValue = printValue(value, depth); } else { strValue = "not set"; } fields.add(tabs(depth) + entry.getValue().fieldName + ": " + strValue); } return Joiner.on("\n").join(fields); }
/** * Prints an object contained in a thrift message. * * @param o The object to print. * @param depth The print nesting level. * @return The pretty-printed version of the thrift field. */ private static String printValue(Object o, int depth) { if (o == null) { return "null"; } else if (TBase.class.isAssignableFrom(o.getClass())) { return "\n" + printTbase((TBase) o, depth + 1); } else if (Map.class.isAssignableFrom(o.getClass())) { return printMap((Map) o, depth + 1); } else if (List.class.isAssignableFrom(o.getClass())) { return printList((List) o, depth + 1); } else if (Set.class.isAssignableFrom(o.getClass())) { return printSet((Set) o, depth + 1); } else if (String.class == o.getClass()) { return '"' + o.toString() + '"'; } else { return o.toString(); } }
public static <P extends TBase<P, ?>, C extends TBase<C, ?>, G extends TBase<G, ?>> FieldGetter<P, G> compose(final FieldGetter<P, C> parent, final FieldGetter<C, G> child) { return new FieldGetter<P, G>() { @Override public Class<P> getStructClass() { return parent.getStructClass(); } @Override public Class<G> getValueClass() { return child.getValueClass(); } @Override public Optional<G> apply(P input) { Optional<C> parentValue = parent.apply(input); if (parentValue.isPresent()) { return child.apply(parentValue.get()); } else { return Optional.absent(); } } }; }
/** * Encodes a thrift object into a DEFLATE-compressed binary array. * * @param tBase Object to encode. * @return Deflated, encoded object. * @throws CodingException If the object could not be encoded. */ public static byte[] deflateNonNull(TBase<?, ?> tBase) throws CodingException { requireNonNull(tBase); // NOTE: Buffering is needed here for performance. // There are actually 2 buffers in play here - the BufferedOutputStream prevents thrift from // causing a call to deflate() on every encoded primitive. The DeflaterOutputStream buffer // allows the underlying Deflater to operate on a larger chunk at a time without stopping to // copy the intermediate compressed output to outBytes. // See http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4986239 ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); TTransport transport = new TIOStreamTransport( new BufferedOutputStream( new DeflaterOutputStream(outBytes, new Deflater(DEFLATE_LEVEL), DEFLATER_BUFFER_SIZE), DEFLATER_BUFFER_SIZE)); try { TProtocol protocol = PROTOCOL_FACTORY.getProtocol(transport); tBase.write(protocol); transport.close(); // calls finish() on the underlying stream, completing the compression return outBytes.toByteArray(); } catch (TException e) { throw new CodingException("Failed to serialize: " + tBase, e); } finally { transport.close(); } }
/** * Decodes a thrift object from a DEFLATE-compressed byte array into a target type. * * @param clazz Class to instantiate and deserialize to. * @param buffer Compressed buffer to decode. * @return A populated message. * @throws CodingException If the message could not be decoded. */ public static <T extends TBase<T, ?>> T inflateNonNull(Class<T> clazz, byte[] buffer) throws CodingException { requireNonNull(clazz); requireNonNull(buffer); T tBase = newInstance(clazz); TTransport transport = new TIOStreamTransport( new InflaterInputStream(new ByteArrayInputStream(buffer))); try { TProtocol protocol = PROTOCOL_FACTORY.getProtocol(transport); tBase.read(protocol); return tBase; } catch (TException e) { throw new CodingException("Failed to deserialize: " + e, e); } finally { transport.close(); } }
@SuppressWarnings({ "rawtypes" }) @Override public void beforeWrite(TMessage msg, TBase args, TBase result) { // reuse message's buffer when write? yes, we use the pool. ByteBuf readedBuf = message.getContent(); int refCount = readedBuf.refCnt(); if (refCount > 0) { readedBuf.release(refCount); } // voidMethod's return message is very short int initialCapacity = serverDef.trafficForecast.getInitBytesForWrite(msg.name); // logger.debug("initialCapacity = {} , msg = {}",initialCapacity, msg); ByteBuf buf = ctx.alloc().buffer(initialCapacity, serverDef.maxFrameSize); message.setContent(buf).beforeWrite(ctx); transport.setOutputBuffer(buf); }
@SuppressWarnings("rawtypes") private void writeResult(final TProtocol out, final TMessage msg, final WriterHandler onComplete, TBase args, final TBase result) { try { onComplete.beforeWrite(msg, args, result); // if (!isOneway()) { out.writeMessageBegin(new TMessage(msg.name, TMessageType.REPLY, msg.seqid)); if (result != null) { result.write(out); } else { out.writeStructBegin(null); out.writeFieldStop(); out.writeStructEnd(); } out.writeMessageEnd(); out.getTransport().flush(); // } onComplete.afterWrite(msg, null, TMessageType.REPLY, args, result); } catch (Throwable e) { onComplete.afterWrite(msg, e, TMessageType.EXCEPTION, args, result); } }
/** * This method will open the dictionary file stream for reading * * @throws IOException thrift reader open method throws IOException */ private void openThriftReader() throws IOException { // initialise dictionary file reader which will return dictionary thrift object // dictionary thrift object contains a list of byte buffer if (null == dictionaryMetadataFileReader) { dictionaryMetadataFileReader = new ThriftReader(this.columnDictionaryMetadataFilePath, new ThriftReader.TBaseCreator() { @Override public TBase create() { return new ColumnDictionaryChunkMeta(); } }); // Open it dictionaryMetadataFileReader.open(); } }
/** * This method will open the dictionary file stream for reading * * @throws IOException thrift reader open method throws IOException */ private void openThriftReader() throws IOException { if (null == dictionaryFileReader) { // initialise dictionary file reader which will return dictionary thrift object // dictionary thrift object contains a list of byte buffer dictionaryFileReader = new ThriftReader(this.columnDictionaryFilePath, new ThriftReader.TBaseCreator() { @Override public TBase create() { return new ColumnDictionaryChunk(); } }); // Open dictionary file reader dictionaryFileReader.open(); } }
public String cypherToken(TBase<?, ?> token, KeyStore keyStore) throws TException { byte[] tokenAesKey = keyStore.getKey(KeyStore.AES_TOKEN); byte[] tokenSipHashkey = keyStore.getKey(KeyStore.SIPHASH_TOKEN); // Serialize the thrift token into byte array byte[] serialized = serializer.serialize(token); // Calculate the SIP long sip = SipHashInline.hash24_palindromic(tokenSipHashkey, serialized); //Create the token byte buffer ByteBuffer buffer = ByteBuffer.allocate(8 + serialized.length); // adds the sip buffer.putLong(sip); // adds the thrift token buffer.put(serialized); // Wrap the TOKEN byte[] wrappedData = CryptoUtils.wrap(tokenAesKey, buffer.array()); String accessToken = new String(OrderPreservingBase64.encode(wrappedData)); return accessToken; }
private TransportRequest buildTransportRequest( DefaultCall<?> call, Context ctx, String procedure, TBase reqBody) throws ThriftEncodingException { TSerializer serializer = new TSerializer(protocolFactory); try { return DefaultTransportRequest.builder() .caller(clientConfig.getCaller()) .service(clientConfig.getService()) .procedure(procedure) .encoding(ThriftEncoding.ENCODING) .headers(call.getHeaders()) .deadline(ctx.getDeadline()) .span(ctx.getSpan()) .body(TransportBody.fromByteArray(serializer.serialize(reqBody))) .build(); } catch (TException e) { throw new ThriftEncodingException(e); } }
/** * Deserialize the Thrift object from a byte array. * * @param bytes The array to read from */ public TBase<?, ?> deserialize(byte[] bytes) throws TException { try { trans.reset(bytes); Header header = readHeader(); final int validate = validate(header); if (validate == HeaderUtils.OK) { TBase<?, ?> base = locator.tBaseLookup(header.getType()); base.read(protocol); return base; } if (validate == HeaderUtils.PASS_L4) { return new L4Packet(header); } throw new IllegalStateException("invalid validate " + validate); } finally { trans.clear(); protocol.reset(); } }
@Deprecated public static void serialize(TSerializer serializer, TBase struct, DataOutput out) throws IOException { assert serializer != null; assert struct != null; assert out != null; byte[] bytes; try { bytes = serializer.serialize(struct); } catch (TException e) { throw new RuntimeException(e); } out.writeInt(bytes.length); out.write(bytes); }
@Deprecated public static void deserialize(TDeserializer deserializer, TBase struct, DataInput in) throws IOException { assert deserializer != null; assert struct != null; assert in != null; byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); try { deserializer.deserialize(struct, bytes); } catch (TException ex) { throw new IOException(ex); } }
@Override public @Nullable <T> T decodeBody(@NotNull ByteBuf arg3, @NotNull Class<T> bodyType) { try { // Create a new instance of type 'T' T base = bodyType.newInstance(); // Get byte[] from ByteBuf byte[] payloadBytes = new byte[arg3.readableBytes()]; arg3.readBytes(payloadBytes); // Actually deserialize the payload TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); deserializer.deserialize((TBase<?, ?>) base, payloadBytes); return base; } catch (InstantiationException | IllegalAccessException | TException e) { logger.error("Failed to decode body to {}", bodyType.getName(), e); } return null; }
@Override public TBase<?, ?> tBaseLookup(short type) throws TException { switch (type) { case SPAN: return new TSpan(); case AGENT_INFO: return new TAgentInfo(); case AGENT_STAT: return new TAgentStat(); case AGENT_STAT_BATCH: return new TAgentStatBatch(); case SPANCHUNK: return new TSpanChunk(); case SQLMETADATA: return new TSqlMetaData(); case APIMETADATA: return new TApiMetaData(); case RESULT: return new TResult(); case STRINGMETADATA: return new TStringMetaData(); case NETWORK_CHECK: return new NetworkAvailabilityCheckPacket(); } throw new TException("Unsupported type:" + type); }
private static void invokeSynchronously( ServiceRequestContext ctx, Object impl, ThriftFunction func, TBase<?, ?> args, DefaultRpcResponse reply) { final ProcessFunction<Object, TBase<?, ?>> f = func.syncFunc(); ctx.blockingTaskExecutor().execute(() -> { if (reply.isDone()) { // Closed already most likely due to timeout. return; } try { final TBase<?, ?> result = f.getResult(impl, args); if (func.isOneWay()) { reply.complete(null); } else { reply.complete(func.getResult(result)); } } catch (Throwable t) { reply.completeExceptionally(t); } }); }
private static RpcRequest toRpcRequest(Class<?> serviceType, String method, TBase<?, ?> thriftArgs) { requireNonNull(thriftArgs, "thriftArgs"); // NB: The map returned by FieldMetaData.getStructMetaDataMap() is an EnumMap, // so the parameter ordering is preserved correctly during iteration. final Set<? extends TFieldIdEnum> fields = FieldMetaData.getStructMetaDataMap(thriftArgs.getClass()).keySet(); // Handle the case where the number of arguments is 0 or 1. final int numFields = fields.size(); switch (numFields) { case 0: return RpcRequest.of(serviceType, method); case 1: return RpcRequest.of(serviceType, method, ThriftFieldAccess.get(thriftArgs, fields.iterator().next())); } // Handle the case where the number of arguments is greater than 1. final List<Object> list = new ArrayList<>(numFields); for (TFieldIdEnum field : fields) { list.add(ThriftFieldAccess.get(thriftArgs, field)); } return RpcRequest.of(serviceType, method, list); }
private void addTSpanChunk(TBase<?, ?> base) throws TException { final TSpanChunk chunk = (TSpanChunk) base; if (chunk.getSpanEventList() == null) { write(base); return; } try { for (TSpanEvent e : chunk.getSpanEventList()) { eventStream.write(e); } write(chunk, FIELD_NAME_SPAN_EVENT_LIST, eventStream.split(chunkSize)); while (!eventStream.isEmpty()) { write(chunk, FIELD_NAME_SPAN_EVENT_LIST, eventStream.split(chunkSize)); } } finally { eventStream.clear(); } }
private void write(final TBase<?, ?> base, final String fieldName, final List<ByteArrayOutput> list) throws TException { final TReplaceListProtocol protocol = new TReplaceListProtocol(protocolFactory.getProtocol(transport)); // write chunk header writeChunkHeader(protocol); // write header writeHeader(protocol, locator.headerLookup(base)); if (list != null && list.size() > 0) { protocol.addReplaceField(fieldName, list); } base.write(protocol); if (isNeedFlush()) { flush(); } }
@Override public Pair<Class<? extends DataSerialization<?>>, TBase<?, ?>> getDeserializationDelegate(SColumnShard serialized) throws DeserializationException { Class<? extends DataSerialization<?>> resClass; switch (serialized.getType()) { case STRING: resClass = DefaultStringStandardColumnShard.class; break; case LONG: resClass = DefaultLongStandardColumnShard.class; break; case DOUBLE: resClass = DefaultDoubleStandardColumnShard.class; break; default: throw new DeserializationException("Cannot deserialize column shard: unknown type."); } return new Pair<>(resClass, serialized); }
public <Request extends Message, Response extends Message> TBase<?, ?> getResult( Controller<Request, Response> iface, ByteBuffer args, String methodName, String ip, TProtocol oprot, int seqid) throws TException { BaseService.execute_result result = new BaseService.execute_result(); Request request = this.parseRequest(iface, args); if (LOGGER.isDebugEnabled()) { LOGGER.debug(new StringBuilder() .append("Begin handling query interface=") .append(methodName).append(", request: ") .append(messageToString(request)).append(", from: ") .append(ip).toString()); } Response response = null; try { response = iface.process(request); } catch (TException e) { if (this.handleException(result, e)) {// 处理异常成功,直接返回 return result; } else { throw e; } } result.success = ByteBuffer.wrap(response.toByteArray()); return result; }
/** * convert thrift object to bytes * * @param obj thrift object * @param <T> Class type * @return bytes data */ public static <T extends TBase<T, ?>> byte[] toBytes(T obj) { Validate.notNull(obj); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); TTransport trans = new TIOStreamTransport(buffer); TCompactProtocol protocol = new TCompactProtocol(trans); try { obj.write(protocol); return buffer.toByteArray(); } catch (TException e) { throw new IllegalStateException("unexpected", e); } }
public byte[] getActorMessageByteArray(Operation operation, TBase<?, ?> base) { byte[] tbaseMessage = null; ActorMessage actorMessage = new ActorMessage(); actorMessage.op = operation; try { actorMessage.messageRaw = getTbaseByteBuffer(base); tbaseMessage = getOnlySerializerByteArray(actorMessage); } catch (TException e) { e.printStackTrace(); } return tbaseMessage; }
private ByteBuffer getTbaseByteBuffer(TBase<?, ?> message) throws TException { byte[] tbaseMessage = getOnlySerializerByteArray(message); ByteBuffer buffer = ByteBuffer.allocate(tbaseMessage.length); buffer.put(tbaseMessage); buffer.flip(); return buffer; }
private Response dumpEntity(final String id, final Quiet<Optional<? extends TBase<?, ?>>> work) { return fillTemplate(template -> { template.setAttribute("id", id); Optional<? extends TBase<?, ?>> struct = storage.read(work); if (struct.isPresent()) { template.setAttribute("structPretty", Util.prettyPrint(struct.get())); template.setAttribute("exception", null); } else { template.setAttribute("exception", "Entity not found"); } }); }
/** * Creates a function that performs a 'deep copy' on a thrift struct of a specific type. The * resulting copied objects will be exactly identical to the original. Mutations to the original * object will not be reflected in the copy, and vice versa. * * @return A copier for the provided type of thrift structs. */ static <T extends TBase<T, ?>> Function<T, T> deepCopier() { return input -> { if (input == null) { return null; } @SuppressWarnings("unchecked") T t = (T) input.deepCopy(); return t; }; }
/** * Identical to {@link #decodeNonNull(Class, byte[])}, but allows for a null buffer. * * @param clazz Class to instantiate and deserialize to. * @param buffer Buffer to decode. * @param <T> Target type. * @return A populated message, or {@code null} if the buffer was {@code null}. * @throws CodingException If the message could not be decoded. */ @Nullable public static <T extends TBase<T, ?>> T decode(Class<T> clazz, @Nullable byte[] buffer) throws CodingException { if (buffer == null) { return null; } return decodeNonNull(clazz, buffer); }
/** * Decodes a binary-encoded byte array into a target type. * * @param clazz Class to instantiate and deserialize to. * @param buffer Buffer to decode. * @param <T> Target type. * @return A populated message. * @throws CodingException If the message could not be decoded. */ public static <T extends TBase<T, ?>> T decodeNonNull(Class<T> clazz, byte[] buffer) throws CodingException { requireNonNull(clazz); requireNonNull(buffer); try { T t = newInstance(clazz); new TDeserializer(PROTOCOL_FACTORY).deserialize(t, buffer); return t; } catch (TException e) { throw new CodingException("Failed to deserialize thrift object.", e); } }
/** * Identical to {@link #encodeNonNull(TBase)}, but allows for a null input. * * @param tBase Object to encode. * @return Encoded object, or {@code null} if the argument was {@code null}. * @throws CodingException If the object could not be encoded. */ @Nullable public static byte[] encode(@Nullable TBase<?, ?> tBase) throws CodingException { if (tBase == null) { return null; } return encodeNonNull(tBase); }
/** * Encodes a thrift object into a binary array. * * @param tBase Object to encode. * @return Encoded object. * @throws CodingException If the object could not be encoded. */ public static byte[] encodeNonNull(TBase<?, ?> tBase) throws CodingException { requireNonNull(tBase); try { return new TSerializer(PROTOCOL_FACTORY).serialize(tBase); } catch (TException e) { throw new CodingException("Failed to serialize: " + tBase, e); } }
public static <T extends TBase> String ThriftToJSON(T thrift) { TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory()); try { return serializer.toString(thrift); } catch (TException e) { } throw new IllegalStateException("Convert to json failed : " + thrift); }
@SuppressWarnings("rawtypes") public void saveWritedBytes(String method, int writedBytes, TBase args, TBase result) { MethodTraffic mt = traffics.get(method.hashCode()); if (mt != null) { mt.save(writedBytes); } }
@SuppressWarnings({ "rawtypes" }) @Override public void afterWrite(TMessage msg, Throwable cause, int code, TBase args, TBase result) { if (transport.isHasFlush()) { message.write(ctx); serverDef.trafficForecast.saveWritedBytes(msg.name, transport.getWrittenByteCount(), args, result); } else { message.getContent().release(); logger.error("fail to process! code={}", code, cause); } }
@SuppressWarnings({ "rawtypes" }) private void writeException(final TProtocol out, final TMessage msg, final WriterHandler onComplete, final TApplicationException x, TBase args) { Throwable cause = null; try { onComplete.beforeWrite(msg, args, null); out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); x.write(out); out.writeMessageEnd(); out.getTransport().flush(); } catch (Throwable e) { cause = e; } onComplete.afterWrite(msg, cause, TMessageType.EXCEPTION, args, null); }
public static CarbonTable readCarbonTableFromStore(CarbonTablePath carbonTablePath, AbsoluteTableIdentifier identifier) throws IOException { String schemaFilePath = carbonTablePath.getSchemaFilePath(); if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) { String tableName = identifier.getCarbonTableIdentifier().getTableName(); ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() { public TBase create() { return new org.apache.carbondata.format.TableInfo(); } }; ThriftReader thriftReader = new ThriftReader(carbonTablePath.getSchemaFilePath(), createTBase); thriftReader.open(); org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read(); thriftReader.close(); SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl(); TableInfo wrapperTableInfo = schemaConverter .fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getStorePath()); wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath)); CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo); return CarbonMetadata.getInstance().getCarbonTable( identifier.getCarbonTableIdentifier().getTableUniqueName()); } else { return null; } }
/** * Write the object to disk. */ public void write(TBase t) throws IOException { try { t.write(binaryOut); dataOutputStream.flush(); } catch (TException e) { throw new IOException(e); } }
/** * Open the thrift reader * * @param filePath * @return * @throws IOException */ private ThriftReader openThriftReader(String filePath) throws IOException { ThriftReader thriftReader = new ThriftReader(filePath, new ThriftReader.TBaseCreator() { @Override public TBase create() { return new FileFooter(); } }); return thriftReader; }
/** * Below method will be used to read the index header * * @return index header * @throws IOException if any problem while reader the header */ public IndexHeader readIndexHeader() throws IOException { IndexHeader indexHeader = (IndexHeader) thriftReader.read(new ThriftReader.TBaseCreator() { @Override public TBase create() { return new IndexHeader(); } }); return indexHeader; }
/** * Below method will be used to read the block index from fie * * @return block index info * @throws IOException if problem while reading the block index */ public BlockIndex readBlockIndexInfo() throws IOException { BlockIndex blockInfo = (BlockIndex) thriftReader.read(new ThriftReader.TBaseCreator() { @Override public TBase create() { return new BlockIndex(); } }); return blockInfo; }