@Override public void map(AvroKey<DocumentContent> key, NullWritable ignore, Context context) throws IOException, InterruptedException { DocumentContent content = key.datum(); String documentId = content.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (content.getPdf()!=null) { ByteBuffer byteBuffer = content.getPdf(); if (byteBuffer.hasArray() && contentApprover.approve(byteBuffer.array())) { try (InputStream inputStream = new ByteBufferInputStream(byteBuffer)) { processStream(documentId, inputStream); } } else { log.info(invalidPdfHeaderMsg); handleException(new InvalidPdfException(invalidPdfHeaderMsg), content.getId().toString()); } } else { log.warn("no byte data found for id: " + content.getId()); } }
void readConnectResult() throws IOException { if (LOG.isTraceEnabled()) { StringBuilder buf = new StringBuilder("0x["); for (byte b : incomingBuffer.array()) { buf.append(Integer.toHexString(b) + ","); } buf.append("]"); LOG.trace("readConnectResult " + incomingBuffer.remaining() + " " + buf.toString()); } ByteBufferInputStream bbis = new ByteBufferInputStream(incomingBuffer); BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis); ConnectResponse conRsp = new ConnectResponse(); conRsp.deserialize(bbia, "connect"); // read "is read-only" flag boolean isRO = false; try { isRO = bbia.readBool("readOnly"); } catch (IOException e) { // this is ok -- just a packet from an old server which // doesn't contain readOnly field LOG.warn("Connected to an old server; r-o mode will be unavailable"); } this.sessionId = conRsp.getSessionId(); sendThread.onConnected(conRsp.getTimeOut(), this.sessionId, conRsp.getPasswd(), isRO); }
private MultiTransactionRecord codeDecode(MultiTransactionRecord request) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryOutputArchive boa = BinaryOutputArchive.getArchive(baos); request.serialize(boa, "request"); baos.close(); ByteBuffer bb = ByteBuffer.wrap(baos.toByteArray()); bb.rewind(); BinaryInputArchive bia = BinaryInputArchive.getArchive(new ByteBufferInputStream(bb)); MultiTransactionRecord decodedRequest = new MultiTransactionRecord(); decodedRequest.deserialize(bia, "request"); return decodedRequest; }
private MultiResponse codeDecode(MultiResponse request) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryOutputArchive boa = BinaryOutputArchive.getArchive(baos); request.serialize(boa, "result"); baos.close(); ByteBuffer bb = ByteBuffer.wrap(baos.toByteArray()); bb.rewind(); BinaryInputArchive bia = BinaryInputArchive.getArchive(new ByteBufferInputStream(bb)); MultiResponse decodedRequest = new MultiResponse(); decodedRequest.deserialize(bia, "result"); return decodedRequest; }
void readConnectResult() throws IOException { if (LOG.isTraceEnabled()) { StringBuffer buf = new StringBuffer("0x["); for (byte b : incomingBuffer.array()) { buf.append(Integer.toHexString(b) + ","); } buf.append("]"); LOG.trace("readConnectRestult " + incomingBuffer.remaining() + " " + buf.toString()); } ByteBufferInputStream bbis = new ByteBufferInputStream(incomingBuffer); BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis); ConnectResponse conRsp = new ConnectResponse(); conRsp.deserialize(bbia, "connect"); // read "is read-only" flag boolean isRO = false; try { isRO = bbia.readBool("readOnly"); } catch (IOException e) { // this is ok -- just a packet from an old server which // doesn't contain readOnly field LOG.warn("Connected to an old server; r-o mode will be unavailable"); } this.sessionId = conRsp.getSessionId(); sendThread.onConnected(conRsp.getTimeOut(), this.sessionId, conRsp.getPasswd(), isRO); }
/** * Gets the table that starts at the given offset. */ public Table getTable(int offset) throws UnknownTableException, DroppedTableException { preBufferAccess(); try { // Duplicate the buffer to allow for concurrent reads without locking ByteBuffer dup = _buffer.asReadOnlyBuffer(); // Move to the given offset dup.position(offset); // Read the length int length = dup.getInt(); if (length < 0) { // This was an exception throwUnknownOrDroppedTableException(dup, length); } // Restrict the buffer to only read the length of the table dup = (ByteBuffer) dup.slice().limit(length); try (InputStream in = new ByteBufferInputStream(dup)) { return getTableSerializer().deserialize(in); } catch (IOException e) { throw Throwables.propagate(e); } } finally { postBufferAccess(); } }
public Request checkUpgradeSession(Request request) throws IOException, KeeperException { // If this is a request for a local session and it is to // create an ephemeral node, then upgrade the session and return // a new session request for the leader. // This is called by the request processor thread (either follower // or observer request processor), which is unique to a learner. // So will not be called concurrently by two threads. if (request.type != OpCode.create || !upgradeableSessionTracker.isLocalSession(request.sessionId)) { return null; } CreateRequest createRequest = new CreateRequest(); request.request.rewind(); ByteBufferInputStream.byteBuffer2Record(request.request, createRequest); request.request.rewind(); CreateMode createMode = CreateMode.fromFlag(createRequest.getFlags()); if (!createMode.isEphemeral()) { return null; } // Uh oh. We need to upgrade before we can proceed. if (!self.isLocalSessionsUpgradingEnabled()) { throw new KeeperException.EphemeralOnLocalSessionException(); } return makeUpgradeRequest(request.sessionId); }
@Test public void testVarInt() throws Exception { //byte[] thingy = new byte[50]; long[] lengths = {1, 20, 200, 1024, 2048, 4000, 10000, 50000, 100000, 1024 * 1024, ((long) Integer.MAX_VALUE) * 100, -1, -200, -5000}; for (long value : lengths) { // do the test: ByteBufferOutputStream bbos = new ByteBufferOutputStream(12); CodedOutputStream cos = CodedOutputStream.newInstance(bbos); long newvalue = (value << 4) | 8; //cos.writeRawVarint64(newvalue); cos.writeSInt64NoTag(newvalue); cos.flush(); ByteBuffer bb = bbos.getByteBuffer(); System.out.println("value: " + value + ", length: " + bb.remaining()); ByteBufferInputStream bbis = new ByteBufferInputStream(bb); CodedInputStream cis = CodedInputStream.newInstance(bbis); long outval = cis.readSInt64(); long actual = outval >> 4; long tag = outval & 0x0F; System.out.println(" transformed we are: " + outval + " actual: " + actual + " tag: " + tag); } }
