/** * Custom serialization method. * * @param oos the object output stream * @throws IOException */ protected void writeObject(ObjectOutputStream oos) throws IOException { // figure out size of the written network CountingOutputStream cos = new CountingOutputStream(new NullOutputStream()); if (replaceMissingFilter != null) { ModelSerializer.writeModel(model, cos, false); } modelSize = cos.getByteCount(); // default serialization oos.defaultWriteObject(); // actually write the network if (replaceMissingFilter != null) { ModelSerializer.writeModel(model, oos, false); } }
public S3TransportBuffer(long maxBytes, boolean useCompression, S3TransportSerializer serializer) throws TransportException { this.maxBytes = maxBytes; this.serializer = serializer; baos = new ByteArrayOutputStream(); cos = new CountingOutputStream(baos); if (useCompression) { this.isCompressed = true; try { os = new BZip2CompressorOutputStream(cos); } catch (IOException e) { throw new TransportException("unable to create BZip2CompressorOutputStream", e); } } else { this.isCompressed = false; os = cos; } }
/** * Establish socket connection with client */ private void create_socket_and_listen() throws Exception { sock = new ServerSocket(EstimateNConfig.socketPort); // create socket and bind to port System.out.println("waiting for client to connect"); clientSocket = sock.accept(); // wait for client to connect System.out.println("client has connected"); CountingOutputStream cos = new CountingOutputStream(clientSocket.getOutputStream()); CountingInputStream cis = new CountingInputStream(clientSocket.getInputStream()); ProgCommon.oos = new ObjectOutputStream(cos); ProgCommon.ois = new ObjectInputStream(cis); StopWatch.cos = cos; StopWatch.cis = cis; }
/** * Encodes and writes a stream directly to an OutputStream. The length of * the stream, in this case, is set on a PDFNumber object that has to be * prepared beforehand. * @param out OutputStream to write to * @param refLength PDFNumber object to receive the stream length * @return number of bytes written (header and trailer included) * @throws IOException in case of an I/O problem */ protected int encodeAndWriteStream(OutputStream out, PDFNumber refLength) throws IOException { int bytesWritten = 0; //Stream header byte[] buf = encode("stream\n"); out.write(buf); bytesWritten += buf.length; //Stream contents CloseBlockerOutputStream cbout = new CloseBlockerOutputStream(out); CountingOutputStream cout = new CountingOutputStream(cbout); OutputStream filteredOutput = getFilterList().applyFilters(cout); outputRawStreamData(filteredOutput); filteredOutput.close(); refLength.setNumber(Integer.valueOf(cout.getCount())); bytesWritten += cout.getCount(); //Stream trailer buf = encode("\nendstream"); out.write(buf); bytesWritten += buf.length; return bytesWritten; }
/** {@inheritDoc} */ @Override public int output(OutputStream stream) throws IOException { CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); textBuffer.append('['); boolean first = true; for (Map.Entry<Integer, Object> entry : this.map.entrySet()) { if (!first) { textBuffer.append(" "); } first = false; formatObject(entry.getKey(), cout, textBuffer); textBuffer.append(" "); formatObject(entry.getValue(), cout, textBuffer); } textBuffer.append(']'); PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
/** {@inheritDoc} */ @Override public int output(OutputStream stream) throws IOException { CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); textBuffer.append('['); for (int i = 0; i < values.size(); i++) { if (i > 0) { textBuffer.append(' '); } Object obj = this.values.get(i); formatObject(obj, cout, textBuffer); } textBuffer.append(']'); PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
/** * Test outputInline() - this writes the object reference if it is a direct object (has an * object number), or writes the String representation if there is no object number. */ @Test public void testOutputInline() { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); CountingOutputStream cout = new CountingOutputStream(outStream); StringBuilder textBuffer = new StringBuilder(); try { // test with no object number set. pdfName.outputInline(outStream, textBuffer); PDFDocument.flushTextBuffer(textBuffer, cout); assertEquals("/TestName", outStream.toString()); outStream.reset(); // test with object number set pdfName.setObjectNumber(1); pdfName.outputInline(outStream, textBuffer); PDFDocument.flushTextBuffer(textBuffer, cout); assertEquals("1 0 R", outStream.toString()); } catch (IOException e) { fail("IOException: " + e.getMessage()); } }
@Override protected void copyData(InputStream inputStream, OutputStream outputStream) throws IOException { final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); Timer timer = new Timer(); try { timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { progressListener.updateProgress(countingOutputStream.getCount()); } }, 0, 50); super.copyData(inputStream, countingOutputStream); } finally { timer.cancel(); } }
/** * Open the file and write its header. */ private void init() throws IOException { FileSystem fs = this.path.getFileSystem(conf); FSDataOutputStream fsOut = fs.create(this.path); this.countingOut = new CountingOutputStream( new BufferedOutputStream(fsOut)); this.out = new DataOutputStream(this.countingOut); // put any necessary config strings into the header. MetaBlock m = this.header.getMetaBlock(); if (isCharData) { m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.CLOB_ENCODING); } else { m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.BLOB_ENCODING); } if (null != codec) { m.put(MetaBlock.COMPRESSION_CODEC_KEY, this.codecName); } // Serialize the value of maxEntriesPerSegment as a VInt in a byte array // and put that into the metablock as ENTRIES_PER_SEGMENT_KEY. int segmentBufLen = WritableUtils.getVIntSize(this.maxEntriesPerSegment); DataOutputBuffer entriesPerSegBuf = new DataOutputBuffer(segmentBufLen); WritableUtils.writeVInt(entriesPerSegBuf, this.maxEntriesPerSegment); byte [] entriesPerSegArray = Arrays.copyOf(entriesPerSegBuf.getData(), segmentBufLen); m.put(MetaBlock.ENTRIES_PER_SEGMENT_KEY, new BytesWritable(entriesPerSegArray)); // Write the file header to the file. this.header.write(out); // Now we're ready to accept record data from the user. }
@Override /** * {@inheritDoc} */ public OutputStream writeBlobRecord(long claimedLen) throws IOException { finishRecord(); // finish any previous record. checkForNull(this.out); startRecordIndex(); this.header.getStartMark().write(out); LOG.debug("Starting new record; id=" + curEntryId + "; claimedLen=" + claimedLen); WritableUtils.writeVLong(out, curEntryId); WritableUtils.writeVLong(out, claimedLen); this.curClaimedLen = claimedLen; this.userCountingOutputStream = new CountingOutputStream( new CloseShieldOutputStream(out)); if (null == this.codec) { // No codec; pass thru the same OutputStream to the user. this.userOutputStream = this.userCountingOutputStream; } else { // Wrap our CountingOutputStream in a compressing OutputStream to // give to the user. this.compressor.reset(); this.userOutputStream = new CompressorStream( this.userCountingOutputStream, compressor); } return this.userOutputStream; }
/** Initialize the OutputStream to the next file to write to. */ private void openNextFile() throws IOException { StringBuffer sb = new StringBuffer(); Formatter fmt = new Formatter(sb); fmt.format("%05d", this.fileNum++); String filename = filePrefix + fmt.toString(); if (codec != null) { filename = filename + codec.getDefaultExtension(); } Path destFile = new Path(destDir, filename); FileSystem fs = destFile.getFileSystem(conf); LOG.debug("Opening next output file: " + destFile); if (fs.exists(destFile)) { Path canonicalDest = destFile.makeQualified(fs); throw new IOException("Destination file " + canonicalDest + " already exists"); } OutputStream fsOut = fs.create(destFile); // Count how many actual bytes hit HDFS. this.countingFilterStream = new CountingOutputStream(fsOut); if (codec != null) { // Wrap that in a compressing stream. this.writeStream = codec.createOutputStream(this.countingFilterStream); } else { // Write to the counting stream directly. this.writeStream = this.countingFilterStream; } }
public FileHolder() throws IOException { super(); this.plainTempFile = Files.createTempFile("", ""); this.plainStream = new CountingOutputStream(Files.newOutputStream(plainTempFile)); this.gzTempFile = Files.createTempFile("", ""); this.gzStream = new CountingOutputStream(Files.newOutputStream(gzTempFile)); this.bzTempFile = Files.createTempFile("", ""); this.bzStream = new CountingOutputStream(Files.newOutputStream(bzTempFile)); }
@Test public void downloadMethodShouldCreateLocalFileStreamContainingProgressListener() throws IOException { FTPFile file = new FTPFile("remote.file", 0l, "path/to", 0, false); ftpConnection.setProgressListener(new ProgressListener()); ftpConnection.download(file, LOCAL_DIRECTORY); verify(mockFileStreamFactory).createOutputStream(LOCAL_DIRECTORY + "/remote.file"); verify(mockFtpClient).retrieveFile(eq("path/to/remote.file"), any(CountingOutputStream.class)); }
@Override public FileWriter getWriter(final long bucketKey, final String fileName) throws IOException { final DataOutputStream dos = getOutputStream(bucketKey, fileName); final CountingOutputStream cos = new CountingOutputStream(dos); final Output out = new Output(cos); return new FileWriter() { @Override public void close() throws IOException { out.close(); cos.close(); dos.close(); } @Override public void append(byte[] key, byte[] value) throws IOException { kryo.writeObject(out, key); kryo.writeObject(out, value); } @Override public long getBytesWritten() { return cos.getCount() + out.position(); } }; }
public RecordWriter(Path path, long timeToLiveMillis, OutputStream textOutputStream, DataGeneratorFactory generatorFactory, StreamCloseEventHandler streamCloseEventHandler) throws StageException, IOException { this(path, timeToLiveMillis, generatorFactory); this.textOutputStream = new CountingOutputStream(textOutputStream); generator = generatorFactory.getGenerator(this.textOutputStream, streamCloseEventHandler); textFile = true; this.idleTimeout = -1L; }
@Override public CountingOutputStream getOutputStream(String filePath) throws StageException, IOException { ADLFileOutputStream stream; if (!client.checkExists(filePath)) { stream = client.createFile(filePath, IfExists.FAIL); } else { stream = client.getAppendStream(filePath); } countingOutputStream = new CountingOutputStream(stream); return countingOutputStream; }
@Override public void writeTo(OutputStream out, String[] ignoreHeaders) throws MessagingException, IOException { if (bytesSent == -1) { CountingOutputStream countingOut = new CountingOutputStream(out); super.writeTo(countingOut, ignoreHeaders); bytesSent = countingOut.getByteCount(); } else { super.writeTo(out, ignoreHeaders); } }
/** * Overload the base object method so we don't have to copy * byte arrays around so much * {@inheritDoc} */ @Override public int output(OutputStream stream) throws IOException { setupFilterList(); CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); StreamCache encodedStream = null; PDFNumber refLength = null; final Object lengthEntry; if (encodeOnTheFly) { refLength = new PDFNumber(); getDocumentSafely().registerObject(refLength); lengthEntry = refLength; } else { encodedStream = encodeStream(); lengthEntry = Integer.valueOf(encodedStream.getSize() + 1); } populateStreamDict(lengthEntry); dictionary.writeDictionary(cout, textBuffer); //Send encoded stream to target OutputStream PDFDocument.flushTextBuffer(textBuffer, cout); if (encodedStream == null) { encodeAndWriteStream(cout, refLength); } else { outputStreamData(encodedStream, cout); encodedStream.clear(); //Encoded stream can now be discarded } PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
/** {@inheritDoc} */ @Override public int output(OutputStream stream) throws IOException { CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); writeDictionary(cout, textBuffer); PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
@Override public int output(OutputStream stream) throws IOException { CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); textBuffer.append(toString()); PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
@Override public int output(OutputStream stream) throws IOException { CountingOutputStream cout = new CountingOutputStream(stream); StringBuilder textBuffer = new StringBuilder(64); formatObject(getIDRef(), cout, textBuffer); textBuffer.append(' '); formatObject(goToReference, cout, textBuffer); PDFDocument.flushTextBuffer(textBuffer, cout); return cout.getCount(); }
/** * Tests writeDictionary() - tests that the dictionary is properly written to the output-stream. */ @Test public void testWriteDictionary() { // Ensure that the objects stored in the dictionary are streamed in the correct format. ByteArrayOutputStream outStream = new ByteArrayOutputStream(); CountingOutputStream cout = new CountingOutputStream(outStream); StringBuilder textBuffer = new StringBuilder(); try { pdfDictUnderTest.writeDictionary(cout, textBuffer); PDFDocument.flushTextBuffer(textBuffer, cout); assertEquals(expectedOutput, outStream.toString()); } catch (IOException e) { fail("IOException: " + e.getMessage()); } }
public Long copyMageTabZipToOutputStream(String api, CaArrayEntityReference experimentReference, boolean compressed) throws Exception { CountingOutputStream outStream = new CountingOutputStream(new NullOutputStream()); dataApiUtils.copyMageTabZipToOutputStream(experimentReference, outStream); return outStream.getByteCount(); }
public Long copyMageTabZipApiUtils(String api, CaArrayEntityReference experimentReference, boolean compressed) throws Exception { CountingOutputStream stream = new CountingOutputStream(new NullOutputStream()); dataApiUtils.copyMageTabZipToOutputStream(experimentReference, stream); return stream.getByteCount(); }
public IndexingProxyStreamOutput(final OutputStream out) { this.out = new CountingOutputStream(new BufferedOutputStream(out)); }
private CountingOutputStream listenOn(OutputStream outputStream) { LOGGER.debug("Creating wrapping output stream for progress listener"); CountingOutputStream countingStream = new CountingOutputStream(outputStream) { @Override protected void beforeWrite(int n) { super.beforeWrite(n); progressListener.setBytesWritten(getByteCount()); } }; return countingStream; }
/** * Writes the section of the file {@link OutputStream}. * * @param start * The byte offset in the input file where the write operation starts. * * @return * if the file is still being written, this method writes the file * until the last newline character and returns the offset to start * the next write operation. */ public long writeLogTo(long start, int size, OutputStream out) throws IOException { if (size <= 0) { return 0; } CountingOutputStream os = new CountingOutputStream(out); Session f = source.open(); f.skip(start); //long end = start + size; byte[] buf = new byte[size]; int sz; if ((sz=f.read(buf))>=0) { os.write(buf,0,sz); } /* if(completed) { } else { ByteBuf buf = new ByteBuf(null,f, size); HeadMark head = new HeadMark(buf); TailMark tail = new TailMark(buf); int readLines = 0; while(tail.moveToNextLine(f) && readLines++ < MAX_LINES_READ) { head.moveTo(tail, os); if (buf.isFull() || os.getCount() >= end) { break; } } head.finish(os); } */ f.close(); os.flush(); return os.getCount()+start; }
public CountingOutputStreamTransportMetrics(OutputStream out) { this.out = new CountingOutputStream(out); }
/** * Creates a copier that writes to the given output file counting the written bytes. * * @param reader * @param outputFile * @param version * version for the created pdf copy, if null the version number is taken from the input PdfReader. * @throws TaskException * if the file is not found or an error occur opening the underlying copier. */ public CountingPdfCopier(PdfReader reader, File outputFile, PdfVersion version) throws TaskException { try { outputStream = new CountingOutputStream(new FileOutputStream(outputFile)); init(reader, outputStream, version); } catch (FileNotFoundException e) { throw new TaskException(String.format("Unable to find the output file %s", outputFile.getPath()), e); } }
