public void encode(OutputStream out) throws IOException { java.util.zip.CRC32 crc32 = new java.util.zip.CRC32(); CheckedOutputStream outChecked = new CheckedOutputStream(out, crc32); // Index Indicator outChecked.write(0x00); // Number of Records EncoderUtil.encodeVLI(outChecked, recordCount); // List of Records for (IndexRecord record : records) { EncoderUtil.encodeVLI(outChecked, record.unpadded); EncoderUtil.encodeVLI(outChecked, record.uncompressed); } // Index Padding for (int i = getIndexPaddingSize(); i > 0; --i) outChecked.write(0x00); // CRC32 long value = crc32.getValue(); for (int i = 0; i < 4; ++i) out.write((byte)(value >>> (i * 8))); }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot) throws IOException { if (!close) { OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(snapShot)); CheckedOutputStream crcOut = new CheckedOutputStream(sessOS, new Adler32()); //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt,sessions,oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); sessOS.flush(); crcOut.close(); sessOS.close(); } }
public static void toZip(File file) { try { File zipFile = new File(file.getParentFile(), stripExtension(file.getName()) + ".zip"); FileOutputStream dest = new FileOutputStream(zipFile); CheckedOutputStream checksum = new CheckedOutputStream(dest, new Adler32()); ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(checksum)); byte data[] = new byte[BUFFER]; FileInputStream fi = new FileInputStream(file); BufferedInputStream origin = new BufferedInputStream(fi, BUFFER); ZipEntry entry = new ZipEntry(file.getName()); out.putNextEntry(entry); int count; while ((count = origin.read(data, 0, BUFFER)) != -1) { out.write(data, 0, count); } out.closeEntry(); origin.close(); out.close(); } catch (IOException e) { e.printStackTrace(); } }
/** * Compressed file or directory * * @param srcPath The address of the file or folder * @param zipFilePath The address of the compressed package */ public static void zipFiles(String srcPath,String zipFilePath) { File file = new File(srcPath); if (!file.exists()) throw new RuntimeException(srcPath + "not exist！"); try { FileOutputStream fileOutputStream = new FileOutputStream(zipFilePath); CheckedOutputStream cos = new CheckedOutputStream(fileOutputStream, new CRC32()); ZipOutputStream out = new ZipOutputStream(cos); String baseDir=""; zip(file,out,baseDir); out.close(); } catch (Exception e) { throw new RuntimeException(e); } }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into * @param fsync sync the file immediately after write */ public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot, boolean fsync) throws IOException { if (!close) { try (CheckedOutputStream crcOut = new CheckedOutputStream(new BufferedOutputStream(fsync ? new AtomicFileOutputStream(snapShot) : new FileOutputStream(snapShot)), new Adler32())) { //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt, sessions, oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); crcOut.flush(); } } }
private static void createIndexFile(File indexFile, Configuration conf) throws IOException { if (indexFile.exists()) { System.out.println("Deleting existing file"); indexFile.delete(); } indexFile.createNewFile(); FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append( new Path(indexFile.getAbsolutePath())); Checksum crc = new PureJavaCrc32(); crc.reset(); CheckedOutputStream chk = new CheckedOutputStream(output, crc); String msg = "Writing new index file. This file will be used only " + "for the testing."; chk.write(Arrays.copyOf(msg.getBytes(), MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH)); output.writeLong(chk.getChecksum().getValue()); output.close(); }
static byte[] getManifestAsBytes(int nchars) throws IOException { crc.reset(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); CheckedOutputStream cos = new CheckedOutputStream(baos, crc); PrintStream ps = new PrintStream(cos); ps.println("Manifest-Version: 1.0"); ps.print("Main-Class: "); for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) { ps.print(i%10); } ps.println(SOME_KLASS); cos.flush(); cos.close(); ps.close(); return baos.toByteArray(); }
/** * Compress a text file using the ZIP compressing algorithm. * * @param filename the path to the file to be compressed */ public static void zipCompress(String filename) throws IOException { FileOutputStream fos = new FileOutputStream(filename + COMPRESSION_SUFFIX); CheckedOutputStream csum = new CheckedOutputStream(fos, new CRC32()); ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(csum)); out.setComment("Failmon records."); BufferedReader in = new BufferedReader(new FileReader(filename)); out.putNextEntry(new ZipEntry(new File(filename).getName())); int c; while ((c = in.read()) != -1) out.write(c); in.close(); out.finish(); out.close(); }
/** * serialize the datatree and session into the file snapshot * @param dt the datatree to be serialized * @param sessions the sessions to be serialized * @param snapShot the file to store snapshot into */ @Override public synchronized void serialize(DataTree dt, Map<Long, Long> sessions, File snapShot) throws IOException { if (!close) { OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(snapShot)); CheckedOutputStream crcOut = new CheckedOutputStream(sessOS, new Adler32()); //CheckedOutputStream cout = new CheckedOutputStream() OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); serialize(dt,sessions,oa, header); long val = crcOut.getChecksum().getValue(); oa.writeLong(val, "val"); oa.writeString("/", "path"); sessOS.flush(); crcOut.close(); sessOS.close(); } }
public static void createIndexFile(File indexFile, Configuration conf) throws IOException { if (indexFile.exists()) { System.out.println("Deleting existing file"); indexFile.delete(); } indexFile.createNewFile(); FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append( new Path(indexFile.getAbsolutePath())); Checksum crc = new PureJavaCrc32(); crc.reset(); CheckedOutputStream chk = new CheckedOutputStream(output, crc); String msg = "Writing new index file. This file will be used only " + "for the testing."; chk.write(Arrays.copyOf(msg.getBytes(), MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH)); output.writeLong(chk.getChecksum().getValue()); output.close(); }
public void encode(OutputStream out) throws IOException { java.util.zip.CRC32 crc32 = new java.util.zip.CRC32(); CheckedOutputStream outChecked = new CheckedOutputStream(out, crc32); // Index Indicator outChecked.write(0x00); // Number of Records EncoderUtil.encodeVLI(outChecked, recordCount); // List of Records for (Iterator i = records.iterator(); i.hasNext(); ) { IndexRecord record = (IndexRecord)i.next(); EncoderUtil.encodeVLI(outChecked, record.unpadded); EncoderUtil.encodeVLI(outChecked, record.uncompressed); } // Index Padding for (int i = getIndexPaddingSize(); i > 0; --i) outChecked.write(0x00); // CRC32 long value = crc32.getValue(); for (int i = 0; i < 4; ++i) out.write((byte)(value >>> (i * 8))); }
public void copyFileToCache(Path inFilePath, Object cache_key) throws IOException { int inBufferSize = 32*1024; int copyBufferSize = 128*1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(inFilePath),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ testCopyFileToCacheWithBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); } }
public void copyCacheToFile(Object cache_key, Path outFilePath) throws IOException { int copyBufferSize = 512 * 1024; //copy buffer size try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(outFilePath)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffers ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double) (end - start) / 1000000) + " millis"); System.out.println("============================================"); } }
void generateBigFile() throws IOException { try ( CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(IN_FILE_PATH)), new CRC32()); ) { System.out.println("============ Generate Initial Big File ===================="); long start = System.nanoTime();; int size = IN_FILE_SIZE; for (int i = 0; i < size; i++) { os.write(i); } long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("CheckSum = " + os.getChecksum().getValue()); System.out.println("============================================"); } }
@Before public void copyFileToCache() throws Exception { int inBufferSize = 32 * 1024; int copyBufferSize = 128 * 1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ copyFileToCache ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); this.fileCheckSum = is.getChecksum().getValue(); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamSmallerCopyBuffer() throws Exception { int inBufferSize = 128 * 1024; //ehcache input stream internal buffer int outBufferSize = 128 * 1024; int copyBufferSize = 64 * 1024; //copy buffer size *smaller* than ehcache input stream internal buffer to make sure it works that way try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key, inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH),outBufferSize), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamSmallerCopyBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamLargerCopyBuffer() throws Exception { int inBufferSize = 128 * 1024; //ehcache input stream internal buffer int outBufferSize = 128 * 1024; int copyBufferSize = 357 * 1024; //copy buffer size *larger* than ehcache input stream internal buffer to make sure it works that way try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key, inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH),outBufferSize), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamLargerCopyBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamDefaultBuffers() throws Exception { int copyBufferSize = 512 * 1024; //copy buffer size try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffers ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamDefaultBuffersByteByByte() throws Exception { try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffersByteByByte ===================="); long start = System.nanoTime();; pipeStreamsByteByByte(is, os); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileNoCacheKey() throws Exception { int copyBufferSize = 512 * 1024; //copy buffer size final String cacheKey = "something-else"; try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cacheKey),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileNoCacheKey ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(0, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), 0); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
private static void generateBigFile() throws Exception { try ( CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(IN_FILE_PATH)), new CRC32()); ) { System.out.println("============ Generate Initial Big File ===================="); long start = System.nanoTime();; int size = IN_FILE_SIZE; for (int i = 0; i < size; i++) { os.write(i); } long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("CheckSum = " + os.getChecksum().getValue()); System.out.println("============================================"); } }
@Test public void testCopyFileToFile() throws Exception { try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH)), new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ testCopyFileToFile ===================="); int copyBufferSize = 32*1024; long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
