/** * Constructor. * @throws IOException */ public SharedSystemData(String path, boolean create) throws IOException { File f = new File(path); if (create) { if (f.exists()) { System.out.println("Existing system detected, deleting"); f.delete(); // Delete if present. } } else { if (!f.exists()) { System.err.println("ERROR, system dont exist"); System.exit(-1); } } channel = FileChannel.open(f.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); buffer = channel.map(MapMode.READ_WRITE, 0, 4000); if (create) { setNextTaskId(0); setShutdownSignal(false); } }
public ByteBuffer call() throws Exception { ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); buff = ByteBuffer.allocate(bufferSize); serverSocketChannel.socket().bind(new InetSocketAddress(port)); while (!stop.isLocked()) { RandomAccessFile temp = new RandomAccessFile(tempName, "rw"); SocketChannel socketChannel = serverSocketChannel.accept(); socketChannel.read(buff); FileChannel channel = temp.getChannel(); channel.write(buff); if (!pause.isLocked()) { MappedByteBuffer b = channel.map(MapMode.READ_WRITE, 0, (long) bufferSize); b.clear(); } temp.close(); buff.clear(); } return null; }
public ByteBuffer call() throws Exception { ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); buff = ByteBuffer.allocate(bufferSize); serverSocketChannel.socket().bind(new InetSocketAddress(port)); RandomAccessFile temp = new RandomAccessFile(tempName, "rw"); MappedByteBuffer b; while (!stop.isLocked()) { sync=0; SocketChannel socketChannel = serverSocketChannel.accept(); socketChannel.read(buff); FileChannel channel = temp.getChannel(); channel.write(buff); if (!pause.isLocked()) { b = channel.map(MapMode.READ_WRITE, 0, (long) bufferSize); sync = 1; if(sync==2){ b.clear(); } } buff.clear(); } temp.close(); return null; }
/** * Maps the heapPointer and code sections of the boot image in a given file into memory. * * @param bootImageFile the file containing the heapPointer and code sections to map into memory * @return a {@link DataAccess} object that can be used to access the mapped sections * @throws IOException if an IO error occurs while performing the memory mapping */ private DataAccess map(File bootImageFile, BootImage bootImage) throws IOException { final RandomAccessFile randomAccessFile = new RandomAccessFile(bootImageFile, "rwd"); final Header header = bootImage.header; int heapOffset = bootImage.heapOffset(); int heapAndCodeSize = header.heapSize + header.codeSize; final MappedByteBuffer bootImageBuffer = randomAccessFile.getChannel().map(MapMode.PRIVATE, heapOffset, heapAndCodeSize); bootImageBuffer.order(platform().endianness().asByteOrder()); randomAccessFile.close(); if (heapPointer.isNotZero()) { long address = (Long) WithoutAccessCheck.getInstanceField(bootImageBuffer, "address"); bootImage.relocate(address, heapPointer); } return new MappedByteBufferDataAccess(bootImageBuffer, heapPointer, header.wordWidth()); }
private boolean checkVersion(FileChannel channel) throws IOException { if (channel.size() > 0) { channel.position(0); ByteBuffer buffer; if (useNIOMemoryMapping) { MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, 8); mbb.load(); buffer = mbb; } else { buffer = ByteBuffer.wrap(new byte[8]); channel.read(buffer); buffer.position(0); } buffer.position(0); long onDiskVersion = buffer.getLong(); return (version == onDiskVersion); } return (version == 0); }
public StoreCheckpoint(final String scpPath) throws IOException { File file = new File(scpPath); MappedFile.ensureDirOK(file.getParent()); boolean fileExists = file.exists(); this.randomAccessFile = new RandomAccessFile(file, "rw"); this.fileChannel = this.randomAccessFile.getChannel(); this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MappedFile.OS_PAGE_SIZE); if (fileExists) { log.info("store checkpoint file exists, " + scpPath); this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0); this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8); this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16); log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp)); log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp)); log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp)); } else { log.info("store checkpoint file not exists, " + scpPath); } }
public void testMap_readWrite() throws IOException { // Test data int size = 1024; byte[] expectedBytes = new byte[size]; byte[] bytes = newPreFilledByteArray(1024); // Setup File file = createTempFile(); Files.write(bytes, file); Random random = new Random(); random.nextBytes(expectedBytes); // Test MappedByteBuffer map = Files.map(file, MapMode.READ_WRITE); map.put(expectedBytes); // Verify byte[] actualBytes = Files.toByteArray(file); assertTrue(Arrays.equals(expectedBytes, actualBytes)); }
public void testMap_readWrite_creates() throws IOException { // Test data int size = 1024; byte[] expectedBytes = newPreFilledByteArray(1024); // Setup File file = createTempFile(); boolean deleted = file.delete(); assertTrue(deleted); assertFalse(file.exists()); // Test MappedByteBuffer map = Files.map(file, MapMode.READ_WRITE, size); map.put(expectedBytes); // Verify assertTrue(file.exists()); assertTrue(file.isFile()); assertEquals(size, file.length()); byte[] actualBytes = Files.toByteArray(file); assertTrue(Arrays.equals(expectedBytes, actualBytes)); }
public StoreCheckpoint(final String scpPath) throws IOException { File file = new File(scpPath); MapedFile.ensureDirOK(file.getParent()); boolean fileExists = file.exists(); this.randomAccessFile = new RandomAccessFile(file, "rw"); this.fileChannel = this.randomAccessFile.getChannel(); this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MapedFile.OS_PAGE_SIZE); if (fileExists) { log.info("store checkpoint file exists, " + scpPath); this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0); this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8); this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16); log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp)); log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp)); log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", " + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp)); } else { log.info("store checkpoint file not exists, " + scpPath); } }
/** * Insert new metadata into file by using memory mapped file, and if fails write in chunks * <p> * But this is problematic on 32bit systems for large flac files may not be able to map a contiguous address space large enough * for a large audio size , so no longer used since better to go straight to using chunks * * @param tag * @param fc * @param blockInfo * @param flacStream * @param neededRoom * @param availableRoom * @throws IOException * @throws UnsupportedEncodingException */ private void insertTagAndShift(File file, Tag tag, FileChannel fc, MetadataBlockInfo blockInfo, FlacStreamReader flacStream, int neededRoom, int availableRoom) throws IOException, UnsupportedEncodingException { int headerLength = flacStream.getStartOfFlacInFile() + FlacStreamReader.FLAC_STREAM_IDENTIFIER_LENGTH + MetadataBlockHeader.HEADER_LENGTH // this should be the length of the block header for the stream info + MetadataBlockDataStreamInfo.STREAM_INFO_DATA_LENGTH; long targetSizeBeforeAudioData = headerLength + neededRoom + FlacTagCreator.DEFAULT_PADDING; long remainderTargetSize = fc.size() - (headerLength + availableRoom); long totalTargetSize = targetSizeBeforeAudioData + remainderTargetSize; MappedByteBuffer mappedFile = null; try { //Use ByteBuffer mappedFile = fc.map(MapMode.READ_WRITE, 0, totalTargetSize); insertTagAndShiftViaMappedByteBuffer(tag, mappedFile, fc, targetSizeBeforeAudioData, totalTargetSize, blockInfo, flacStream, neededRoom, availableRoom); } catch (IOException ioe) { //#175: Flac Map error on write if (mappedFile == null) { insertUsingChunks(file, tag, fc, blockInfo, flacStream, neededRoom + FlacTagCreator.DEFAULT_PADDING, availableRoom); } else { logger.log(Level.SEVERE, ioe.getMessage(), ioe); throw ioe; } } }
/** * Load the block. * * mmap and mlock the block, and then verify its checksum. * * @param length The current length of the block. * @param blockIn The block input stream. Should be positioned at the * start. The caller must close this. * @param metaIn The meta file input stream. Should be positioned at * the start. The caller must close this. * @param blockFileName The block file name, for logging purposes. * * @return The Mappable block. */ public static MappableBlock load(long length, FileInputStream blockIn, FileInputStream metaIn, String blockFileName) throws IOException { MappableBlock mappableBlock = null; MappedByteBuffer mmap = null; FileChannel blockChannel = null; try { blockChannel = blockIn.getChannel(); if (blockChannel == null) { throw new IOException("Block InputStream has no FileChannel."); } mmap = blockChannel.map(MapMode.READ_ONLY, 0, length); NativeIO.POSIX.getCacheManipulator().mlock(blockFileName, mmap, length); verifyChecksum(length, metaIn, blockChannel, blockFileName); mappableBlock = new MappableBlock(mmap, length); } finally { IOUtils.closeQuietly(blockChannel); if (mappableBlock == null) { if (mmap != null) { NativeIO.POSIX.munmap(mmap); // unmapping also unlocks } } } return mappableBlock; }
/** * Maps blah file with a random offset and checks to see if data * written out to the file can be read back in */ private static void testWrite() throws Exception { StringBuilder sb = new StringBuilder(); sb.setLength(4); for (int x=0; x<1000; x++) { try (RandomAccessFile raf = new RandomAccessFile(blah, "rw")) { FileChannel fc = raf.getChannel(); long offset = generator.nextInt(1000); MappedByteBuffer b = fc.map(MapMode.READ_WRITE, offset, 100); for (int i=0; i<4; i++) { b.put(i, (byte)('0' + i)); } for (int i=0; i<4; i++) { byte aByte = b.get(i); sb.setCharAt(i, (char)aByte); } if (!sb.toString().equals("0123")) throw new Exception("Write test failed"); } } }
/** * load bio information;egg : chromosome or dbsnp */ protected void loadInformation(String path) throws IOException { RandomAccessFile raf = new RandomAccessFile(path, "r"); FileChannel fc = raf.getChannel(); fcSize = (int) (fc.size() & 0xffffffff); int blocks = (int) ((fcSize / Integer.MAX_VALUE) + 1); byteBuffer = new MappedByteBuffer[blocks]; int start = 0; long remain = 0; int size = 0; for (int i = 0; i < blocks; i++) { start = Integer.MAX_VALUE * i; remain = (long) (fc.size() - start); size = (int) ((remain > Integer.MAX_VALUE) ? Integer.MAX_VALUE : remain); MappedByteBuffer mapedBB = fc.map(MapMode.READ_ONLY, start, size); byteBuffer[i] = mapedBB; } raf.close(); }
/** * on posix systems: allocates disk-backed bytebuffer and immediately unlinks the file * on others: simply returns a direct bytebuffer */ public static ByteBuffer allocate(int size) { if(MAP_AND_UNLINK_SUPPORTED) { try { Path p = Files.createTempFile("anon-mapping", ".tmp"); ByteBuffer mapped; FileChannel chan = FileChannel.open(p, StandardOpenOption.READ, StandardOpenOption.WRITE); chan.position(size); chan.write(ByteBuffer.allocate(1)); mapped = chan.map(MapMode.READ_WRITE, 0, size); chan.close(); Files.delete(p); return mapped; } catch (IOException e) { e.printStackTrace(); } } return ByteBuffer.allocateDirect(size); }
BootFile(Path bootPath) throws IOException { Objects.requireNonNull(bootPath); _bootPath = bootPath; long bootSize = Files.size(bootPath); if (bootSize <= 0) { throw new IllegalStateException("Unexpected boot size for " + bootPath); } if (bootSize >= Integer.MAX_VALUE - 1) { throw new IllegalStateException("Mmapped file is too large for " + bootPath + " " + _bootSize); } _bootSize = (int) bootSize; _bootChannel = (FileChannel) Files.newByteChannel(_bootPath, StandardOpenOption.READ); _bootMap = _bootChannel.map(MapMode.READ_ONLY, 0, _bootSize); readJar(); readManifest(); }
static LargeLongArray create(Iterator<Long> iterator, int size) { try { File file = File.createTempFile("list", ".tmp"); file.deleteOnExit(); try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { FileChannel channel = raf.getChannel(); MappedByteBuffer map = channel .map(MapMode.READ_WRITE, 0, size * 8L); for (int i = 0; i < size; i++) { long x = iterator.next(); map.putLong(x); } return new LargeLongArray(size, file, channel, map); } } catch (IOException e) { throw new RuntimeException(e); } }
/** Maps a file into a set of buffers */ ByteBuffer[] map(RandomAccessFile raf, long offset, long length) throws IOException { if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString()); final long chunkSize = 1L << chunkSizePower; // we always allocate one more buffer, the last one may be a 0 byte one final int nrBuffers = (int) (length >>> chunkSizePower) + 1; ByteBuffer buffers[] = new ByteBuffer[nrBuffers]; long bufferStart = 0L; FileChannel rafc = raf.getChannel(); for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { int bufSize = (int) ( (length > (bufferStart + chunkSize)) ? chunkSize : (length - bufferStart) ); buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); bufferStart += bufSize; } return buffers; }
static final FileSegment create(File nsDir, int segmentNumber, int dataSegmentSize, SyncMode syncMode, NamespaceOptions nsOptions) throws IOException { RandomAccessFile raFile; byte[] header; ByteBuffer dataBuf; int indexOffset; indexOffset = dataSegmentSize; raFile = new RandomAccessFile(fileForSegment(nsDir, segmentNumber), syncModeToFileOpenMode(syncMode)); header = SegmentFormat.newHeader(segmentNumber, dataOffset, indexOffset); dataBuf = raFile.getChannel().map(MapMode.READ_WRITE, 0, dataSegmentSize); dataBuf.put(header); //raFile.getFD().sync(); // For now we leave this out and let SyncMode cover this return new FileSegment(nsDir, segmentNumber, raFile, dataBuf, dataSegmentSize, nsOptions); }
public static byte[] readBytesByMapped(String path) { FileChannel fc = null; try { fc = new RandomAccessFile(path, "r").getChannel(); MappedByteBuffer byteBuffer = fc.map(MapMode.READ_ONLY, 0, fc.size()).load(); byte[] bytes = new byte[(int) fc.size()]; if (byteBuffer.remaining() > 0) { byteBuffer.get(bytes, 0, byteBuffer.remaining()); } fc.close(); return bytes; } catch (Exception e) { Log.logger.warning("读取" + path + "时发生异常" + e.getMessage()); } return null; }
public void open(MapMode mode, int fileSize) throws IOException { int fileLength = (int)this.file.length(); int size = (mode == MapMode.READ_WRITE) ? fileSize : fileLength; this.mmf = new MemoryMappedFile(this.file, size, mode == MapMode.READ_WRITE ? "rw" : "r"); if (mode == MapMode.READ_WRITE) { File parent = file.getAbsoluteFile().getParentFile(); long free = parent.getUsableSpace(); if ((fileSize * 4) > free) { throw new HumpbackException("out of storage space: " + this.file.toString() + ' ' + free); } } if (mode == MapMode.READ_WRITE) { this.mmf.buf.load(); } this.mmf.buf.order(ByteOrder.LITTLE_ENDIAN); this.addr = this.mmf.getAddress(); this.spStart = SpaceManager.makeSpacePointer(this.id, 0); this.spEnd = fileSize + this.spStart; this.allocPointer = new AtomicInteger(fileLength); this.capacity = this.mmf.buf.capacity(); }
private CrailBuffer allocateRegion() throws IOException { if (currentRegion >= allocationCount){ return null; } String path = directory + "/" + currentRegion++; RandomAccessFile randomFile = new RandomAccessFile(path, "rw"); randomFile.setLength(CrailConstants.REGION_SIZE); FileChannel channel = randomFile.getChannel(); MappedByteBuffer _mappedBuffer = channel.map(MapMode.READ_WRITE, 0, CrailConstants.REGION_SIZE); CrailBuffer mappedBuffer = OffHeapBuffer.wrap(_mappedBuffer); randomFile.close(); channel.close(); CrailBuffer firstBuffer = slice(mappedBuffer, 0); for (int j = 1; j < bufferCount; j++) { int position = j * CrailConstants.BUFFER_SIZE; CrailBuffer sliceBuffer = slice(mappedBuffer, position); this.putBufferInternal(sliceBuffer); } mappedBuffer.clear(); return firstBuffer; }
private TrieDictionary(File data) throws IOException { this.array = null; FileInputStream ins = new FileInputStream(data); FileChannel channel = ins.getChannel(); try { this.buffer = channel.map(MapMode.READ_ONLY, 0, channel.size()); } finally { channel.close(); ins.close(); } }
private boolean enlargeFile(long newFileLength) { try { long newBufferLength = newFileLength; if (!readOnly) { newBufferLength = largeBufferSize; } MapMode mapMode = readOnly ? FileChannel.MapMode.READ_ONLY : FileChannel.MapMode.READ_WRITE; if (!readOnly && file.length() < fileLength + newBufferLength) { file.seek(fileLength + newBufferLength - 1); file.writeByte(0); } MappedByteBuffer[] newBuffers = new MappedByteBuffer[buffers.length + 1]; MappedByteBuffer newBuffer = channel.map(mapMode, fileLength, newBufferLength); System.arraycopy(buffers, 0, newBuffers, 0, buffers.length); newBuffers[buffers.length] = newBuffer; buffers = newBuffers; fileLength += newBufferLength; logger.logDetailEvent("NIO buffer instance, file size " + fileLength); } catch (Throwable e) { logger.logDetailEvent("NOI buffer allocate failed, file size " + newFileLength); return false; } return true; }
/** * 将文件转换为字节数组, * 使用MappedByteBuffer,可以在处理大文件时,提升性能 * * @param file 文件 * @return 二维码图片的字节数组 */ private static byte[] toByteArray(File file) { try (FileChannel fc = new RandomAccessFile(file, "r").getChannel();) { MappedByteBuffer byteBuffer = fc.map(MapMode.READ_ONLY, 0, fc.size()).load(); byte[] result = new byte[(int) fc.size()]; if (byteBuffer.remaining() > 0) { byteBuffer.get(result, 0, byteBuffer.remaining()); } return result; } catch (Exception e) { logger.warn("文件转换成byte[]发生异常!", e); return null; } }
public static String computeHash(File file) throws IOException { if (!file.isFile()) { return ""; } long size = file.length(); long chunkSizeForFile = Math.min(HASH_CHUNK_SIZE, size); try (FileChannel fileChannel = new FileInputStream(file).getChannel()) { long head = computeHashForChunk(fileChannel.map(MapMode.READ_ONLY, 0, chunkSizeForFile)); long tail = computeHashForChunk(fileChannel.map(MapMode.READ_ONLY, Math.max(size - HASH_CHUNK_SIZE, 0), chunkSizeForFile)); return String.format("%016x", size + head + tail); } }
private static MappedByteBuffer map(RandomAccessFile raf, MapMode mode, long size) throws IOException { Closer closer = Closer.create(); try { FileChannel channel = closer.register(raf.getChannel()); return channel.map(mode, 0, size); } catch (Throwable e) { throw closer.rethrow(e); } finally { closer.close(); } }
public PackageSanityTests() { setDefault(BaseEncoding.class, BaseEncoding.base64()); setDefault(int.class, 32); setDefault(String.class, "abcd"); setDefault(Method.class, AbstractPackageSanityTests.class.getDeclaredMethods()[0]); setDefault(MapMode.class, MapMode.READ_ONLY); setDefault(CharsetEncoder.class, Charsets.UTF_8.newEncoder()); }
public void testMap_readWrite_max_value_plus_1() throws IOException { // Setup File file = createTempFile(); // Test try { Files.map(file, MapMode.READ_WRITE, (long) Integer.MAX_VALUE + 1); fail("Should throw when size exceeds Integer.MAX_VALUE"); } catch (IllegalArgumentException expected) { } }