/** * Set up variables and get BucketCache and WriterThread into state where tests can manually * control the running of WriterThread and BucketCache is empty. * @throws Exception */ @Before public void setUp() throws Exception { // Arbitrary capacity. final int capacity = 16; // Run with one writer thread only. Means there will be one writer queue only too. We depend // on this in below. final int writerThreadsCount = 1; this.bc = new MockBucketCache("heap", capacity, 1, new int [] {1}, writerThreadsCount, capacity, null, 100/*Tolerate ioerrors for 100ms*/); assertEquals(writerThreadsCount, bc.writerThreads.length); assertEquals(writerThreadsCount, bc.writerQueues.size()); // Get reference to our single WriterThread instance. this.wt = bc.writerThreads[0]; this.q = bc.writerQueues.get(0); wt.disableWriter(); this.plainKey = new BlockCacheKey("f", 0); this.plainCacheable = Mockito.mock(Cacheable.class); assertThat(bc.ramCache.isEmpty(), is(true)); assertTrue(q.isEmpty()); }
@Override public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) { CacheablePair contentBlock = backingMap.get(key); if (contentBlock == null) { if (!repeat) stats.miss(caching); return null; } stats.hit(caching); // If lock cannot be obtained, that means we're undergoing eviction. try { contentBlock.recentlyAccessed.set(System.nanoTime()); synchronized (contentBlock) { if (contentBlock.serializedData == null) { // concurrently evicted LOG.warn("Concurrent eviction of " + key); return null; } return contentBlock.deserializer .deserialize(contentBlock.serializedData.asReadOnlyBuffer()); } } catch (Throwable t) { LOG.error("Deserializer threw an exception. This may indicate a bug.", t); return null; } }
/** * Cache the block with the specified key and buffer. First finds what size * SingleSlabCache it should fit in. If the block doesn't fit in any, it will * return without doing anything. * <p> * It is assumed this will NEVER be called on an already cached block. If that * is done, it is assumed that you are reinserting the same exact block due to * a race condition, and will throw a runtime exception. * * @param cacheKey block cache key * @param cachedItem block buffer */ public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem) { Entry<Integer, SingleSizeCache> scacheEntry = getHigherBlock(cachedItem .getSerializedLength()); this.requestStats.addin(cachedItem.getSerializedLength()); if (scacheEntry == null) { return; // we can't cache, something too big. } this.successfullyCachedStats.addin(cachedItem.getSerializedLength()); SingleSizeCache scache = scacheEntry.getValue(); /* * This will throw a runtime exception if we try to cache the same value * twice */ scache.cacheBlock(cacheKey, cachedItem); }
/** * Get the buffer of the block with the specified name. * @param caching * @param key * @param repeat * * @return buffer of specified block name, or null if not in cache */ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) { SingleSizeCache cachedBlock = backingStore.get(key); if (cachedBlock == null) { if (!repeat) stats.miss(caching); return null; } Cacheable contentBlock = cachedBlock.getBlock(key, caching, false); if (contentBlock != null) { stats.hit(caching); } else { if (!repeat) stats.miss(caching); } return contentBlock; }
/** * Get the buffer of the block with the specified name. * * @return buffer of specified block name, or null if not in cache */ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, boolean updateCacheMetrics) { SingleSizeCache cachedBlock = backingStore.get(key); if (cachedBlock == null) { if (!repeat) stats.miss(caching); return null; } Cacheable contentBlock = cachedBlock.getBlock(key, caching, false, updateCacheMetrics); if (contentBlock != null) { if (updateCacheMetrics) stats.hit(caching); } else if (!repeat) { if (updateCacheMetrics) stats.miss(caching); } return contentBlock; }
/** * Transfers data from file to the given byte buffer * @param offset The offset in the file where the first byte to be read * @param length The length of buffer that should be allocated for reading * from the file channel * @return number of bytes read * @throws IOException */ @Override public Cacheable read(long offset, int length, CacheableDeserializer<Cacheable> deserializer) throws IOException { Preconditions.checkArgument(length >= 0, "Length of read can not be less than 0."); ByteBuffer dstBuffer = ByteBuffer.allocate(length); if (length != 0) { accessFile(readAccessor, dstBuffer, offset); // The buffer created out of the fileChannel is formed by copying the data from the file // Hence in this case there is no shared memory that we point to. Even if the BucketCache evicts // this buffer from the file the data is already copied and there is no need to ensure that // the results are not corrupted before consuming them. if (dstBuffer.limit() != length) { throw new RuntimeException("Only " + dstBuffer.limit() + " bytes read, " + length + " expected"); } } return deserializer.deserialize(new SingleByteBuff(dstBuffer), true, MemoryType.EXCLUSIVE); }
/** * Set up variables and get BucketCache and WriterThread into state where tests can manually * control the running of WriterThread and BucketCache is empty. * @throws Exception */ @Before public void setUp() throws Exception { // Arbitrary capacity. final int capacity = 16; // Run with one writer thread only. Means there will be one writer queue only too. We depend // on this in below. final int writerThreadsCount = 1; this.bc = new MockBucketCache("offheap", capacity, 1, new int [] {1}, writerThreadsCount, capacity, null, 100/*Tolerate ioerrors for 100ms*/); assertEquals(writerThreadsCount, bc.writerThreads.length); assertEquals(writerThreadsCount, bc.writerQueues.size()); // Get reference to our single WriterThread instance. this.wt = bc.writerThreads[0]; this.q = bc.writerQueues.get(0); wt.disableWriter(); this.plainKey = new BlockCacheKey("f", 0); this.plainCacheable = Mockito.mock(Cacheable.class); assertThat(bc.ramCache.isEmpty(), is(true)); assertTrue(q.isEmpty()); }
@Override public Cacheable getBlock(BlockCacheKey key, boolean caching) { CacheablePair contentBlock = backingMap.get(key); if (contentBlock == null) { stats.miss(caching); return null; } stats.hit(caching); // If lock cannot be obtained, that means we're undergoing eviction. try { contentBlock.recentlyAccessed.set(System.nanoTime()); synchronized (contentBlock) { if (contentBlock.serializedData == null) { // concurrently evicted LOG.warn("Concurrent eviction of " + key); return null; } return contentBlock.deserializer .deserialize(contentBlock.serializedData.asReadOnlyBuffer()); } } catch (Throwable t) { LOG.error("Deserializer threw an exception. This may indicate a bug.", t); return null; } }
/** * Get the buffer of the block with the specified name. * * @param key * @param caching * @return buffer of specified block name, or null if not in cache */ public Cacheable getBlock(BlockCacheKey key, boolean caching) { SingleSizeCache cachedBlock = backingStore.get(key); if (cachedBlock == null) { stats.miss(caching); return null; } Cacheable contentBlock = cachedBlock.getBlock(key, caching); if (contentBlock != null) { stats.hit(caching); } else { stats.miss(caching); } return contentBlock; }
/** * Store external. * * @param blockName the block name * @param buf the buf * @param inMemory the in memory * @throws IOException Signals that an I/O exception has occurred. */ @SuppressWarnings("unused") private void storeExternal(String blockName, Cacheable buf, boolean inMemory) throws IOException{ // If external storage is disable - bail out if(overflowExtEnabled == false) return; // Check if we have already this block in external storage cache if(extStorageCache.contains(blockName)) return; ByteBuffer buffer = extStorageCache.getLocalBufferWithAddress().getBuffer(); deserializer.set(buf.getDeserializer()); buffer.clear(); buffer.position(4); buffer.put( inMemory? (byte) 1: (byte) 0); buf.serialize(buffer); buffer.putInt(0, buffer.position() - 4); StorageHandle handle = storage.storeData(buffer); try{ extStorageCache.put(blockName, handle); } catch(Exception e){ throw new IOException(e); } }
@SuppressWarnings("unchecked") public static void setHFileDeserializer() { Field field = getProtectedField(HFileBlock.class, "blockDeserializer"); if (field == null){ LOG.error("Could not get access to HFileBlock.blockDeserializer"); return; } try { CacheableDeserializer<Cacheable> serde = (CacheableDeserializer<Cacheable>) field.get(null); if(serde != null){ deserializer.set(serde); } else{ LOG.warn("HFileBlock.blockDeserializer is null"); } } catch (Exception e) { LOG.warn("unable to read HFileBlock.blockDeserializer"); } }
/** * Get the buffer of the block with the specified name. * @param caching * @param key * * @return buffer of specified block name, or null if not in cache */ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat) { SingleSizeCache cachedBlock = backingStore.get(key); if (cachedBlock == null) { if (!repeat) stats.miss(caching); return null; } Cacheable contentBlock = cachedBlock.getBlock(key, caching, false); if (contentBlock != null) { stats.hit(caching); } else if (!repeat) { stats.miss(caching); } return contentBlock; }
/** * Cache the block to ramCache * @param cacheKey block's cache key * @param cachedItem block buffer * @param inMemory if block is in-memory * @param wait if true, blocking wait when queue is full */ public void cacheBlockWithWait(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory, boolean wait) { if (!cacheEnabled) { return; } if (backingMap.containsKey(cacheKey)) { return; } /* * Stuff the entry into the RAM cache so it can get drained to the persistent store */ RAMQueueEntry re = new RAMQueueEntry(cacheKey, cachedItem, accessCount.incrementAndGet(), inMemory); if (ramCache.putIfAbsent(cacheKey, re) != null) { return; } int queueNum = (cacheKey.hashCode() & 0x7FFFFFFF) % writerQueues.size(); BlockingQueue<RAMQueueEntry> bq = writerQueues.get(queueNum); boolean successfulAddition = false; if (wait) { try { successfulAddition = bq.offer(re, DEFAULT_CACHE_WAIT_TIME, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } else { successfulAddition = bq.offer(re); } if (!successfulAddition) { ramCache.remove(cacheKey); cacheStats.failInsert(); } else { this.blockNumber.incrementAndGet(); this.heapSize.addAndGet(cachedItem.heapSize()); blocksByHFile.put(cacheKey.getHfileName(), cacheKey); } }
public RAMQueueEntry(BlockCacheKey bck, Cacheable data, long accessCounter, boolean inMemory) { this.key = bck; this.data = data; this.accessCounter = accessCounter; this.inMemory = inMemory; }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, boolean cacheDataInL1) { if (super.getBlock(cacheKey, true, false, true) != null) { throw new RuntimeException("Cached an already cached block"); } super.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); }
