CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize); minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemstoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7); // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F); minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); LOG.info(this); }
CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemstoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7); // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F); minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); LOG.info(this); }
CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE); minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size", storeConfigInfo.getMemstoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(MIN_KEY, /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(MAX_KEY, 10); compactionRatio = conf.getFloat(RATIO_KEY, 1.2F); offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); shouldDeleteExpired = conf.getBoolean("hbase.store.delete.expired.storefile", true); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7); // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F); LOG.info(this); }
CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE); minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size", storeConfigInfo.getMemstoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(MIN_KEY, /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(MAX_KEY, 10); compactionRatio = conf.getFloat(RATIO_KEY, 1.2F); offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7); // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F); LOG.info(this); }
CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE); minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size", storeConfigInfo.getMemstoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(CONFIG_PREFIX + "min", /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(CONFIG_PREFIX + "max", 10); compactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio", 1.2F); offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); shouldDeleteExpired = conf.getBoolean("hbase.store.delete.expired.storefile", true); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24*7); // Make it 0.5 so jitter has us fall evenly either side of when the compaction should run majorCompactionJitter = conf.getFloat("hbase.hregion.majorcompaction.jitter", 0.50F); LOG.info(this); }
private static StripeCompactionPolicy createPolicy(Configuration conf, long splitSize, float splitCount, int initialCount, boolean hasTtl) throws Exception { conf.setLong(StripeStoreConfig.SIZE_TO_SPLIT_KEY, splitSize); conf.setFloat(StripeStoreConfig.SPLIT_PARTS_KEY, splitCount); conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, initialCount); StoreConfigInformation sci = mock(StoreConfigInformation.class); when(sci.getStoreFileTtl()).thenReturn(hasTtl ? defaultTtl : Long.MAX_VALUE); StripeStoreConfig ssc = new StripeStoreConfig(conf, sci); return new StripeCompactionPolicy(conf, sci, ssc); }
/** * Test the perf of a CompactionPolicy with settings. * @param cpClass The compaction policy to test * @param inMmax The maximum number of file to compact * @param inMin The min number of files to compact * @param inRatio The ratio that files must be under to be compacted. */ public PerfTestCompactionPolicies( final Class<? extends CompactionPolicy> cpClass, final Class<? extends StoreFileListGenerator> fileGenClass, final int inMmax, final int inMin, final float inRatio) throws IllegalAccessException, InstantiationException { super(PerfTestCompactionPolicies.class); this.fileGenClass = fileGenClass; this.max = inMmax; this.min = inMin; this.ratio = inRatio; // Hide lots of logging so the system out is usable as a tab delimited file. org.apache.log4j.Logger.getLogger(CompactionConfiguration.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR); Configuration configuration = HBaseConfiguration.create(); // Make sure that this doesn't include every file. configuration.setInt("hbase.hstore.compaction.max", max); configuration.setInt("hbase.hstore.compaction.min", min); configuration.setFloat("hbase.hstore.compaction.ratio", ratio); store = createMockStore(); this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(), new Class[] {Configuration.class, StoreConfigInformation.class }, new Object[] {configuration, store }); this.generator = fileGenClass.newInstance(); // Used for making paths }
CompactionConfiguration(Configuration conf, StoreConfigInformation storeConfigInfo) { this.conf = conf; this.storeConfigInfo = storeConfigInfo; maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE); offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, maxCompactSize); minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, storeConfigInfo.getMemStoreFlushSize()); minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY, /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10); compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F); offPeakCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F); throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", 2 * maxFilesToCompact * storeConfigInfo.getMemStoreFlushSize()); majorCompactionPeriod = conf.getLong(HConstants.MAJOR_COMPACTION_PERIOD, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); majorCompactionJitter = conf.getFloat(HConstants.MAJOR_COMPACTION_JITTER, HConstants.DEFAULT_MAJOR_COMPACTION_JITTER); minLocalityToForceCompact = conf.getFloat(HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT, 0f); dateTieredMaxStoreFileAgeMillis = conf.getLong(DATE_TIERED_MAX_AGE_MILLIS_KEY, Long.MAX_VALUE); dateTieredIncomingWindowMin = conf.getInt(DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 6); compactionPolicyForDateTieredWindow = conf.get( COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS_KEY, DEFAULT_COMPACTION_POLICY_CLASS_FOR_DATE_TIERED_WINDOWS.getName()); dateTieredSingleOutputForMinorCompaction = conf .getBoolean(DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, true); this.dateTieredCompactionWindowFactory = conf.get( DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS_KEY, DEFAULT_DATE_TIERED_COMPACTION_WINDOW_FACTORY_CLASS.getName()); LOG.info(toString()); }
/** * Test the perf of a CompactionPolicy with settings. * @param cpClass The compaction policy to test * @param inMmax The maximum number of file to compact * @param inMin The min number of files to compact * @param inRatio The ratio that files must be under to be compacted. */ public PerfTestCompactionPolicies( final Class<? extends CompactionPolicy> cpClass, final Class<? extends StoreFileListGenerator> fileGenClass, final int inMmax, final int inMin, final float inRatio) throws IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { super(PerfTestCompactionPolicies.class); this.fileGenClass = fileGenClass; this.max = inMmax; this.min = inMin; this.ratio = inRatio; // Hide lots of logging so the system out is usable as a tab delimited file. org.apache.log4j.Logger.getLogger(CompactionConfiguration.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR); Configuration configuration = HBaseConfiguration.create(); // Make sure that this doesn't include every file. configuration.setInt("hbase.hstore.compaction.max", max); configuration.setInt("hbase.hstore.compaction.min", min); configuration.setFloat("hbase.hstore.compaction.ratio", ratio); store = createMockStore(); this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(), new Class[] {Configuration.class, StoreConfigInformation.class }, new Object[] {configuration, store }); this.generator = fileGenClass.getDeclaredConstructor().newInstance(); // Used for making paths }
public CompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) { this.storeConfigInfo = storeConfigInfo; this.comConf = new CompactionConfiguration(conf, this.storeConfigInfo); }
public FIFOCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); }
public RatioBasedCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); }
public StripeCompactionPolicy( Configuration conf, StoreConfigInformation storeConfigInfo, StripeStoreConfig config) { super(conf, storeConfigInfo); this.config = config; stripePolicy = new ExploringCompactionPolicy(conf, storeConfigInfo); }
public SortedCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); }
/** * Constructor for ExploringCompactionPolicy. * @param conf The configuration object * @param storeConfigInfo An object to provide info about the store. */ public ExploringCompactionPolicy(final Configuration conf, final StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); }
/** * Constructor. * * @param conf The Conf. * @param storeConfigInfo Info about the store. */ public EverythingPolicy(final Configuration conf, final StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); }
