Java 类org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration 实例源码

项目:hbase    文件:TestDateTieredCompactionPolicy.java   
@Override
protected void config() {
  super.config();

  // Set up policy
  conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
    "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
  conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
  conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
  conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
  conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
  conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
    false);

  // Special settings for compaction policy per window
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
  this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);

  conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
  conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 5);
}
项目:hbase    文件:TestDateTieredCompactionPolicyOverflow.java   
@Override
protected void config() {
  super.config();

  // Set up policy
  conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
    "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
  conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
  conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
  conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, Long.MAX_VALUE / 2);
  conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 2);
  conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
    false);

  // Special settings for compaction policy per window
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
  this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);

  conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
  conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
}
项目:hbase    文件:TestCompactionWithThroughputController.java   
private long testCompactionWithoutThroughputLimit() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DefaultStoreEngine.class.getName());
  conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100);
  conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 200);
  conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
  conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY,
    NoLimitThroughputController.class.getName());
  TEST_UTIL.startMiniCluster(1);
  try {
    HStore store = prepareData();
    assertEquals(10, store.getStorefilesCount());
    long startTime = System.currentTimeMillis();
    TEST_UTIL.getAdmin().majorCompact(tableName);
    while (store.getStorefilesCount() != 1) {
      Thread.sleep(20);
    }
    return System.currentTimeMillis() - startTime;
  } finally {
    TEST_UTIL.shutdownMiniCluster();
  }
}
项目:ditb    文件:StripeStoreConfig.java   
public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
  this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
  this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
  int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
  int minFiles = config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, -1);
  this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
  this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
      config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10));
  this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);

  float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
  if (Math.abs(splitPartCount - 1.0) < EPSILON) {
    LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
    splitPartCount = 2f;
  }
  this.splitPartCount = splitPartCount;
  // Arbitrary default split size - 4 times the size of one L0 compaction.
  // If we flush into L0 there's no split compaction, but for default value it is ok.
  double flushSize = sci.getMemstoreFlushSize();
  if (flushSize == 0) {
    flushSize = 128 * 1024 * 1024;
  }
  long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
  this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
  int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
  if (initialCount == 0) {
    LOG.error("Initial stripe count is 0, using the default");
    initialCount = 1;
  }
  this.initialCount = initialCount;
  this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
}
项目:ditb    文件:DefaultStoreFileManager.java   
public DefaultStoreFileManager(KVComparator kvComparator, Configuration conf,
    CompactionConfiguration comConf) {
  this.kvComparator = kvComparator;
  this.comConf = comConf;
  this.blockingFileCount =
      conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
}
项目:pbase    文件:StripeStoreConfig.java   
public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
  this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
  this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
  int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
  int minFiles = config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, -1);
  this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
  this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
      config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10));
  this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);

  float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
  if (Math.abs(splitPartCount - 1.0) < EPSILON) {
    LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
    splitPartCount = 2f;
  }
  this.splitPartCount = splitPartCount;
  // Arbitrary default split size - 4 times the size of one L0 compaction.
  // If we flush into L0 there's no split compaction, but for default value it is ok.
  double flushSize = sci.getMemstoreFlushSize();
  if (flushSize == 0) {
    flushSize = 128 * 1024 * 1024;
  }
  long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
  this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
  int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
  if (initialCount == 0) {
    LOG.error("Initial stripe count is 0, using the default");
    initialCount = 1;
  }
  this.initialCount = initialCount;
  this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
}
项目:HIndex    文件:StripeStoreConfig.java   
public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
  this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
  this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
  int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
  int minFiles = config.getInt(CompactionConfiguration.MIN_KEY, -1);
  this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
  this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
      config.getInt(CompactionConfiguration.MAX_KEY, 10));
  this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);

  float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
  if (Math.abs(splitPartCount - 1.0) < EPSILON) {
    LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
    splitPartCount = 2f;
  }
  this.splitPartCount = splitPartCount;
  // Arbitrary default split size - 4 times the size of one L0 compaction.
  // If we flush into L0 there's no split compaction, but for default value it is ok.
  double flushSize = sci.getMemstoreFlushSize();
  if (flushSize == 0) {
    flushSize = 128 * 1024 * 1024;
  }
  long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
  this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
  int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
  if (initialCount == 0) {
    LOG.error("Initial stripe count is 0, using the default");
    initialCount = 1;
  }
  this.initialCount = initialCount;
  this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
}
项目:hbase    文件:StripeStoreConfig.java   
public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
  this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
  this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
  int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
  int minFiles = config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, -1);
  this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
  this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
      config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10));
  this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);

  float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
  if (Math.abs(splitPartCount - 1.0) < EPSILON) {
    LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
    splitPartCount = 2f;
  }
  this.splitPartCount = splitPartCount;
  // Arbitrary default split size - 4 times the size of one L0 compaction.
  // If we flush into L0 there's no split compaction, but for default value it is ok.
  double flushSize = sci.getMemStoreFlushSize();
  if (flushSize == 0) {
    flushSize = 128 * 1024 * 1024;
  }
  long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
  this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
  int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
  if (initialCount == 0) {
    LOG.error("Initial stripe count is 0, using the default");
    initialCount = 1;
  }
  this.initialCount = initialCount;
  this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
}
项目:hbase    文件:DefaultStoreFileManager.java   
public DefaultStoreFileManager(CellComparator cellComparator,
    Comparator<HStoreFile> storeFileComparator, Configuration conf,
    CompactionConfiguration comConf) {
  this.cellComparator = cellComparator;
  this.storeFileComparator = storeFileComparator;
  this.comConf = comConf;
  this.blockingFileCount =
      conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
}
项目:hbase    文件:TestCompactionPolicy.java   
/**
 * setup config values necessary for store
 */
protected void config() {
  this.conf = TEST_UTIL.getConfiguration();
  this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, minFiles);
  this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, maxFiles);
  this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, minSize);
  this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, maxSize);
  this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F);
}
项目:hbase    文件:TestCompactionWithThroughputController.java   
private long testCompactionWithThroughputLimit() throws Exception {
  long throughputLimit = 1024L * 1024;
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DefaultStoreEngine.class.getName());
  conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100);
  conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 200);
  conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
  conf.setLong(
    PressureAwareCompactionThroughputController
      .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND,
    throughputLimit);
  conf.setLong(
    PressureAwareCompactionThroughputController
      .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND,
    throughputLimit);
  conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY,
    PressureAwareCompactionThroughputController.class.getName());
  TEST_UTIL.startMiniCluster(1);
  try {
    HStore store = prepareData();
    assertEquals(10, store.getStorefilesCount());
    long startTime = System.currentTimeMillis();
    TEST_UTIL.getAdmin().majorCompact(tableName);
    while (store.getStorefilesCount() != 1) {
      Thread.sleep(20);
    }
    long duration = System.currentTimeMillis() - startTime;
    double throughput = (double) store.getStorefilesSize() / duration * 1000;
    // confirm that the speed limit work properly(not too fast, and also not too slow)
    // 20% is the max acceptable error rate.
    assertTrue(throughput < throughputLimit * 1.2);
    assertTrue(throughput > throughputLimit * 0.8);
    return System.currentTimeMillis() - startTime;
  } finally {
    TEST_UTIL.shutdownMiniCluster();
  }
}
项目:PyroDB    文件:StripeStoreConfig.java   
public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
  this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
  this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
  int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
  int minFiles = config.getInt(CompactionConfiguration.MIN_KEY, -1);
  this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
  this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
      config.getInt(CompactionConfiguration.MAX_KEY, 10));
  this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);

  float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
  if (Math.abs(splitPartCount - 1.0) < EPSILON) {
    LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
    splitPartCount = 2f;
  }
  this.splitPartCount = splitPartCount;
  // Arbitrary default split size - 4 times the size of one L0 compaction.
  // If we flush into L0 there's no split compaction, but for default value it is ok.
  double flushSize = sci.getMemstoreFlushSize();
  if (flushSize == 0) {
    flushSize = 128 * 1024 * 1024;
  }
  long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
  this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
  int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
  if (initialCount == 0) {
    LOG.error("Initial stripe count is 0, using the default");
    initialCount = 1;
  }
  this.initialCount = initialCount;
  this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
}
项目:hbase    文件:TestCompactionLifeCycleTracker.java   
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  UTIL.getConfiguration().setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
  UTIL.startMiniCluster(3);
}