/** * Create a default snapshot handler - uses a zookeeper based member controller. * @param rss region server running the handler * @throws KeeperException if the zookeeper cluster cannot be reached */ @Override public void initialize(RegionServerServices rss) throws KeeperException { this.rss = rss; ZooKeeperWatcher zkw = rss.getZooKeeper(); this.memberRpcs = new ZKProcedureMemberRpcs(zkw, SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION); // read in the snapshot request configuration properties Configuration conf = rss.getConfiguration(); long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT); // create the actual snapshot procedure member ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive); this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder()); }
private static void setupConf(Configuration conf) { // enable snapshot support conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around // some files in the store conf.setInt("hbase.hstore.compaction.min", 10); conf.setInt("hbase.hstore.compactionThreshold", 10); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); conf.setInt("hbase.regionserver.msginterval", 100); conf.setBoolean("hbase.master.enabletable.roundrobin", true); // Avoid potentially aggressive splitting which would cause snapshot to fail conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 10); conf.setInt("hbase.hstore.compactionThreshold", 10); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 2); conf.setInt("hbase.hstore.compactionThreshold", 5); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
@BeforeClass public static void setUpBeforeClass() throws Exception { UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); UTIL.getConfiguration().setInt("hbase.client.pause", 250); UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); UTIL.getConfiguration().setBoolean( "hbase.master.enabletable.roundrobin", true); // Enable snapshot UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY, RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2); UTIL.startMiniCluster(3); }
/** * Create a default snapshot handler - uses a zookeeper based member controller. * @param rss region server running the handler * @throws KeeperException if the zookeeper cluster cannot be reached */ public RegionServerSnapshotManager(RegionServerServices rss) throws KeeperException { this.rss = rss; ZooKeeperWatcher zkw = rss.getZooKeeper(); this.memberRpcs = new ZKProcedureMemberRpcs(zkw, SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION); // read in the snapshot request configuration properties Configuration conf = rss.getConfiguration(); long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT); // create the actual snapshot procedure member ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive); this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 10); conf.setInt("hbase.hstore.compactionThreshold", 10); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // drop the number of attempts for the hbase admin conf.setInt("hbase.client.retries.number", 1); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); // prevent aggressive region split conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 3); conf.setInt("hbase.hstore.compactionThreshold", 5); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // drop the number of attempts for the hbase admin conf.setInt("hbase.client.retries.number", 1); // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); // prevent aggressive region split conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 3); conf.setInt("hbase.hstore.compactionThreshold", 5); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
/** * Create a default snapshot handler - uses a zookeeper based member controller. * @param rss region server running the handler * @throws KeeperException if the zookeeper cluster cannot be reached */ @Override public void initialize(RegionServerServices rss) throws KeeperException { this.rss = rss; ZKWatcher zkw = rss.getZooKeeper(); this.memberRpcs = new ZKProcedureMemberRpcs(zkw, SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION); // read in the snapshot request configuration properties Configuration conf = rss.getConfiguration(); long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT); // create the actual snapshot procedure member ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive); this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder()); }
@Test public void testExecProcedure() throws Exception { String snapshotString = "offlineTableSnapshot"; try { Table table = TEST_UTIL.createTable(tableName, Bytes.toBytes("cf")); for (int i = 0; i < 100; i++) { Put put = new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("cf"), null, Bytes.toBytes(i)); table.put(put); } // take a snapshot of the enabled table Map<String, String> props = new HashMap<>(); props.put("table", tableName.getNameAsString()); admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, snapshotString, props).get(); LOG.debug("Snapshot completed."); } finally { admin.deleteSnapshot(snapshotString).join(); TEST_UTIL.deleteTable(tableName); } }
static void setupConf(Configuration conf) { // Up the handlers; this test needs more than usual. conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 15); // enable snapshot support conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around // some files in the store conf.setInt("hbase.hstore.compaction.min", 10); conf.setInt("hbase.hstore.compactionThreshold", 10); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); conf.setInt("hbase.regionserver.msginterval", 100); conf.setBoolean("hbase.master.enabletable.roundrobin", true); // Avoid potentially aggressive splitting which would cause snapshot to fail conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); // Execute cleaner frequently to induce failures conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL); conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL); // Effectively disable TimeToLiveHFileCleaner. Don't want to fully disable it because that // will even trigger races between creating the directory containing back references and // the back reference itself. conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL); }
protected static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 10); conf.setInt("hbase.hstore.compactionThreshold", 10); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", 12); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); }
private static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files conf.setInt("hbase.hregion.memstore.flush.size", 25000); // so make sure we get a compaction when doing a load, but keep around some // files in the store conf.setInt("hbase.hstore.compaction.min", 2); conf.setInt("hbase.hstore.compactionThreshold", 5); // block writes if we get to 12 store files conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles); // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); // Enable snapshot conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); conf.setInt("hbase.hfile.compactions.cleaner.interval", 20 * 1000); }
