private int start() throws Exception { Configuration conf = getConf(); HRegionServer.loadWinterConf(conf, null); CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); try { // If 'local', don't start a region server here. Defer to // LocalHBaseCluster. It manages 'local' clusters. if (LocalHBaseCluster.isLocal(conf)) { LOG.warn("Not starting a distinct region server because " + HConstants.CLUSTER_DISTRIBUTED + " is false"); } else { logProcessInfo(getConf()); HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf, cp); hrs.start(); hrs.join(); if (hrs.isAborted()) { throw new RuntimeException("HRegionServer Aborted"); } } } catch (Throwable t) { LOG.error("Region server exiting", t); return 1; } return 0; }
@BeforeClass public static void setUp() throws Exception { Properties conf = MiniKdc.createConf(); conf.put(MiniKdc.DEBUG, true); KDC = new MiniKdc(conf, new File(TEST_UTIL.getDataTestDir("kdc").toUri().getPath())); KDC.start(); USERNAME = UserGroupInformation.getLoginUser().getShortUserName(); PRINCIPAL = USERNAME + "/" + HOST; HTTP_PRINCIPAL = "HTTP/" + HOST; KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL); TEST_UTIL.startMiniZKCluster(); HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath()); HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm()); HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration()); setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration()); UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration()); TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); TEST_UTIL.startMiniDFSCluster(1); Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken"); FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir); CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1); CLUSTER.startup(); }
private int start() throws Exception { Configuration conf = getConf(); CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); try { // If 'local', don't start a region server here. Defer to // LocalHBaseCluster. It manages 'local' clusters. if (LocalHBaseCluster.isLocal(conf)) { LOG.warn("Not starting a distinct region server because " + HConstants.CLUSTER_DISTRIBUTED + " is false"); } else { logProcessInfo(getConf()); HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf, cp); hrs.start(); hrs.join(); if (hrs.isAborted()) { throw new RuntimeException("HRegionServer Aborted"); } } } catch (Throwable t) { LOG.error("Region server exiting", t); return 1; } return 0; }
private int start() throws Exception { Configuration conf = getConf(); try { // If 'local', don't start a region server here. Defer to // LocalHBaseCluster. It manages 'local' clusters. if (LocalHBaseCluster.isLocal(conf)) { LOG.warn("Not starting a distinct region server because " + HConstants.CLUSTER_DISTRIBUTED + " is false"); } else { logProcessInfo(getConf()); HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf); Thread rsThread = HRegionServer.startRegionServer(hrs); rsThread.join(); if (hrs.isAborted()) { throw new RuntimeException("HRegionServer Aborted"); } } } catch (Throwable t) { LOG.error("Region server exiting", t); return 1; } return 0; }
private int start() throws Exception { Configuration conf = getConf(); TraceUtil.initTracer(conf); try { // If 'local', don't start a region server here. Defer to // LocalHBaseCluster. It manages 'local' clusters. if (LocalHBaseCluster.isLocal(conf)) { LOG.warn("Not starting a distinct region server because " + HConstants.CLUSTER_DISTRIBUTED + " is false"); } else { logProcessInfo(getConf()); HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf); hrs.start(); hrs.join(); if (hrs.isAborted()) { throw new RuntimeException("HRegionServer Aborted"); } } } catch (Throwable t) { LOG.error("Region server exiting", t); return 1; } return 0; }
/** * Setup and start kerberos, hbase */ @BeforeClass public static void setUp() throws Exception { KDC = TEST_UTIL.setupMiniKdc(KEYTAB_FILE); USERNAME = UserGroupInformation.getLoginUser().getShortUserName(); PRINCIPAL = USERNAME + "/" + HOST; HTTP_PRINCIPAL = "HTTP/" + HOST; KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL); TEST_UTIL.startMiniZKCluster(); HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm()); HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration()); setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration()); UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration()); TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); TEST_UTIL.startMiniDFSCluster(1); Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken"); FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir); CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1); CLUSTER.startup(); }
private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException{ List<JVMClusterUtil.MasterThread> masters = cluster.getMasters(); List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers(); if (masters != null) { for (JVMClusterUtil.MasterThread t : masters) { t.join(); if(t.getMaster().isAborted()) { closeAllRegionServerThreads(regionservers); throw new RuntimeException("HMaster Aborted"); } } } }
@Before public void setUp() throws Exception { testUtil = new HBaseTestingUtility(); testUtil.startMiniDFSCluster(1); testUtil.startMiniZKCluster(1); testUtil.createRootDir(); cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0); }
public static synchronized Configuration getInstanceConfig() throws Exception { if (conf == null) { File zooRoot = File.createTempFile("hbase-zookeeper", ""); zooRoot.delete(); ZooKeeperServer zookeper = new ZooKeeperServer(zooRoot, zooRoot, 2000); ServerCnxnFactory factory = ServerCnxnFactory.createFactory(new InetSocketAddress("localhost", 0), 5000); factory.startup(zookeper); YarnConfiguration yconf = new YarnConfiguration(); String argLine = System.getProperty("argLine"); if (argLine != null) { yconf.set("yarn.app.mapreduce.am.command-opts", argLine.replace("jacoco.exec", "jacocoMR.exec")); } yconf.setBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); yconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MiniMRYarnCluster miniCluster = new MiniMRYarnCluster("testCluster"); miniCluster.init(yconf); yconf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); miniCluster.start(); File hbaseRoot = File.createTempFile("hbase-root", ""); hbaseRoot.delete(); conf = HBaseConfiguration.create(miniCluster.getConfig()); conf.set(HConstants.HBASE_DIR, hbaseRoot.toURI().toURL().toString()); conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, factory.getLocalPort()); conf.set("hbase.master.hostname", "localhost"); conf.set("hbase.regionserver.hostname", "localhost"); conf.setInt("hbase.master.info.port", -1); conf.set("hbase.fs.tmp.dir", new File(System.getProperty("java.io.tmpdir")).toURI().toURL().toString()); LocalHBaseCluster cluster = new LocalHBaseCluster(conf); cluster.startup(); } return conf; }
private int start() throws Exception { Configuration conf = getConf(); // If 'local', don't start a region server here. Defer to // LocalHBaseCluster. It manages 'local' clusters. if (LocalHBaseCluster.isLocal(conf)) { LOG.warn("Not starting a distinct region server because " + HConstants.CLUSTER_DISTRIBUTED + " is false"); } else { logJVMInfo(); HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf); HRegionServer.startRegionServer(hrs); } return 0; }
public static void startLocalCluster() throws IOException, InterruptedException { startZooKeeperServer(); //Configuration conf = HBaseConfiguration.create(); Configuration conf = getConfiguration(); LocalHBaseCluster lc = new LocalHBaseCluster(conf); lc.startup(); }
public static void startCluster() throws Exception { Configuration hbaseConf = HBaseBasedAuditRepository.getHBaseConfiguration(ApplicationProperties.get()); hbaseTestUtility = HBaseTestingUtility.createLocalHTU(hbaseConf); int zkPort = hbaseConf.getInt("hbase.zookeeper.property.clientPort", 19026); hbaseTestUtility.startMiniZKCluster(1, zkPort); hbaseCluster = new LocalHBaseCluster(hbaseTestUtility.getConfiguration()); hbaseCluster.startup(); }
private void waitOnMasterThreads(LocalHBaseCluster cluster) throws InterruptedException { List<JVMClusterUtil.MasterThread> masters = cluster.getMasters(); List<JVMClusterUtil.RegionServerThread> regionservers = cluster.getRegionServers(); if (masters != null) { for (JVMClusterUtil.MasterThread t : masters) { t.join(); if (t.getMaster().isAborted()) { closeAllRegionServerThreads(regionservers); throw new RuntimeException("HMaster Aborted"); } } } }
@Test(timeout = 60000) public void testMasterShutdownBeforeStartingAnyRegionServer() throws Exception { final int NUM_MASTERS = 1; final int NUM_RS = 0; // Create config to use for this cluster Configuration conf = HBaseConfiguration.create(); conf.setInt("hbase.ipc.client.failed.servers.expiry", 200); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); // Start the cluster final HBaseTestingUtility util = new HBaseTestingUtility(conf); util.startMiniDFSCluster(3); util.startMiniZKCluster(); util.createRootDir(); final LocalHBaseCluster cluster = new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class, MiniHBaseCluster.MiniHBaseClusterRegionServer.class); final int MASTER_INDEX = 0; final MasterThread master = cluster.getMasters().get(MASTER_INDEX); master.start(); LOG.info("Called master start on " + master.getName()); Thread shutdownThread = new Thread() { public void run() { LOG.info("Before call to shutdown master"); try { try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) { try (Admin admin = connection.getAdmin()) { admin.shutdown(); } } LOG.info("After call to shutdown master"); cluster.waitOnMaster(MASTER_INDEX); } catch (Exception e) { } }; }; shutdownThread.start(); LOG.info("Called master join on " + master.getName()); master.join(); shutdownThread.join(); List<MasterThread> masterThreads = cluster.getMasters(); // make sure all the masters properly shutdown assertEquals(0, masterThreads.size()); util.shutdownMiniZKCluster(); util.shutdownMiniDFSCluster(); util.cleanupTestDir(); }
/** * Test verifies whether a region server is removing from online servers list in master if it went * down after registering with master. * @throws Exception */ @Test(timeout = 180000) public void testRSTermnationAfterRegisteringToMasterBeforeCreatingEphemeralNod() throws Exception { final int NUM_MASTERS = 1; final int NUM_RS = 2; firstRS.set(true); // Create config to use for this cluster Configuration conf = HBaseConfiguration.create(); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); // Start the cluster final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); TEST_UTIL.startMiniDFSCluster(3); TEST_UTIL.startMiniZKCluster(); TEST_UTIL.createRootDir(); final LocalHBaseCluster cluster = new LocalHBaseCluster(conf, NUM_MASTERS, NUM_RS, HMaster.class, MockedRegionServer.class); final MasterThread master = cluster.getMasters().get(0); master.start(); try { long startTime = System.currentTimeMillis(); while (!master.getMaster().isActiveMaster()) { try { Thread.sleep(100); } catch (InterruptedException ignored) { } if (System.currentTimeMillis() > startTime + 30000) { throw new RuntimeException("Master not active after 30 seconds"); } } masterActive = true; cluster.getRegionServers().get(0).start(); cluster.getRegionServers().get(1).start(); Thread.sleep(10000); List<ServerName> onlineServersList = master.getMaster().getServerManager().getOnlineServersList(); while (onlineServersList.size() > 1) { Thread.sleep(100); onlineServersList = master.getMaster().getServerManager().getOnlineServersList(); } assertEquals(onlineServersList.size(), 1); cluster.shutdown(); } finally { masterActive = false; firstRS.set(true); TEST_UTIL.shutdownMiniCluster(); } }
private int startMaster() { Configuration conf = getConf(); try { // If 'local', defer to LocalHBaseCluster instance. Starts master // and regionserver both in the one JVM. if (LocalHBaseCluster.isLocal(conf)) { final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(); File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR)); int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0); if (zkClientPort == 0) { throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT); } zooKeeperCluster.setDefaultClientPort(zkClientPort); // login the zookeeper server principal (if using security) ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file", "hbase.zookeeper.server.kerberos.principal", null); int clientPort = zooKeeperCluster.startup(zkDataPath); if (clientPort != zkClientPort) { String errorMsg = "Could not start ZK at requested port of " + zkClientPort + ". ZK was started at port: " + clientPort + ". Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum."; System.err.println(errorMsg); throw new IOException(errorMsg); } conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); // Need to have the zk cluster shutdown when master is shutdown. // Run a subclass that does the zk cluster shutdown on its way out. LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1, LocalHMaster.class, HRegionServer.class); ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster); cluster.startup(); waitOnMasterThreads(cluster); } else { HMaster master = HMaster.constructMaster(masterClass, conf); if (master.isStopped()) { LOG.info("Won't bring the Master up as a shutdown is requested"); return -1; } master.start(); master.join(); if(master.isAborted()) throw new RuntimeException("HMaster Aborted"); } } catch (Throwable t) { LOG.error("Failed to start master", t); return -1; } return 0; }
private int startMaster() { Configuration conf = getConf(); try { // If 'local', defer to LocalHBaseCluster instance. Starts master // and regionserver both in the one JVM. if (LocalHBaseCluster.isLocal(conf)) { DefaultMetricsSystem.setMiniClusterMode(true); final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf); File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR)); int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0); if (zkClientPort == 0) { throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT); } zooKeeperCluster.setDefaultClientPort(zkClientPort); // login the zookeeper server principal (if using security) ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file", "hbase.zookeeper.server.kerberos.principal", null); int localZKClusterSessionTimeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10 * 1000); conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout); LOG.info("Starting a zookeeper cluster"); int clientPort = zooKeeperCluster.startup(zkDataPath); if (clientPort != zkClientPort) { String errorMsg = "Could not start ZK at requested port of " + zkClientPort + ". ZK was started at port: " + clientPort + ". Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum."; System.err.println(errorMsg); throw new IOException(errorMsg); } conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); // Need to have the zk cluster shutdown when master is shutdown. // Run a subclass that does the zk cluster shutdown on its way out. int mastersCount = conf.getInt("hbase.masters", 1); int regionServersCount = conf.getInt("hbase.regionservers", 1); LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount + ", regionserversCount=" + regionServersCount); LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount, LocalHMaster.class, HRegionServer.class); ((LocalHMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster); cluster.startup(); waitOnMasterThreads(cluster); } else { logProcessInfo(getConf());//将系统的运行配置参数以及JVM的状态存到日志中 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); HMaster master = HMaster.constructMaster(masterClass, conf, csm); if (master.isStopped()) { LOG.info("Won't bring the Master up as a shutdown is requested"); return 1; } master.start(); master.join(); if (master.isAborted()) throw new RuntimeException("HMaster Aborted"); } } catch (Throwable t) { LOG.error("Master exiting", t); return 1; } return 0; }
