public void init(String additionalConf) throws IOException { configuration = HBaseConfiguration.create(); HRegionServer.loadWinterConf(configuration, additionalConf); conn = ConnectionFactory.createConnection(configuration); IndexTableRelation relation; if (IndexType.isUserDefinedIndex(indexType)) relation = getUserDefinedIndexTableRelation(tableName, indexType); else relation = getRegularIndexTableRelation(tableName, indexType); admin = new IndexTableAdmin(configuration, conn, relation); if (indexType == IndexType.LCIndex) admin.setLCIndexRange(getLCIndexRangeStr()); // admin.createTable(false, false); byte[][] splits = new byte[10][]; for (int i = 0; i < 10; i++) { splits[i] = Bytes.toBytes(i * 1000); } admin.createTable(true, true, splits); }
public AbstractWorkload(String descFilePath) throws IOException { this.descFilePath = descFilePath; // load parameters from descFile paramMap = new HashMap<>(); loadParamsFromFile(); // init params nbRegion = loadIntParam("region.number", 10); nbLCStatRange = loadIntParam("lcindex.range.number", 1000); scanFilterDir = loadStringParam("scan.filter.dir", "/tmp/should-assign"); scanRunTimes = loadIntParam("scan.run.times", 1); scanCacheSize = loadIntParam("scan.cache.size", 100); mdBucketThreshold = loadIntParam("md.bucket.threshold", 1000); tableName = TableName.valueOf(loadStringParam("table.name", "tbl_x")); nbTotalColumns = loadIntParam("total.columns.number", 20); nbExistingRows = loadIntParam("number.existing.rows", 100); skipFilterPrefix = loadStringParam("skip.filter.prefix", "skip"); // init hbase conf conf = HBaseConfiguration.create(); if (paramMap.containsKey("hbase.conf.path")) conf.addResource(paramMap.get("hbase.conf.path")); if (paramMap.containsKey("hbase.conf.path.additional")) HRegionServer.loadWinterConf(conf, paramMap.get("hbase.conf.path.additional")); }
@SuppressWarnings("unchecked") public JVMClusterUtil.RegionServerThread addRegionServer( Configuration config, final int index) throws IOException { // Create each regionserver with its own Configuration instance so each has // its HConnection instance rather than share (see HBASE_INSTANCES down in // the guts of HConnectionManager. // Also, create separate CoordinatedStateManager instance per Server. // This is special case when we have to have more than 1 CoordinatedStateManager // within 1 process. CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); JVMClusterUtil.RegionServerThread rst = JVMClusterUtil.createRegionServerThread(config, cp, (Class<? extends HRegionServer>) conf .getClass(HConstants.REGION_SERVER_IMPL, this.regionServerClass), index); this.regionThreads.add(rst); return rst; }
/** * @param sn * @return Admin interface for the remote regionserver named <code>sn</code> * @throws IOException * @throws RetriesExhaustedException wrapping a ConnectException if failed */ private AdminService.BlockingInterface getRsAdmin(final ServerName sn) throws IOException { AdminService.BlockingInterface admin = this.rsAdmins.get(sn); if (admin == null) { LOG.debug("New admin connection to " + sn.toString()); if (sn.equals(master.getServerName()) && master instanceof HRegionServer) { // A master is also a region server now, see HBASE-10569 for details admin = ((HRegionServer)master).getRSRpcServices(); } else { admin = this.connection.getAdmin(sn); } this.rsAdmins.put(sn, admin); } return admin; }
/** * Creates a {@link RegionServerThread}. * Call 'start' on the returned thread to make it run. * @param c Configuration to use. * @param cp consensus provider to use * @param hrsc Class to create. * @param index Used distinguishing the object returned. * @throws IOException * @return Region server added. */ public static JVMClusterUtil.RegionServerThread createRegionServerThread( final Configuration c, CoordinatedStateManager cp, final Class<? extends HRegionServer> hrsc, final int index) throws IOException { HRegionServer server; try { Constructor<? extends HRegionServer> ctor = hrsc.getConstructor(Configuration.class, CoordinatedStateManager.class); ctor.setAccessible(true); server = ctor.newInstance(c, cp); } catch (InvocationTargetException ite) { Throwable target = ite.getTargetException(); throw new RuntimeException("Failed construction of RegionServer: " + hrsc.toString() + ((target.getCause() != null)? target.getCause().getMessage(): ""), target); } catch (Exception e) { IOException ioe = new IOException(); ioe.initCause(e); throw ioe; } return new JVMClusterUtil.RegionServerThread(server, index); }
@Test(timeout = 180000, expected = ServiceException.class) public void testReplicateWALEntryWhenReplicationIsDisabled() throws Exception { LOG.info("testSimplePutDelete"); baseConfiguration.setBoolean(HConstants.REPLICATION_ENABLE_KEY, false); Table[] htables = null; try { startMiniClusters(1); createTableOnClusters(table); htables = getHTablesOnClusters(tableName); HRegionServer rs = utilities[0].getRSForFirstRegionInTable(tableName); RSRpcServices rsrpc = new RSRpcServices(rs); rsrpc.replicateWALEntry(null, null); } finally { close(htables); shutDownMiniClusters(); } }
/** * Test that scanner can continue even if the region server it was reading * from failed. Before 2772, it reused the same scanner id. * @throws Exception */ @Test(timeout=300000) public void test2772() throws Exception { LOG.info("START************ test2772"); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME); Scan scan = new Scan(); // Set a very high timeout, we want to test what happens when a RS // fails but the region is recovered before the lease times out. // Since the RS is already created, this conf is client-side only for // this new table Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SCANNER_TIMEOUT * 100); Table higherScanTimeoutTable = new HTable(conf, TABLE_NAME); ResultScanner r = higherScanTimeoutTable.getScanner(scan); // This takes way less than SCANNER_TIMEOUT*100 rs.abort("die!"); Result[] results = r.next(NB_ROWS); assertEquals(NB_ROWS, results.length); r.close(); higherScanTimeoutTable.close(); LOG.info("END ************ test2772"); }
@Test (timeout=300000) public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion1"); createTableWithDefaultConf(TABLENAME); HRegionInfo info = null; HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion1")) { info = regionInfo; try { admin.closeRegionWithEncodedRegionName("sample", rs.getServerName() .getServerName()); } catch (NotServingRegionException nsre) { // expected, ignore it } } } } onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); assertTrue("The region should be present in online regions list.", onlineRegions.contains(info)); }
@Test (timeout=300000) public void testCloseRegionWhenServerNameIsNull() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion3"); createTableWithDefaultConf(TABLENAME); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); try { List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString() .contains("TestHBACloseRegion3")) { admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), null); } } } fail("The test should throw exception if the servername passed is null."); } catch (IllegalArgumentException e) { } }
@Test (timeout=300000) public void testCloseRegionWhenServerNameIsEmpty() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); createTableWithDefaultConf(TABLENAME); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); try { List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString() .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), " "); } } } fail("The test should throw exception if the servername passed is empty."); } catch (IllegalArgumentException e) { } }
@Test (timeout=300000) public void testCloseRegionWhenEncodedRegionNameIsNotGiven() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion4"); createTableWithDefaultConf(TABLENAME); HRegionInfo info = null; HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion4")) { info = regionInfo; try { admin.closeRegionWithEncodedRegionName(regionInfo .getRegionNameAsString(), rs.getServerName().getServerName()); } catch (NotServingRegionException nsre) { // expected, ignore it. } } } } onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); assertTrue("The region should be present in online regions list.", onlineRegions.contains(info)); }
@Test (timeout=300000) public void testWALRollWriting() throws Exception { setUpforLogRolling(); String className = this.getClass().getName(); StringBuilder v = new StringBuilder(className); while (v.length() < 1000) { v.append(className); } byte[] value = Bytes.toBytes(v.toString()); HRegionServer regionServer = startAndWriteData(TableName.valueOf("TestLogRolling"), value); LOG.info("after writing there are " + DefaultWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)) + " log files"); // flush all regions for (Region r : regionServer.getOnlineRegionsLocalContext()) { r.flush(true); } admin.rollWALWriter(regionServer.getServerName()); int count = DefaultWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); }
/** * Tool to get the reference to the region server object that holds the * region of the specified user table. * It first searches for the meta rows that contain the region of the * specified table, then gets the index of that RS, and finally retrieves * the RS's reference. * @param tableName user table to lookup in hbase:meta * @return region server that holds it, null if the row doesn't exist * @throws IOException * @throws InterruptedException */ public HRegionServer getRSForFirstRegionInTable(TableName tableName) throws IOException, InterruptedException { List<byte[]> metaRows = getMetaTableRows(tableName); if (metaRows == null || metaRows.isEmpty()) { return null; } LOG.debug("Found " + metaRows.size() + " rows for table " + tableName); byte [] firstrow = metaRows.get(0); LOG.debug("FirstRow=" + Bytes.toString(firstrow)); long pause = getConfiguration().getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE); int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS); while(retrier.shouldRetry()) { int index = getMiniHBaseCluster().getServerWith(firstrow); if (index != -1) { return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer(); } // Came back -1. Region may not be online yet. Sleep a while. retrier.sleepUntilNextRetry(); } return null; }
/** * Make sure that at least the specified number of region servers * are running. We don't count the ones that are currently stopping or are * stopped. * @param num minimum number of region servers that should be running * @return true if we started some servers * @throws IOException */ public boolean ensureSomeNonStoppedRegionServersAvailable(final int num) throws IOException { boolean startedServer = ensureSomeRegionServersAvailable(num); int nonStoppedServers = 0; for (JVMClusterUtil.RegionServerThread rst : getMiniHBaseCluster().getRegionServerThreads()) { HRegionServer hrs = rst.getRegionServer(); if (hrs.isStopping() || hrs.isStopped()) { LOG.info("A region server is stopped or stopping:"+hrs); } else { nonStoppedServers++; } } for (int i=nonStoppedServers; i<num; ++i) { LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer()); startedServer = true; } return startedServer; }
protected void runwarmup() throws InterruptedException{ Thread thread = new Thread(new Runnable() { @Override public void run() { HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); HRegion region = TEST_UTIL.getMiniHBaseCluster().getRegions(TABLENAME).get(0); HRegionInfo info = region.getRegionInfo(); try { HTableDescriptor htd = table.getTableDescriptor(); for (int i = 0; i < 10; i++) { warmupHRegion(info, htd, rs.getWAL(info), rs.getConfiguration(), rs, null); } } catch (IOException ie) { LOG.error("Failed warming up region " + info.getRegionNameAsString(), ie); } } }); thread.start(); thread.join(); }
@BeforeClass public static void setUpBeforeClass() throws Exception { cluster = TEST_UTIL.startMiniCluster(1, ServerNum); table = TEST_UTIL.createTable(tableName, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); TEST_UTIL.waitTableAvailable(tableName, 1000); TEST_UTIL.loadTable(table, FAMILY); for (int i = 0; i < ServerNum; i++) { HRegionServer server = cluster.getRegionServer(i); for (Region region : server.getOnlineRegions(tableName)) { region.flush(true); } } finder.setConf(TEST_UTIL.getConfiguration()); finder.setServices(cluster.getMaster()); finder.setClusterStatus(cluster.getMaster().getClusterStatus()); }
@Test public void testInternalGetTopBlockLocation() throws Exception { for (int i = 0; i < ServerNum; i++) { HRegionServer server = cluster.getRegionServer(i); for (Region region : server.getOnlineRegions(tableName)) { // get region's hdfs block distribution by region and RegionLocationFinder, // they should have same result HDFSBlocksDistribution blocksDistribution1 = region.getHDFSBlocksDistribution(); HDFSBlocksDistribution blocksDistribution2 = finder.getBlockDistribution(region .getRegionInfo()); assertEquals(blocksDistribution1.getUniqueBlocksTotalWeight(), blocksDistribution2.getUniqueBlocksTotalWeight()); if (blocksDistribution1.getUniqueBlocksTotalWeight() != 0) { assertEquals(blocksDistribution1.getTopHosts().get(0), blocksDistribution2.getTopHosts() .get(0)); } } } }
@Test public void testGetTopBlockLocations() throws Exception { for (int i = 0; i < ServerNum; i++) { HRegionServer server = cluster.getRegionServer(i); for (Region region : server.getOnlineRegions(tableName)) { List<ServerName> servers = finder.getTopBlockLocations(region.getRegionInfo()); // test table may have empty region if (region.getHDFSBlocksDistribution().getUniqueBlocksTotalWeight() == 0) { continue; } List<String> topHosts = region.getHDFSBlocksDistribution().getTopHosts(); // rs and datanode may have different host in local machine test if (!topHosts.contains(server.getServerName().getHostname())) { continue; } for (int j = 0; j < ServerNum; j++) { ServerName serverName = cluster.getRegionServer(j).getServerName(); assertTrue(servers.contains(serverName)); } } } }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(null, regionServer.getRSRpcServices(), regionServer.getServerName(), REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
@Override public void preMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB, List<Mutation> metaEntries) throws IOException { preMergeBeforePONRCalled = true; RegionServerCoprocessorEnvironment environment = ctx.getEnvironment(); HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); List<Region> onlineRegions = rs.getOnlineRegions(TableName.valueOf("testRegionServerObserver_2")); rmt = (RegionMergeTransactionImpl) new RegionMergeTransactionFactory(rs.getConfiguration()) .create(onlineRegions.get(0), onlineRegions.get(1), true); if (!rmt.prepare(rs)) { LOG.error("Prepare for the region merge of table " + onlineRegions.get(0).getTableDesc().getNameAsString() + " failed. So returning null. "); ctx.bypass(); return; } mergedRegion = rmt.stepsBeforePONR(rs, rs, false); rmt.prepareMutationsForMerge(mergedRegion.getRegionInfo(), regionA.getRegionInfo(), regionB.getRegionInfo(), rs.getServerName(), metaEntries, regionA.getTableDesc().getRegionReplication()); MetaTableAccessor.mutateMetaTable(rs.getConnection(), metaEntries); }
@Test(timeout=60000) public void testExceptionDuringInitialization() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast. conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); TEST_UTIL.startMiniCluster(2); try { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // Trigger one regionserver to fail as if it came up with a coprocessor // that fails during initialization final HRegionServer regionServer = cluster.getRegionServer(0); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, FailedInitializationObserver.class.getName()); regionServer.getRegionServerCoprocessorHost().loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); TEST_UTIL.waitFor(10000, 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return regionServer.isAborted(); } }); } finally { TEST_UTIL.shutdownMiniCluster(); } }
/** * Tests that the LogRoller perform the roll even if there are no edits */ @Test public void testNoEdits() throws Exception { TableName tableName = TableName.valueOf("TestLogRollPeriodNoEdits"); TEST_UTIL.createTable(tableName, "cf"); try { Table table = new HTable(TEST_UTIL.getConfiguration(), tableName); try { HRegionServer server = TEST_UTIL.getRSForFirstRegionInTable(tableName); WAL log = server.getWAL(null); checkMinLogRolls(log, 5); } finally { table.close(); } } finally { TEST_UTIL.deleteTable(tableName); } }
private void compactAndWait() throws IOException, InterruptedException { LOG.debug("Compacting table " + tableName); HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.majorCompact(tableName); // Waiting for the compaction to start, at least .5s. final long maxWaitime = System.currentTimeMillis() + 500; boolean cont; do { cont = rs.compactSplitThread.getCompactionQueueSize() == 0; Threads.sleep(1); } while (cont && System.currentTimeMillis() < maxWaitime); while (rs.compactSplitThread.getCompactionQueueSize() > 0) { Threads.sleep(1); } LOG.debug("Compaction queue size reached 0, continuing"); }
/** * Create an unmanaged {@link HConnection} based on the environment in which we are running the * coprocessor. The {@link HConnection} must be externally cleaned up (we bypass the usual HTable * cleanup mechanisms since we own everything). * @param env environment hosting the {@link HConnection} * @return an unmanaged {@link HConnection}. * @throws IOException if we cannot create the basic connection */ @SuppressWarnings("resource") public static HConnection getConnectionForEnvironment(CoprocessorEnvironment env) throws IOException { Configuration conf = env.getConfiguration(); HConnection connection = null; // this bit is a little hacky - we need to reach kind far into the internals. However, since we // are in a coprocessor (which is part of the internals), this is more ok. if (env instanceof RegionCoprocessorEnvironment) { RegionCoprocessorEnvironment e = (RegionCoprocessorEnvironment) env; RegionServerServices services = e.getRegionServerServices(); if (services instanceof HRegionServer) { connection = new CoprocessorHConnection(conf, (HRegionServer) services); } } // didn't create the custom HConnection, so just create the usual connection. Saves us some conf // lookups, but no network accesses or anything else with excessive overhead. if (connection == null) { connection = HConnectionManager.createConnection(conf); } return connection; }
/** * Test we reopen a region once closed. * @throws Exception */ @Test (timeout=300000) public void testReOpenRegion() throws Exception { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size()); int rsIdx = 0; HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx); HRegionInfo hri = getNonMetaRegion(ProtobufUtil.getOnlineRegions(regionServer)); LOG.debug("Asking RS to close region " + hri.getRegionNameAsString()); LOG.info("Unassign " + hri.getRegionNameAsString()); cluster.getMaster().assignmentManager.unassign(hri); while (!cluster.getMaster().assignmentManager.wasClosedHandlerCalled(hri)) { Threads.sleep(100); } while (!cluster.getMaster().assignmentManager.wasOpenedHandlerCalled(hri)) { Threads.sleep(100); } LOG.info("Done with testReOpenRegion"); }
/** * Get the location of the specified region * @param regionName Name of the region in bytes * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()} * of HRS carrying .META.. Returns -1 if none found. */ public int getServerWith(byte[] regionName) { int index = -1; int count = 0; for (JVMClusterUtil.RegionServerThread rst: getRegionServerThreads()) { HRegionServer hrs = rst.getRegionServer(); HRegion metaRegion = hrs.getOnlineRegion(regionName); if (metaRegion != null) { index = count; break; } count++; } return index; }
@Test public void testCloseRegionWhenServerNameIsEmpty() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); createTableWithDefaultConf(TABLENAME); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); try { List<HRegionInfo> onlineRegions = rs.getOnlineRegions(); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString() .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), " "); } } } fail("The test should throw exception if the servername passed is empty."); } catch (IllegalArgumentException e) { } }
@Test public void testCloseRegionWhenEncodedRegionNameIsNotGiven() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion4"); createTableWithDefaultConf(TABLENAME); HRegionInfo info = null; HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); List<HRegionInfo> onlineRegions = rs.getOnlineRegions(); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion4")) { info = regionInfo; admin.closeRegionWithEncodedRegionName(regionInfo .getRegionNameAsString(), rs.getServerName().getServerName()); } } } onlineRegions = rs.getOnlineRegions(); assertTrue("The region should be present in online regions list.", onlineRegions.contains(info)); }
@Test (timeout=300000) public void testHLogRollWriting() throws Exception { setUpforLogRolling(); String className = this.getClass().getName(); StringBuilder v = new StringBuilder(className); while (v.length() < 1000) { v.append(className); } byte[] value = Bytes.toBytes(v.toString()); HRegionServer regionServer = startAndWriteData("TestLogRolling", value); LOG.info("after writing there are " + HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()) + " log files"); // flush all regions List<HRegion> regions = new ArrayList<HRegion>(regionServer .getOnlineRegionsLocalContext()); for (HRegion r : regions) { r.flushcache(); } admin.rollHLogWriter(regionServer.getServerName().getServerName()); int count = HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); }
void populateDataInTable(int nrows, String fname) throws Exception { byte [] family = Bytes.toBytes(fname); List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads(); assertEquals(NUM_RS, rsts.size()); for (RegionServerThread rst : rsts) { HRegionServer hrs = rst.getRegionServer(); List<HRegionInfo> hris = hrs.getOnlineRegions(); for (HRegionInfo hri : hris) { if (hri.isMetaTable()) { continue; } LOG.debug("adding data to rs = " + rst.getName() + " region = "+ hri.getRegionNameAsString()); HRegion region = hrs.getOnlineRegion(hri.getRegionName()); assertTrue(region != null); putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family); } } }
@Test (timeout=300000) public void testWALRollWriting() throws Exception { setUpforLogRolling(); String className = this.getClass().getName(); StringBuilder v = new StringBuilder(className); while (v.length() < 1000) { v.append(className); } byte[] value = Bytes.toBytes(v.toString()); HRegionServer regionServer = startAndWriteData(TableName.valueOf("TestLogRolling"), value); LOG.info("after writing there are " + DefaultWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)) + " log files"); // flush all regions List<HRegion> regions = new ArrayList<HRegion>(regionServer .getOnlineRegionsLocalContext()); for (HRegion r : regions) { r.flushcache(); } admin.rollWALWriter(regionServer.getServerName()); int count = DefaultWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); }
/** * If region open fails with IOException in openRegion() while doing tableDescriptors.get() * the region should not add into regionsInTransitionInRS map * @throws Exception */ @Test public void testRegionOpenFailsDueToIOException() throws Exception { HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0); TableDescriptors htd = Mockito.mock(TableDescriptors.class); Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors"); Whitebox.setInternalState(regionServer, "tableDescriptors", htd); Mockito.doThrow(new IOException()).when(htd).get((TableName) Mockito.any()); try { ProtobufUtil.openRegion(regionServer, regionServer.getServerName(), REGIONINFO); fail("It should throw IOException "); } catch (IOException e) { } Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState); assertFalse("Region should not be in RIT", regionServer.getRegionsInTransitionInRS().containsKey(REGIONINFO.getEncodedNameAsBytes())); }
@Override public void preMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> ctx, HRegion regionA, HRegion regionB, List<Mutation> metaEntries) throws IOException { preMergeBeforePONRCalled = true; RegionServerCoprocessorEnvironment environment = ctx.getEnvironment(); HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); List<HRegion> onlineRegions = rs.getOnlineRegions(TableName.valueOf("testRegionServerObserver_2")); rmt = new RegionMergeTransaction(onlineRegions.get(0), onlineRegions.get(1), true); if (!rmt.prepare(rs)) { LOG.error("Prepare for the region merge of table " + onlineRegions.get(0).getTableDesc().getNameAsString() + " failed. So returning null. "); ctx.bypass(); return; } mergedRegion = rmt.stepsBeforePONR(rs, rs, false); rmt.prepareMutationsForMerge(mergedRegion.getRegionInfo(), regionA.getRegionInfo(), regionB.getRegionInfo(), rs.getServerName(), metaEntries); MetaTableAccessor.mutateMetaTable(rs.getConnection(), metaEntries); }
/** * Test that scanner can continue even if the region server it was reading * from failed. Before 2772, it reused the same scanner id. * @throws Exception */ @Test(timeout=300000) public void test2772() throws Exception { LOG.info("START************ test2772"); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME); Scan scan = new Scan(); // Set a very high timeout, we want to test what happens when a RS // fails but the region is recovered before the lease times out. // Since the RS is already created, this conf is client-side only for // this new table Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SCANNER_TIMEOUT * 100); HTable higherScanTimeoutTable = new HTable(conf, TABLE_NAME); ResultScanner r = higherScanTimeoutTable.getScanner(scan); // This takes way less than SCANNER_TIMEOUT*100 rs.abort("die!"); Result[] results = r.next(NB_ROWS); assertEquals(NB_ROWS, results.length); r.close(); higherScanTimeoutTable.close(); LOG.info("END ************ test2772"); }
@Test(timeout = 300000) public void testLogReplayWithNonMetaRSDown() throws Exception { LOG.info("testLogReplayWithNonMetaRSDown"); conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024); // create more than one wal conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); startCluster(NUM_RS); final int NUM_REGIONS_TO_CREATE = 40; final int NUM_LOG_LINES = 1000; // turn off load balancing to prevent regions from moving around otherwise // they will consume recovered.edits master.balanceSwitch(false); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HRegionServer hrs = findRSToKill(false, "table"); List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); // wait for abort completes this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); ht.close(); zkw.close(); }
@Override public void postStartRegionOperation(ObserverContext<RegionCoprocessorEnvironment> e, Operation op) throws IOException { if (op.equals(Operation.BATCH_MUTATE)) { HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices(); HRegion userRegion = e.getEnvironment().getRegion(); HTableDescriptor userTableDesc = userRegion.getTableDesc(); String tableName = userTableDesc.getNameAsString(); if (isNotIndexedTableDescriptor(userTableDesc)) { return; } if (!isValidIndexMutation(userTableDesc)) { return; } acquireLockOnIndexRegion(tableName, userRegion, rs, op); } }
@Test (timeout=300000) public void testCloseRegionWhenServerNameIsEmpty() throws Exception { byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); createTableWithDefaultConf(TABLENAME); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); try { List<HRegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs); for (HRegionInfo regionInfo : onlineRegions) { if (!regionInfo.isMetaTable()) { if (regionInfo.getRegionNameAsString() .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), " "); } } } fail("The test should throw exception if the servername passed is empty."); } catch (IllegalArgumentException e) { } }
private void compactAndWait() throws IOException, InterruptedException { LOG.debug("Compacting table " + tableName); admin.majorCompact(tableName); HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); // Waiting for the compaction to start, at least .5s. final long maxWaitime = System.currentTimeMillis() + 500; boolean cont; do { cont = rs.compactSplitThread.getCompactionQueueSize() == 0; Threads.sleep(1); } while (cont && System.currentTimeMillis() < maxWaitime); while (rs.compactSplitThread.getCompactionQueueSize() > 0) { Threads.sleep(5); } LOG.debug("Compaction queue size reached 0, continuing"); }
public TTLStoreScanner(Store store, long smallestReadPoint, long earliestTS, ScanType type, List<? extends KeyValueScanner> scanners, TTLExpiryChecker ttlExpiryChecker, String actualTableName, HRegionServer rs) throws IOException { this.store = store; this.smallestReadPoint = smallestReadPoint; this.earliestTS = earliestTS; this.type = type; Scan scan = new Scan(); scan.setMaxVersions(store.getFamily().getMaxVersions()); delegate = new StoreScanner(store, store.getScanInfo(), scan, scanners, type, this.smallestReadPoint, this.earliestTS); this.ttlExpiryChecker = ttlExpiryChecker; this.actualTableName = actualTableName; this.rs = rs; }