/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableAvailability(tableName); } @Override public boolean evaluate() throws IOException { boolean tableAvailable = getHBaseAdmin().isTableAvailable(tableName); if (tableAvailable) { try { Canary.sniff(getHBaseAdmin(), tableName); } catch (Exception e) { throw new IOException("Canary sniff failed for table " + tableName, e); } } return tableAvailable; } }; }
@Test(timeout=60000) public void testExceptionDuringInitialization() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast. conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); TEST_UTIL.startMiniCluster(2); try { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // Trigger one regionserver to fail as if it came up with a coprocessor // that fails during initialization final HRegionServer regionServer = cluster.getRegionServer(0); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, FailedInitializationObserver.class.getName()); regionServer.getRegionServerCoprocessorHost().loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); TEST_UTIL.waitFor(10000, 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return regionServer.isAborted(); } }); } finally { TEST_UTIL.shutdownMiniCluster(); } }
/** wal replication is async, we have to wait until the replication catches up, or we timeout */ private void verifyNumericRowsWithTimeout(final Table table, final byte[] f, final int startRow, final int endRow, final int replicaId, final long timeout) throws Exception { try { HTU.waitFor(timeout, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { try { HTU.verifyNumericRows(table, f, startRow, endRow, replicaId); return true; } catch (AssertionError ae) { return false; } } }); } catch (Throwable t) { // ignore this, but redo the verify do get the actual exception HTU.verifyNumericRows(table, f, startRow, endRow, replicaId); } }
@Test(timeout=60000) public void testExceptionDuringInitialization() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast. conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); TEST_UTIL.startMiniCluster(2); try { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); // Trigger one regionserver to fail as if it came up with a coprocessor // that fails during initialization final HRegionServer regionServer = cluster.getRegionServer(0); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, FailedInitializationObserver.class.getName()); regionServer.getCoprocessorHost().loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); TEST_UTIL.waitFor(10000, 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return regionServer.isAborted(); } }); } finally { TEST_UTIL.shutdownMiniCluster(); } }
private void waitForTableToEnterQuotaViolation(TableName tn) throws Exception { // Verify that the RegionServer has the quota in violation final HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0); Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { Map<TableName,SpaceQuotaSnapshot> snapshots = rs.getRegionServerSpaceQuotaManager().copyQuotaSnapshots(); SpaceQuotaSnapshot snapshot = snapshots.get(tn); if (snapshot == null) { LOG.info("Found no snapshot for " + tn); return false; } LOG.info("Found snapshot " + snapshot); return snapshot.getQuotaStatus().isInViolation(); } }); }
@Test public void testRegionSizesFromMaster() throws Exception { final long tableSize = 1024L * 10L; // 10KB final int numRegions = 10; final TableName tn = helper.createTableWithRegions(numRegions); // Will write at least `tableSize` data helper.writeData(tn, tableSize); final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); final MasterQuotaManager quotaManager = master.getMasterQuotaManager(); // Make sure the master has all of the reports Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { Map<RegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes(); LOG.trace("Region sizes=" + regionSizes); return numRegions == countRegionsForTable(tn, regionSizes) && tableSize <= getTableSize(tn, regionSizes); } }); Map<TableName,Long> sizes = QuotaTableUtil.getMasterReportedTableSizes(TEST_UTIL.getConnection()); Long size = sizes.get(tn); assertNotNull("No reported size for " + tn, size); assertTrue("Reported table size was " + size, size.longValue() >= tableSize); }
/** Test to unload a regionserver first and then load it using no Ack mode * we check if some regions are loaded on the region server(since no ack is best effort) */ @Test public void testLoadWithoutAck() throws Exception { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HRegionServer regionServer = cluster.getRegionServer(0); String rsName = regionServer.getServerName().getHostname(); int port = regionServer.getServerName().getPort(); int noRegions = regionServer.getNumberOfOnlineRegions(); String rs = rsName + ":" + Integer.toString(port); RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs).ack(true); RegionMover rm = rmBuilder.build(); rm.setConf(TEST_UTIL.getConfiguration()); LOG.info("Unloading " + rs); rm.unload(); assertEquals(0, regionServer.getNumberOfOnlineRegions()); LOG.info("Successfully Unloaded\nNow Loading"); rm = rmBuilder.ack(false).build(); rm.setConf(TEST_UTIL.getConfiguration()); rm.load(); TEST_UTIL.waitFor(5000, 500, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return regionServer.getNumberOfOnlineRegions() > 0; } }); }
@Test public void testUnloadWithoutAck() throws Exception { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); final HRegionServer regionServer = cluster.getRegionServer(0); final int noRegions = regionServer.getNumberOfOnlineRegions(); String rsName = regionServer.getServerName().getHostname(); int port = regionServer.getServerName().getPort(); String rs = rsName + ":" + Integer.toString(port); RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs).ack(false); RegionMover rm = rmBuilder.build(); rm.setConf(TEST_UTIL.getConfiguration()); LOG.info("Unloading " + rs); rm.unload(); TEST_UTIL.waitFor(5000, 500, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return regionServer.getNumberOfOnlineRegions() < noRegions; } }); }
@Test public void testNamespaceCreateAndAssign() throws Exception { LOG.info("testNamespaceCreateAndAssign"); String nsName = tablePrefix+"_foo"; final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign"); RSGroupInfo appInfo = addGroup("appInfo", 1); admin.createNamespace(NamespaceDescriptor.create(nsName) .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build()); final HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor("f")); admin.createTable(desc); //wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return getTableRegionMap().get(desc.getTableName()) != null; } }); ServerName targetServer = ServerName.parseServerName(appInfo.getServers().iterator().next().toString()); AdminProtos.AdminService.BlockingInterface rs = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); //verify it was assigned to the right group Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(rs).size()); }
@Test public void testDefaultNamespaceCreateAndAssign() throws Exception { LOG.info("testDefaultNamespaceCreateAndAssign"); String tableName = tablePrefix + "_testCreateAndAssign"; admin.modifyNamespace(NamespaceDescriptor.create("default") .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build()); final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); desc.addFamily(new HColumnDescriptor("f")); admin.createTable(desc); //wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return getTableRegionMap().get(desc.getTableName()) != null; } }); }
@SuppressWarnings("rawtypes") private static void updateACLs(final HBaseTestingUtility util, Callable c) throws Exception { // Get the current mtimes for all access controllers final Map<AccessController,Long> oldMTimes = getAuthManagerMTimes(util.getHBaseCluster()); // Run the update action c.call(); // Wait until mtimes for all access controllers have incremented util.waitFor(WAIT_TIME, 100, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { Map<AccessController,Long> mtimes = getAuthManagerMTimes(util.getHBaseCluster()); for (Map.Entry<AccessController,Long> e: mtimes.entrySet()) { if (!oldMTimes.containsKey(e.getKey())) { LOG.error("Snapshot of AccessController state does not include instance on region " + e.getKey().getRegion().getRegionInfo().getRegionNameAsString()); // Error out the predicate, we will try again return false; } long old = oldMTimes.get(e.getKey()); long now = e.getValue(); if (now <= old) { LOG.info("AccessController on region " + e.getKey().getRegion().getRegionInfo().getRegionNameAsString() + " has not updated: mtime=" + now); return false; } } return true; } }); }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableEnabled(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableState(tableName); } @Override public boolean evaluate() throws IOException { return getHBaseAdmin().tableExists(tableName) && getHBaseAdmin().isTableEnabled(tableName); } }; }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableDisabled(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableState(tableName); } @Override public boolean evaluate() throws IOException { return getHBaseAdmin().isTableDisabled(tableName); } }; }
@Test public void testAborting() throws Exception { final Configuration conf = HBaseConfiguration.create(); final Server server = mock(Server.class); when(server.getConfiguration()).thenReturn(conf); ExecutorService executorService = new ExecutorService("unit_test"); executorService.startExecutorService( ExecutorType.MASTER_SERVER_OPERATIONS, 1); executorService.submit(new EventHandler(server, EventType.M_SERVER_SHUTDOWN) { @Override public void process() throws IOException { throw new RuntimeException("Should cause abort"); } }); Waiter.waitFor(conf, 30000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { try { verify(server, times(1)).abort(anyString(), (Throwable) anyObject()); return true; } catch (Throwable t) { return false; } } }); executorService.shutdown(); }
@Before @Override public void setUp() throws Exception { // Initialize the cluster. This invokes LoadTestTool -init_only, which // will create the test table, appropriately pre-split super.setUp(); if (!initialized) { return; } // Update the test table schema so HFiles from this point will be written with // encryption features enabled. final Admin admin = util.getHBaseAdmin(); HTableDescriptor tableDescriptor = new HTableDescriptor(admin.getTableDescriptor(getTablename())); for (HColumnDescriptor columnDescriptor: tableDescriptor.getColumnFamilies()) { columnDescriptor.setEncryptionType("AES"); LOG.info("Updating CF schema for " + getTablename() + "." + columnDescriptor.getNameAsString()); admin.disableTable(getTablename()); admin.modifyColumn(getTablename(), columnDescriptor); admin.enableTable(getTablename()); util.waitFor(30000, 1000, true, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { return admin.isTableAvailable(getTablename()); } }); } }
@SuppressWarnings("rawtypes") private static void updateACLs(final HBaseTestingUtility util, Callable c) throws Exception { // Get the current mtimes for all access controllers final Map<AccessController,Long> oldMTimes = getAuthManagerMTimes(util.getHBaseCluster()); // Run the update action c.call(); // Wait until mtimes for all access controllers have incremented util.waitFor(WAIT_TIME, 100, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { Map<AccessController,Long> mtimes = getAuthManagerMTimes(util.getHBaseCluster()); for (Map.Entry<AccessController,Long> e: mtimes.entrySet()) { if (!oldMTimes.containsKey(e.getKey())) { LOG.error("Snapshot of AccessController state does not include instance on region " + e.getKey().getRegion().getRegionNameAsString()); // Error out the predicate, we will try again return false; } long old = oldMTimes.get(e.getKey()); long now = e.getValue(); if (now <= old) { LOG.info("AccessController on region " + e.getKey().getRegion().getRegionNameAsString() + " has not updated: mtime=" + now); return false; } } return true; } }); }
/** * Returns a {@link Predicate} for checking that there are no regions in transition in master */ public Waiter.Predicate<Exception> predicateNoRegionsInTransition() { return new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { final RegionStates regionStates = getMiniHBaseCluster().getMaster() .getAssignmentManager().getRegionStates(); return !regionStates.isRegionsInTransition(); } }; }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<Exception> predicateTableEnabled(final TableName tableName) { return new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return getHBaseAdmin().isTableEnabled(tableName); } }; }
/** * Wait until all regions for a table in hbase:meta have a non-empty * info:server, or until timeout. This means all regions have been deployed, * master has been informed and updated hbase:meta with the regions deployed * server. * @param tableName the table name * @param timeout timeout, in milliseconds * @throws IOException */ public void waitUntilAllRegionsAssigned(final TableName tableName, final long timeout) throws IOException { final HTable meta = new HTable(getConfiguration(), TableName.META_TABLE_NAME); try { waitFor(timeout, 200, true, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { boolean allRegionsAssigned = true; Scan scan = new Scan(); scan.addFamily(HConstants.CATALOG_FAMILY); ResultScanner s = meta.getScanner(scan); try { Result r; while ((r = s.next()) != null) { byte [] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); HRegionInfo info = HRegionInfo.parseFromOrNull(b); if (info != null && info.getTable().equals(tableName)) { b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); allRegionsAssigned &= (b != null); } } } finally { s.close(); } return allRegionsAssigned; } }); } finally { meta.close(); } }
@Before @Override public void setUp() throws Exception { // Initialize the cluster. This invokes LoadTestTool -init_only, which // will create the test table, appropriately pre-split super.setUp(); // Update the test table schema so HFiles from this point will be written with // encryption features enabled. final HBaseAdmin admin = util.getHBaseAdmin(); HTableDescriptor tableDescriptor = new HTableDescriptor(admin.getTableDescriptor(Bytes.toBytes(getTablename()))); for (HColumnDescriptor columnDescriptor: tableDescriptor.getColumnFamilies()) { columnDescriptor.setEncryptionType("AES"); LOG.info("Updating CF schema for " + getTablename() + "." + columnDescriptor.getNameAsString()); admin.disableTable(getTablename()); admin.modifyColumn(getTablename(), columnDescriptor); admin.enableTable(getTablename()); util.waitFor(30000, 1000, true, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { return admin.isTableAvailable(getTablename()); } }); } }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableEnabled(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableState(tableName, TableState.State.ENABLED); } @Override public boolean evaluate() throws IOException { return getAdmin().tableExists(tableName) && getAdmin().isTableEnabled(tableName); } }; }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableDisabled(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableState(tableName, TableState.State.DISABLED); } @Override public boolean evaluate() throws IOException { return getAdmin().isTableDisabled(tableName); } }; }
/** * Returns a {@link Predicate} for checking that table is enabled */ public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) { return new ExplainingPredicate<IOException>() { @Override public String explainFailure() throws IOException { return explainTableAvailability(tableName); } @Override public boolean evaluate() throws IOException { boolean tableAvailable = getAdmin().isTableAvailable(tableName); if (tableAvailable) { try (Table table = getConnection().getTable(tableName)) { TableDescriptor htd = table.getDescriptor(); for (HRegionLocation loc : getConnection().getRegionLocator(tableName) .getAllRegionLocations()) { Scan scan = new Scan().withStartRow(loc.getRegionInfo().getStartKey()) .withStopRow(loc.getRegionInfo().getEndKey()).setOneRowLimit() .setMaxResultsPerColumnFamily(1).setCacheBlocks(false); for (byte[] family : htd.getColumnFamilyNames()) { scan.addFamily(family); } try (ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } } } } return tableAvailable; } }; }
/** * Waits {@code timeout} milliseconds for the HBase quota table to exist. */ void waitForQuotaTable(Connection conn, long timeout) throws IOException { testUtil.waitFor(timeout, 1000, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { return conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME); } }); }
@BeforeClass public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); TEST_UTIL.getConfiguration().setFloat( "hbase.master.balancer.stochastic.tableSkewCost", 6000); TEST_UTIL.getConfiguration().set( HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName()); TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1); TEST_UTIL.getConfiguration().setInt( ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE - 1); TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); admin = TEST_UTIL.getAdmin(); cluster = TEST_UTIL.getHBaseCluster(); master = ((MiniHBaseCluster)cluster).getMaster(); //wait for balancer to come online TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return master.isInitialized() && ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline(); } }); admin.setBalancerRunning(false,true); rsGroupAdmin = new VerifyingRSGroupAdminClient( new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration()); rsGroupAdminEndpoint = (RSGroupAdminEndpoint) master.getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); }
@After public void afterMethod() throws Exception { deleteTableIfNecessary(); deleteNamespaceIfNecessary(); deleteGroups(); int missing = NUM_SLAVES_BASE - getNumServers(); LOG.info("Restoring servers: "+missing); for(int i=0; i<missing; i++) { ((MiniHBaseCluster)cluster).startRegionServer(); } rsGroupAdmin.addRSGroup("master"); ServerName masterServerName = ((MiniHBaseCluster)cluster).getMaster().getServerName(); try { rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master"); } catch (Exception ex) { LOG.warn("Got this on setup, FYI", ex); } TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups()); //Might be greater since moving servers back to default //is after starting a server return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE; } }); }
@Test public void testMisplacedRegions() throws Exception { final TableName tableName = TableName.valueOf(tablePrefix+"_testMisplacedRegions"); LOG.info("testMisplacedRegions"); final RSGroupInfo RSGroupInfo = addGroup("testMisplacedRegions", 1); TEST_UTIL.createMultiRegionTable(tableName, new byte[]{'f'}, 15); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); rsGroupAdminEndpoint.getGroupInfoManager() .moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName()); admin.setBalancerRunning(true,true); assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName())); admin.setBalancerRunning(false,true); TEST_UTIL.waitFor(60000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { ServerName serverName = ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1); return admin.getConnection().getAdmin() .getOnlineRegions(serverName).size() == 15; } }); }
@Before @Override public void setUp() throws Exception { // Initialize the cluster. This invokes LoadTestTool -init_only, which // will create the test table, appropriately pre-split super.setUp(); if (!initialized) { return; } // Update the test table schema so HFiles from this point will be written with // encryption features enabled. final Admin admin = util.getAdmin(); TableDescriptor tableDescriptor = admin.getDescriptor(getTablename()); for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { ColumnFamilyDescriptor updatedColumn = ColumnFamilyDescriptorBuilder .newBuilder(columnDescriptor).setEncryptionType("AES").build(); LOG.info( "Updating CF schema for " + getTablename() + "." + columnDescriptor.getNameAsString()); admin.disableTable(getTablename()); admin.modifyColumnFamily(getTablename(), updatedColumn); admin.enableTable(getTablename()); util.waitFor(30000, 1000, true, new Predicate<IOException>() { @Override public boolean evaluate() throws IOException { return admin.isTableAvailable(getTablename()); } }); } }