/** * Check if the slave is down by trying to establish a connection * @return true if down, false if up * @throws InterruptedException */ public boolean isSlaveDown() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); Thread pingThread = new Thread() { public void run() { try { HRegionInterface rrs = getRS(); // Dummy call which should fail rrs.getHServerInfo(); latch.countDown(); } catch (IOException ex) { if (ex instanceof RemoteException) { ex = ((RemoteException) ex).unwrapRemoteException(); } LOG.info("Slave cluster looks down: " + ex.getMessage()); } } }; pingThread.start(); // awaits returns true if countDown happened boolean down = ! latch.await(this.sleepForRetries, TimeUnit.MILLISECONDS); pingThread.interrupt(); return down; }
/** * Fully scan a given region, on a given server starting with given row. * @param hRegionInterface region server * @param visitor visitor * @param regionName name of region * @param startrow start row * @throws IOException * @deprecated Does not retry; use fullScan xxx instead. x */ public static void fullScan(HRegionInterface hRegionInterface, Visitor visitor, final byte[] regionName, byte[] startrow) throws IOException { if (hRegionInterface == null) return; Scan scan = new Scan(); if (startrow != null) scan.setStartRow(startrow); scan.addFamily(HConstants.CATALOG_FAMILY); long scannerid = hRegionInterface.openScanner(regionName, scan); try { Result data; while((data = hRegionInterface.next(scannerid)) != null) { if (!data.isEmpty()) visitor.visit(data); } } finally { hRegionInterface.close(scannerid); } return; }
/** * For expert-admins. Runs close on the regionserver. Closes a region based on the encoded region * name. The region server name is mandatory. If the servername is provided then based on the * online regions in the specified regionserver the specified region will be closed. The master * will not be informed of the close. Note that the regionname is the encoded regionname. * @param encodedRegionName The encoded region name; i.e. the hash that makes up the region name * suffix: e.g. if regionname is * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code> , * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>. * @param serverName The servername of the regionserver. A server name is made of host, port and * startcode. This is mandatory. Here is an example: * <code> host187.example.com,60020,1289493121758</code> * @return true if the region was closed, false if not. * @throws IOException if a remote or network exception occurs */ public boolean closeRegionWithEncodedRegionName(final String encodedRegionName, final String serverName) throws IOException { byte[] encodedRegionNameInBytes = Bytes.toBytes(encodedRegionName); if (null == serverName || ("").equals(serverName.trim())) { throw new IllegalArgumentException("The servername cannot be null or empty."); } ServerName sn = new ServerName(serverName); HRegionInterface rs = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); // Close the region without updating zk state. boolean isRegionClosed = rs.closeRegion(encodedRegionNameInBytes, false); if (false == isRegionClosed) { LOG.error("Not able to close the region " + encodedRegionName + "."); } return isRegionClosed; }
private void compact(final ServerName sn, final HRegionInfo hri, final boolean major, final byte[] family) throws IOException { HRegionInterface rs = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); if (family != null) { try { rs.compactRegion(hri, major, family); } catch (IOException ioe) { String notFoundMsg = "java.lang.NoSuchMethodException: org.apache.hadoop.hbase.ipc.HRegionInterface." + "compactRegion(org.apache.hadoop.hbase.HRegionInfo, boolean, [B)"; if (ioe.getMessage().contains(notFoundMsg)) { throw new IOException("per-column family compaction not supported on this version " + "of the HBase server. You may still compact at the table or region level by " + "omitting the column family name. Alternatively, you can upgrade the HBase server"); } throw ioe; } } else { rs.compactRegion(hri, major); } }
@Override HRegionInterface getHRegionConnection(final String hostname, final int port, final InetSocketAddress isa, final boolean master) throws IOException { // check to see where the server is running // need this isa stuff here since its what the HConnectionManager is doing too boolean isRemote = false; if (isa != null) { isRemote = checkRemote(isa.getHostName(), isa.getPort()); } else { isRemote = checkRemote(hostname, port); } // if we aren't talking to the local HRegionServer, then do the usual thing if (isRemote) { return super.getHRegionConnection(hostname, port, isa, master); } // local access, so just pass the actual server, rather than a proxy return this.server; }
/** * Contacts a region server and waits up to hbase.hbck.close.timeout ms * (default 120s) to close the region. This bypasses the active hmaster. */ public static void closeRegionSilentlyAndWait(HBaseAdmin admin, ServerName server, HRegionInfo region) throws IOException, InterruptedException { HConnection connection = admin.getConnection(); HRegionInterface rs = connection.getHRegionConnection(server.getHostname(), server.getPort()); rs.closeRegion(region, false); long timeout = admin.getConfiguration() .getLong("hbase.hbck.close.timeout", 120000); long expiration = timeout + System.currentTimeMillis(); while (System.currentTimeMillis() < expiration) { try { HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName()); if (rsRegion == null) return; } catch (IOException ioe) { return; } Thread.sleep(1000); } throw new IOException("Region " + region + " failed to close within" + " timeout " + timeout); }
private void testVerifyMetaRegionLocationWithException(Exception ex) throws IOException, InterruptedException, KeeperException { // Mock an HRegionInterface. final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); HConnection connection = mockConnection(implementation); // If a 'get' is called on mocked interface, throw connection refused. Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())). thenThrow(ex); // Now start up the catalogtracker with our doctored Connection. final CatalogTracker ct = constructAndStartCatalogTracker(connection); RootLocationEditor.setRootLocation(this.watcher, SN); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); }
/** * Test get of root region fails properly if nothing to connect to. * @throws IOException * @throws InterruptedException * @throws KeeperException */ @Test public void testVerifyRootRegionLocationFails() throws IOException, InterruptedException, KeeperException { HConnection connection = Mockito.mock(HConnection.class); ConnectException connectException = new ConnectException("Connection refused"); final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())). thenThrow(connectException); Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean())). thenReturn(implementation); final CatalogTracker ct = constructAndStartCatalogTracker(connection); RootLocationEditor.setRootLocation(this.watcher, new ServerName("example.com", 1234, System.currentTimeMillis())); Assert.assertFalse(ct.verifyRootRegionLocation(100)); }
/** * @param implementation An {@link HRegionInterface} instance; you'll likely * want to pass a mocked HRS; can be null. * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when * {@link HConnection#getConfiguration()} is called, a 'location' when * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called, * and that returns the passed {@link HRegionInterface} instance when * {@link HConnection#getHRegionConnection(String, int)} * is called (Be sure call * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)} * when done with this mocked Connection. * @throws IOException */ private HConnection mockConnection(final HRegionInterface implementation) throws IOException { HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); Mockito.doNothing().when(connection).close(); // Make it so we return any old location when asked. final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(), SN.getPort()); Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(anyLocation); Mockito.when(connection.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())). thenReturn(anyLocation); if (implementation != null) { // If a call to getHRegionConnection, return this implementation. Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())). thenReturn(implementation); } return connection; }
private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table, byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception { // connection needed for poll-wait HConnection conn = HConnectionManager.getConnection(TEST_UTIL .getConfiguration()); HRegionLocation loc = table.getRegionLocation(row, true); HRegionInterface server = conn.getHRegionConnection(loc.getHostname(), loc .getPort()); byte[] regName = loc.getRegionInfo().getRegionName(); for (int i = 0; i < nFlushes; i++) { randomCFPuts(table, row, family, nPuts); int sfCount = server.getStoreFileList(regName, FAMILY).size(); // TODO: replace this api with a synchronous flush after HBASE-2949 admin.flush(table.getTableName()); // synchronously poll wait for a new storefile to appear (flush happened) while (server.getStoreFileList(regName, FAMILY).size() == sfCount) { Thread.sleep(40); } } }
MockServer(final HBaseTestingUtility htu) throws NotAllMetaRegionsOnlineException, IOException, InterruptedException { this.c = htu.getConfiguration(); // Mock an HConnection and a HRegionInterface implementation. Have the // HConnection return the HRI. Have the HRI return a few mocked up responses // to make our test work. this.connection = HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c, Mockito.mock(HRegionInterface.class), new ServerName("example.org,12345,6789"), HRegionInfo.FIRST_META_REGIONINFO); // Set hbase.rootdir into test dir. FileSystem fs = FileSystem.get(this.c); Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR))); this.c.set(HConstants.HBASE_DIR, rootdir.toString()); this.ct = Mockito.mock(CatalogTracker.class); HRegionInterface hri = Mockito.mock(HRegionInterface.class); Mockito.when(this.ct.getConnection()).thenReturn(this.connection); Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri); }
private HConnection getMockedConnection(final Configuration conf) throws IOException { HConnection c = Mockito.mock(HConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, "example.org", 1234); Mockito.when(c.getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())). thenReturn(loc); Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())). thenReturn(loc); HRegionInterface hri = Mockito.mock(HRegionInterface.class); Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any(), Mockito.anyBoolean())).thenThrow(new IOException("injecting bulk load error")); Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())). thenReturn(hri); return c; }
/** * Get region info from local cluster. */ Map<ServerName, List<String>> getDeployedHRIs(HBaseAdmin admin) throws IOException { ClusterStatus status = admin.getMaster().getClusterStatus(); Collection<ServerName> regionServers = status.getServers(); Map<ServerName, List<String>> mm = new HashMap<ServerName, List<String>>(); HConnection connection = admin.getConnection(); for (ServerName hsi : regionServers) { HRegionInterface server = connection.getHRegionConnection(hsi.getHostname(), hsi.getPort()); // list all online regions from this region server List<HRegionInfo> regions = server.getOnlineRegions(); List<String> regionNames = new ArrayList<String>(); for (HRegionInfo hri : regions) { regionNames.add(hri.getRegionNameAsString()); } mm.put(hsi, regionNames); } return mm; }
private void compact(final ServerName sn, final HRegionInfo hri, final boolean major, final byte [] family) throws IOException { HRegionInterface rs = this.connection.getHRegionConnection(sn.getHostname(), sn.getPort()); if (family != null) { try { rs.compactRegion(hri, major, family); } catch (IOException ioe) { String notFoundMsg = "java.lang.NoSuchMethodException: org.apache.hadoop.hbase.ipc.HRegionInterface." + "compactRegion(org.apache.hadoop.hbase.HRegionInfo, boolean, [B)"; if (ioe.getMessage().contains(notFoundMsg)) { throw new IOException("per-column family compaction not supported on this version " + "of the HBase server. You may still compact at the table or region level by " + "omitting the column family name. Alternatively, you can upgrade the HBase server"); } throw ioe; } } else { rs.compactRegion(hri, major); } }
void close(boolean stopProxy) { if (this.closed) { return; } if (master != null) { if (stopProxy) { HBaseRPC.stopProxy(master); } master = null; masterChecked = false; } if (stopProxy) { for (HRegionInterface i : servers.values()) { HBaseRPC.stopProxy(i); } } this.servers.clear(); if (this.zooKeeper != null) { LOG.info("Closed zookeeper sessionid=0x" + Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId())); this.zooKeeper.close(); this.zooKeeper = null; } this.closed = true; }
public void compactAllRegionPerRSThatNeedIt(int minStoreFiles, String columnFamily) throws IOException, InterruptedException { System.out.println("Table Name:" + tableName); for (HRegionInfo region: regions) { System.out.println("Region: " + Bytes.toString(region.getRegionName()) + " " + region.getRegionId()); System.out.println(" StartKey: " + Bytes.toString(region.getStartKey()) + ", EndKey: " + Bytes.toString(region.getEndKey())); System.out.println(" hasSplit: " + region.isSplit()); System.out.println(" hasSplitPatant: " + region.isSplitParent()); System.out.println(" maxFileSize: " + tableDescriptor.getMaxFileSize()); System.out.println(" SplitPolicy: " + tableDescriptor.getRegionSplitPolicyClassName()); List<HRegionLocation> regionLocationList = table.getRegionsInRange(region.getStartKey(), region.getEndKey()); for (HRegionLocation regionLocation: regionLocationList) { HRegionInterface rs = hConnection.getHRegionConnection(regionLocation.getHostname(), regionLocation.getPort()); List<String> storeFileList = rs.getStoreFileList(region.getRegionName()); System.out.println(" Compaction State: " + rs.getCompactionState(region.getRegionName())); System.out.println(" Store File Count: " + storeFileList.size()); if ( storeFileList.size() > minStoreFiles) { System.out.println(" !!! Compacting !!!"); rs.compactRegion(region, true, Bytes.toBytes(columnFamily)); } } } }
/** * Test get of root region fails properly if nothing to connect to. * @throws IOException * @throws InterruptedException * @throws KeeperException */ @Test public void testVerifyRootRegionLocationFails() throws IOException, InterruptedException, KeeperException { HConnection connection = Mockito.mock(HConnection.class); ConnectException connectException = new ConnectException("Connection refused"); final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())). thenThrow(connectException); Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean())). thenReturn(implementation); final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { RootLocationEditor.setRootLocation(this.watcher, new ServerName("example.com", 1234, System.currentTimeMillis())); Assert.assertFalse(ct.verifyRootRegionLocation(100)); } finally { // Clean out root location or later tests will be confused... they presume // start fresh in zk. RootLocationEditor.deleteRootLocation(this.watcher); } }
private void testVerifyMetaRegionLocationWithException(Exception ex) throws IOException, InterruptedException, KeeperException { // Mock an HRegionInterface. final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); HConnection connection = mockConnection(implementation); try { // If a 'get' is called on mocked interface, throw connection refused. Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())). thenThrow(ex); // Now start up the catalogtracker with our doctored Connection. final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { RootLocationEditor.setRootLocation(this.watcher, SN); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); } finally { // Clean out root location or later tests will be confused... they // presume start fresh in zk. RootLocationEditor.deleteRootLocation(this.watcher); } } finally { // Clear out our doctored connection or could mess up subsequent tests. HConnectionManager.deleteConnection(UTIL.getConfiguration()); } }
