private void assignRegionToAvailableFavoredNode(Map<ServerName, List<HRegionInfo>> assignmentMapForFavoredNodes, HRegionInfo region, ServerName primaryHost, ServerName secondaryHost, ServerName tertiaryHost) { if (primaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, primaryHost); } else if (secondaryHost != null && tertiaryHost != null) { // assign the region to the one with a lower load // (both have the desired hdfs blocks) ServerName s; ServerLoad tertiaryLoad = super.services.getServerManager().getLoad(tertiaryHost); ServerLoad secondaryLoad = super.services.getServerManager().getLoad(secondaryHost); if (secondaryLoad.getLoad() < tertiaryLoad.getLoad()) { s = secondaryHost; } else { s = tertiaryHost; } addRegionToMap(assignmentMapForFavoredNodes, region, s); } else if (secondaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, secondaryHost); } else if (tertiaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, tertiaryHost); } }
/** * Let the server manager know a new regionserver has come online * @param request the startup request * @param ia the InetAddress from which request is received * @return The ServerName we know this server as. * @throws IOException */ ServerName regionServerStartup(RegionServerStartupRequest request, InetAddress ia) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has // not yet run, or from a server whose fail we are currently processing. // Test its host+port combo is present in serverAddresstoServerInfo. If it // is, reject the server and trigger its expiration. The next time it comes // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. final String hostname = request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() :ia.getHostName(); ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode()); checkClockSkew(sn, request.getServerCurrentTime()); checkIsDead(sn, "STARTUP"); if (!checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) { LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn); } return sn; }
void regionServerReport(ServerName sn, ServerLoad sl) throws YouAreDeadException { checkIsDead(sn, "REPORT"); if (null == this.onlineServers.replace(sn, sl)) { // Already have this host+port combo and its just different start code? // Just let the server in. Presume master joining a running cluster. // recordNewServer is what happens at the end of reportServerStartup. // The only thing we are skipping is passing back to the regionserver // the ServerName to use. Here we presume a master has already done // that so we'll press on with whatever it gave us for ServerName. if (!checkAndRecordNewServer(sn, sl)) { LOG.info("RegionServerReport ignored, could not record the server: " + sn); return; // Not recorded, so no need to move on } } updateLastFlushedSequenceIds(sn, sl); }
@Override public RegionServerReportResponse regionServerReport( RpcController controller, RegionServerReportRequest request) throws ServiceException { try { master.checkServiceStarted(); ClusterStatusProtos.ServerLoad sl = request.getLoad(); ServerName serverName = ProtobufUtil.toServerName(request.getServer()); ServerLoad oldLoad = master.serverManager.getLoad(serverName); master.serverManager.regionServerReport(serverName, new ServerLoad(sl)); if (sl != null && master.metricsMaster != null) { // Up our metrics. master.metricsMaster.incrementRequests(sl.getTotalNumberOfRequests() - (oldLoad != null ? oldLoad.getTotalNumberOfRequests() : 0)); } } catch (IOException ioe) { throw new ServiceException(ioe); } return RegionServerReportResponse.newBuilder().build(); }
/** * return the subset of all regionservers * (actually returns set of ServerLoads) * which host some region in a given table. * used by assertAllRegionServers() below to * test reporting of loaded coprocessors. * @param tableName : given table. * @return subset of all servers. */ Map<ServerName, ServerLoad> serversForTable(String tableName) { Map<ServerName, ServerLoad> serverLoadHashMap = new HashMap<ServerName, ServerLoad>(); for(Map.Entry<ServerName,ServerLoad> server: TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager(). getOnlineServers().entrySet()) { for( Map.Entry<byte[], RegionLoad> region: server.getValue().getRegionsLoad().entrySet()) { if (region.getValue().getNameAsString().equals(tableName)) { // this server hosts a region of tableName: add this server.. serverLoadHashMap.put(server.getKey(),server.getValue()); // .. and skip the rest of the regions that it hosts. break; } } } return serverLoadHashMap; }
private ClusterStatus mockCluster(ServerLoad[] servers) { List<ServerName> serverNames = new ArrayList<ServerName>(); ClusterStatus clusterStatus = Mockito.mock(ClusterStatus.class); when(clusterStatus.getServers()).thenReturn(serverNames); int serverCounter = 0; for (ServerLoad server : servers) { ServerName serverName = mock(ServerName.class); when(serverName.getServerName()).thenReturn("server" + (serverCounter++)); serverNames.add(serverName); when(clusterStatus.getLoad(serverName)).thenReturn(server); } return clusterStatus; }
/** * Let the server manager know a new regionserver has come online * @param ia The remote address * @param port The remote port * @param serverStartcode * @param serverCurrentTime The current time of the region server in ms * @return The ServerName we know this server as. * @throws IOException */ ServerName regionServerStartup(final InetAddress ia, final int port, final long serverStartcode, long serverCurrentTime) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has // not yet run, or from a server whose fail we are currently processing. // Test its host+port combo is present in serverAddresstoServerInfo. If it // is, reject the server and trigger its expiration. The next time it comes // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode); checkClockSkew(sn, serverCurrentTime); checkIsDead(sn, "STARTUP"); if (!checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) { LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn); } return sn; }
/** * Updates last flushed sequence Ids for the regions on server sn * @param sn * @param hsl */ private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) { Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad(); for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) { byte[] encodedRegionName = Bytes.toBytes(HRegionInfo.encodeRegionName(entry.getKey())); Long existingValue = flushedSequenceIdByRegion.get(encodedRegionName); long l = entry.getValue().getCompleteSequenceId(); if (existingValue != null) { if (l != -1 && l < existingValue) { LOG.warn("RegionServer " + sn + " indicates a last flushed sequence id (" + entry.getValue() + ") that is less than the previous last flushed sequence id (" + existingValue + ") for region " + Bytes.toString(entry.getKey()) + " Ignoring."); continue; // Don't let smaller sequence ids override greater sequence ids. } } flushedSequenceIdByRegion.put(encodedRegionName, l); } }
/** * Updates last flushed sequence Ids for the regions on server sn * @param sn * @param hsl */ private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) { Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad(); for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) { Long existingValue = flushedSequenceIdByRegion.get(entry.getKey()); long l = entry.getValue().getCompleteSequenceId(); if (existingValue != null) { if (l != -1 && l < existingValue) { if (LOG.isDebugEnabled()) { LOG.debug("RegionServer " + sn + " indicates a last flushed sequence id (" + entry.getValue() + ") that is less than the previous last flushed sequence id (" + existingValue + ") for region " + Bytes.toString(entry.getKey()) + " Ignoring."); } continue; // Don't let smaller sequence ids override greater // sequence ids. } } flushedSequenceIdByRegion.put(entry.getKey(), l); } }
@Override public RegionServerReportResponse regionServerReport( RpcController controller, RegionServerReportRequest request) throws ServiceException { try { ClusterStatusProtos.ServerLoad sl = request.getLoad(); ServerName serverName = ProtobufUtil.toServerName(request.getServer()); ServerLoad oldLoad = serverManager.getLoad(serverName); this.serverManager.regionServerReport(serverName, new ServerLoad(sl)); if (sl != null && this.metricsMaster != null) { // Up our metrics. this.metricsMaster.incrementRequests(sl.getTotalNumberOfRequests() - (oldLoad != null ? oldLoad.getTotalNumberOfRequests() : 0)); } } catch (IOException ioe) { throw new ServiceException(ioe); } return RegionServerReportResponse.newBuilder().build(); }
protected void unbalanceRegions(ClusterStatus clusterStatus, List<ServerName> fromServers, List<ServerName> toServers, double fractionOfRegions) throws Exception { List<byte[]> victimRegions = new LinkedList<byte[]>(); for (ServerName server : fromServers) { ServerLoad serverLoad = clusterStatus.getLoad(server); // Ugh. List<byte[]> regions = new LinkedList<byte[]>(serverLoad.getRegionsLoad().keySet()); int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size()); LOG.debug("Removing " + victimRegionCount + " regions from " + server.getServerName()); for (int i = 0; i < victimRegionCount; ++i) { int victimIx = RandomUtils.nextInt(regions.size()); String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx)); victimRegions.add(Bytes.toBytes(regionId)); } } LOG.info("Moving " + victimRegions.size() + " regions from " + fromServers.size() + " servers to " + toServers.size() + " different servers"); HBaseAdmin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin(); for (byte[] victimRegion : victimRegions) { int targetIx = RandomUtils.nextInt(toServers.size()); admin.move(victimRegion, Bytes.toBytes(toServers.get(targetIx).getServerName())); } }
/** * return the subset of all regionservers * (actually returns set of ServerLoads) * which host some region in a given table. * used by assertAllRegionServers() below to * test reporting of loaded coprocessors. * @param tableName : given table. * @return subset of all servers. */ Map<ServerName, ServerLoad> serversForTable(String tableName) { Map<ServerName, ServerLoad> serverLoadHashMap = new HashMap<>(); for(Map.Entry<ServerName,ServerLoad> server: TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager(). getOnlineServers().entrySet()) { for( Map.Entry<byte[], RegionLoad> region: server.getValue().getRegionsLoad().entrySet()) { if (region.getValue().getNameAsString().equals(tableName)) { // this server hosts a region of tableName: add this server.. serverLoadHashMap.put(server.getKey(),server.getValue()); // .. and skip the rest of the regions that it hosts. break; } } } return serverLoadHashMap; }
private void assignRegionToAvailableFavoredNode(Map<ServerName, List<RegionInfo>> assignmentMapForFavoredNodes, RegionInfo region, ServerName primaryHost, ServerName secondaryHost, ServerName tertiaryHost) { if (primaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, primaryHost); } else if (secondaryHost != null && tertiaryHost != null) { // assign the region to the one with a lower load // (both have the desired hdfs blocks) ServerName s; ServerLoad tertiaryLoad = super.services.getServerManager().getLoad(tertiaryHost); ServerLoad secondaryLoad = super.services.getServerManager().getLoad(secondaryHost); if (secondaryLoad.getLoad() < tertiaryLoad.getLoad()) { s = secondaryHost; } else { s = tertiaryHost; } addRegionToMap(assignmentMapForFavoredNodes, region, s); } else if (secondaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, secondaryHost); } else if (tertiaryHost != null) { addRegionToMap(assignmentMapForFavoredNodes, region, tertiaryHost); } }
/** * Let the server manager know a new regionserver has come online * @param request the startup request * @param ia the InetAddress from which request is received * @return The ServerName we know this server as. * @throws IOException */ ServerName regionServerStartup(RegionServerStartupRequest request, InetAddress ia) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has // not yet run, or from a server whose fail we are currently processing. // Test its host+port combo is present in serverAddressToServerInfo. If it // is, reject the server and trigger its expiration. The next time it comes // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. final String hostname = request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() :ia.getHostName(); ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode()); checkClockSkew(sn, request.getServerCurrentTime()); checkIsDead(sn, "STARTUP"); if (!checkAndRecordNewServer(sn, new ServerLoad(ServerMetricsBuilder.of(sn)))) { LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn); } return sn; }
@VisibleForTesting public void regionServerReport(ServerName sn, ServerLoad sl) throws YouAreDeadException { checkIsDead(sn, "REPORT"); if (null == this.onlineServers.replace(sn, sl)) { // Already have this host+port combo and its just different start code? // Just let the server in. Presume master joining a running cluster. // recordNewServer is what happens at the end of reportServerStartup. // The only thing we are skipping is passing back to the regionserver // the ServerName to use. Here we presume a master has already done // that so we'll press on with whatever it gave us for ServerName. if (!checkAndRecordNewServer(sn, sl)) { LOG.info("RegionServerReport ignored, could not record the server: " + sn); return; // Not recorded, so no need to move on } } updateLastFlushedSequenceIds(sn, sl); }
/** * @param keys The target server name * @param idleServerPredicator Evaluates the server on the given load * @return A copy of the internal list of online servers matched by the predicator */ public List<ServerName> getOnlineServersListWithPredicator(List<ServerName> keys, Predicate<ServerLoad> idleServerPredicator) { List<ServerName> names = new ArrayList<>(); if (keys != null && idleServerPredicator != null) { keys.forEach(name -> { ServerLoad load = onlineServers.get(name); if (load != null) { if (idleServerPredicator.test(load)) { names.add(name); } } }); } return names; }
@Override public RegionServerReportResponse regionServerReport( RpcController controller, RegionServerReportRequest request) throws ServiceException { try { master.checkServiceStarted(); ClusterStatusProtos.ServerLoad sl = request.getLoad(); ServerName serverName = ProtobufUtil.toServerName(request.getServer()); ServerLoad oldLoad = master.getServerManager().getLoad(serverName); ServerLoad newLoad = new ServerLoad(serverName, sl); master.getServerManager().regionServerReport(serverName, newLoad); int version = VersionInfoUtil.getCurrentClientVersionNumber(); master.getAssignmentManager().reportOnlineRegions(serverName, version, newLoad.getRegionsLoad().keySet()); if (sl != null && master.metricsMaster != null) { // Up our metrics. master.metricsMaster.incrementRequests(sl.getTotalNumberOfRequests() - (oldLoad != null ? oldLoad.getTotalNumberOfRequests() : 0)); } } catch (IOException ioe) { throw new ServiceException(ioe); } return RegionServerReportResponse.newBuilder().build(); }
/** * Call this restart method only after running MockMasterServices#start() * The RSs can be differentiated by the port number, see * ServerName in MockMasterServices#start() method above. * Restart of region server will have new startcode in server name * * @param serverName Server name to be restarted */ public void restartRegionServer(ServerName serverName) throws IOException { List<ServerName> onlineServers = serverManager.getOnlineServersList(); long startCode = -1; for (ServerName s : onlineServers) { if (s.getAddress().equals(serverName.getAddress())) { startCode = s.getStartcode() + 1; break; } } if (startCode == -1) { return; } ServerName sn = ServerName.valueOf(serverName.getAddress().toString(), startCode); serverManager.regionServerReport(sn, new ServerLoad(ServerMetricsBuilder.of(sn))); }
/** * Check is a server of same host and port already exists, * if not, or the existed one got a smaller start code, record it. * * @param sn the server to check and record * @param sl the server load on the server * @return true if the server is recorded, otherwise, false */ boolean checkAndRecordNewServer( final ServerName serverName, final ServerLoad sl) { ServerName existingServer = null; synchronized (this.onlineServers) { existingServer = findServerWithSameHostnamePortWithLock(serverName); if (existingServer != null && (existingServer.getStartcode() > serverName.getStartcode())) { LOG.info("Server serverName=" + serverName + " rejected; we already have " + existingServer.toString() + " registered with same hostname and port"); return false; } recordNewServerWithLock(serverName, sl); } // Note that we assume that same ts means same server, and don't expire in that case. // TODO: ts can theoretically collide due to clock shifts, so this is a bit hacky. if (existingServer != null && (existingServer.getStartcode() < serverName.getStartcode())) { LOG.info("Triggering server recovery; existingServer " + existingServer + " looks stale, new server:" + serverName); expireServer(existingServer); } return true; }
/** * Adds the onlineServers list. onlineServers should be locked. * @param serverName The remote servers name. * @param sl * @return Server load from the removed server, if any. */ @VisibleForTesting void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) { LOG.info("Registering server=" + serverName); if (checkingBackupMaster) { ZooKeeperWatcher zooKeeper = master.getZooKeeper(); String backupZNode = ZKUtil.joinZNode( zooKeeper.backupMasterAddressesZNode, serverName.toString()); try { if (ZKUtil.checkExists(zooKeeper, backupZNode) != -1) { balancer.excludeServer(serverName); } } catch (KeeperException e) { master.abort("Failed to check if a new server a backup master", e); } } this.onlineServers.put(serverName, sl); this.rsAdmins.remove(serverName); }
/** * Let the server manager know a new regionserver has come online * @param ia The remote address * @param port The remote port * @param serverStartcode * @param serverCurrentTime The current time of the region server in ms * @return The ServerName we know this server as. * @throws IOException */ ServerName regionServerStartup(final InetAddress ia, final int port, final long serverStartcode, long serverCurrentTime) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has // not yet run, or from a server whose fail we are currently processing. // Test its host+port combo is present in serverAddresstoServerInfo. If it // is, reject the server and trigger its expiration. The next time it comes // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. ServerName sn = ServerName.valueOf(ia.getHostName(), port, serverStartcode); checkClockSkew(sn, serverCurrentTime); checkIsDead(sn, "STARTUP"); if (!checkAlreadySameHostPortAndRecordNewServer( sn, ServerLoad.EMPTY_SERVERLOAD)) { LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn); } return sn; }
