@Test(timeout = 180000, expected = ServiceException.class) public void testReplicateWALEntryWhenReplicationIsDisabled() throws Exception { LOG.info("testSimplePutDelete"); baseConfiguration.setBoolean(HConstants.REPLICATION_ENABLE_KEY, false); Table[] htables = null; try { startMiniClusters(1); createTableOnClusters(table); htables = getHTablesOnClusters(tableName); HRegionServer rs = utilities[0].getRSForFirstRegionInTable(tableName); RSRpcServices rsrpc = new RSRpcServices(rs); rsrpc.replicateWALEntry(null, null); } finally { close(htables); shutDownMiniClusters(); } }
@Test @SuppressWarnings("deprecation") public void testShortCircuitConnection() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = UTIL.createTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf")); htd.addFamily(hcd); UTIL.createTable(htd, null); HRegionServer regionServer = UTIL.getRSForFirstRegionInTable(tableName); ClusterConnection connection = regionServer.getClusterConnection(); Table tableIf = connection.getTable(tableName); assertTrue(tableIf instanceof HTable); HTable table = (HTable) tableIf; assertTrue(table.getConnection() == connection); AdminService.BlockingInterface admin = connection.getAdmin(regionServer.getServerName()); ClientService.BlockingInterface client = connection.getClient(regionServer.getServerName()); assertTrue(admin instanceof RSRpcServices); assertTrue(client instanceof RSRpcServices); ServerName anotherSn = ServerName.valueOf(regionServer.getServerName().getHostAndPort(), EnvironmentEdgeManager.currentTime()); admin = connection.getAdmin(anotherSn); client = connection.getClient(anotherSn); assertFalse(admin instanceof RSRpcServices); assertFalse(client instanceof RSRpcServices); assertTrue(connection.getAdmin().getConnection() == connection); }
@Test @SuppressWarnings("deprecation") public void testShortCircuitConnection() throws IOException, InterruptedException { String tnAsString = "testShortCircuitConnection"; TableName tn = TableName.valueOf(tnAsString); HTableDescriptor htd = UTIL.createTableDescriptor(tnAsString); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf")); htd.addFamily(hcd); UTIL.createTable(htd, null); HRegionServer regionServer = UTIL.getRSForFirstRegionInTable(tn); ClusterConnection connection = regionServer.getConnection(); HTableInterface tableIf = connection.getTable(tn); assertTrue(tableIf instanceof HTable); HTable table = (HTable) tableIf; assertTrue(table.getConnection() == connection); AdminService.BlockingInterface admin = connection.getAdmin(regionServer.getServerName()); ClientService.BlockingInterface client = connection.getClient(regionServer.getServerName()); assertTrue(admin instanceof RSRpcServices); assertTrue(client instanceof RSRpcServices); ServerName anotherSn = ServerName.valueOf(regionServer.getServerName().getHostAndPort(), EnvironmentEdgeManager.currentTime()); admin = connection.getAdmin(anotherSn); client = connection.getClient(anotherSn); assertFalse(admin instanceof RSRpcServices); assertFalse(client instanceof RSRpcServices); assertTrue(connection.getAdmin().getConnection() == connection); }
@Override protected RSRpcServices createRpcServices() throws IOException { return new MasterRpcServices(this); }
public MasterAnnotationReadingPriorityFunction(final RSRpcServices rpcServices) { this(rpcServices, rpcServices.getClass()); }
public MasterAnnotationReadingPriorityFunction(RSRpcServices rpcServices, Class<? extends RSRpcServices> clz) { super(rpcServices, clz); }
@Override public boolean isMajorCompaction(final Collection<StoreFile> filesToCompact) throws IOException { boolean result = false; long mcTime = getNextMajorCompactTime(filesToCompact); if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) { return result; } // TODO: Use better method for determining stamp of last major (HBASE-2990) long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); long now = System.currentTimeMillis(); if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) { // Major compaction time has elapsed. long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (filesToCompact.size() == 1) { // Single file StoreFile sf = filesToCompact.iterator().next(); Long minTimestamp = sf.getMinimumTimestamp(); long oldest = (minTimestamp == null) ? Long.MIN_VALUE : now - minTimestamp.longValue(); if (sf.isMajorCompaction() && (cfTtl == HConstants.FOREVER || oldest < cfTtl)) { float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex( RSRpcServices.getHostname(comConf.conf, false) ); if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) { if (LOG.isDebugEnabled()) { LOG.debug("Major compaction triggered on only store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); } result = true; } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping major compaction of " + this + " because one (major) compacted file only, oldestTime " + oldest + "ms is < ttl=" + cfTtl + " and blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); } } } else if (cfTtl != HConstants.FOREVER && oldest > cfTtl) { LOG.debug("Major compaction triggered on store " + this + ", because keyvalues outdated; time since last major compaction " + (now - lowTimestamp) + "ms"); result = true; } } else { if (LOG.isDebugEnabled()) { LOG.debug("Major compaction triggered on store " + this + "; time since last major compaction " + (now - lowTimestamp) + "ms"); } result = true; } } return result; }
@Test public void testMetaUpdatesGoToPriorityQueue() throws Exception { // This test has to be end-to-end, and do the verification from the server side Configuration c = UTIL.getConfiguration(); c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS, SpyingRpcSchedulerFactory.class.getName()); // restart so that new config takes place afterClass(); beforeClass(); TableName tableName = TableName.valueOf("foo"); try (Admin admin = connection.getAdmin(); RegionLocator rl = connection.getRegionLocator(tableName)) { // create a table and prepare for a manual split UTIL.createTable(tableName, "cf1"); HRegionLocation loc = rl.getAllRegionLocations().get(0); HRegionInfo parent = loc.getRegionInfo(); long rid = 1000; byte[] splitKey = Bytes.toBytes("a"); HRegionInfo splitA = new HRegionInfo(parent.getTable(), parent.getStartKey(), splitKey, false, rid); HRegionInfo splitB = new HRegionInfo(parent.getTable(), splitKey, parent.getEndKey(), false, rid); // find the meta server MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster(); int rsIndex = cluster.getServerWithMeta(); HRegionServer rs; if (rsIndex >= 0) { rs = cluster.getRegionServer(rsIndex); } else { // it is in master rs = cluster.getMaster(); } SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler(); long prevCalls = scheduler.numPriorityCalls; MetaTableAccessor.splitRegion(connection, parent, splitA, splitB, loc.getServerName(), 1); assertTrue(prevCalls < scheduler.numPriorityCalls); } }
@Override protected RSRpcServices createRpcServices() throws IOException { return new FakeRSRpcServices(this); }
protected RSRpcServices createRpcServices() throws IOException { return new RSRpcServicesWithScanTimeout(this); }
public boolean isMajorCompaction(final Collection<StoreFile> filesToCompact) throws IOException { boolean result = false; long mcTime = getNextMajorCompactTime(filesToCompact); if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) { return result; } // TODO: Use better method for determining stamp of last major (HBASE-2990) long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); long now = System.currentTimeMillis(); if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) { // Major compaction time has elapsed. long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (filesToCompact.size() == 1) { // Single file StoreFile sf = filesToCompact.iterator().next(); Long minTimestamp = sf.getMinimumTimestamp(); long oldest = (minTimestamp == null) ? Long.MIN_VALUE : now - minTimestamp.longValue(); if (sf.isMajorCompaction() && (cfTtl == HConstants.FOREVER || oldest < cfTtl)) { float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex( RSRpcServices.getHostname(comConf.conf) ); if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) { if (LOG.isDebugEnabled()) { LOG.debug("Major compaction triggered on only store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); } result = true; } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping major compaction of " + this + " because one (major) compacted file only, oldestTime " + oldest + "ms is < ttl=" + cfTtl + " and blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); } } } else if (cfTtl != HConstants.FOREVER && oldest > cfTtl) { LOG.debug("Major compaction triggered on store " + this + ", because keyvalues outdated; time since last major compaction " + (now - lowTimestamp) + "ms"); result = true; } } else { if (LOG.isDebugEnabled()) { LOG.debug("Major compaction triggered on store " + this + "; time since last major compaction " + (now - lowTimestamp) + "ms"); } result = true; } } return result; }
protected final void uncaughtException(Thread t, Throwable e) { RSRpcServices.exitIfOOME(e); LOG.error("Unexpected exception in " + t.getName() + " currentPath=" + getCurrentPath(), e); server.abort("Unexpected exception in " + t.getName(), e); }
@Override public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact) throws IOException { long mcTime = getNextMajorCompactTime(filesToCompact); if (filesToCompact == null || mcTime == 0) { if (LOG.isDebugEnabled()) { LOG.debug("filesToCompact: " + filesToCompact + " mcTime: " + mcTime); } return false; } // TODO: Use better method for determining stamp of last major (HBASE-2990) long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); long now = EnvironmentEdgeManager.currentTime(); if (lowTimestamp <= 0L || lowTimestamp >= (now - mcTime)) { if (LOG.isDebugEnabled()) { LOG.debug("lowTimestamp: " + lowTimestamp + " lowTimestamp: " + lowTimestamp + " now: " + now + " mcTime: " + mcTime); } return false; } long cfTTL = this.storeConfigInfo.getStoreFileTtl(); HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); List<Long> boundaries = getCompactBoundariesForMajor(filesToCompact, now); boolean[] filesInWindow = new boolean[boundaries.size()]; for (HStoreFile file: filesToCompact) { OptionalLong minTimestamp = file.getMinimumTimestamp(); long oldest = minTimestamp.isPresent() ? now - minTimestamp.getAsLong() : Long.MIN_VALUE; if (cfTTL != Long.MAX_VALUE && oldest >= cfTTL) { LOG.debug("Major compaction triggered on store " + this + "; for TTL maintenance"); return true; } if (!file.isMajorCompactionResult() || file.isBulkLoadResult()) { LOG.debug("Major compaction triggered on store " + this + ", because there are new files and time since last major compaction " + (now - lowTimestamp) + "ms"); return true; } int lowerWindowIndex = Collections.binarySearch(boundaries, minTimestamp.orElse(Long.MAX_VALUE)); int upperWindowIndex = Collections.binarySearch(boundaries, file.getMaximumTimestamp().orElse(Long.MAX_VALUE)); // Handle boundary conditions and negative values of binarySearch lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex; upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex; if (lowerWindowIndex != upperWindowIndex) { LOG.debug("Major compaction triggered on store " + this + "; because file " + file.getPath() + " has data with timestamps cross window boundaries"); return true; } else if (filesInWindow[upperWindowIndex]) { LOG.debug("Major compaction triggered on store " + this + "; because there are more than one file in some windows"); return true; } else { filesInWindow[upperWindowIndex] = true; } hdfsBlocksDistribution.add(file.getHDFSBlockDistribution()); } float blockLocalityIndex = hdfsBlocksDistribution .getBlockLocalityIndex(RSRpcServices.getHostname(comConf.conf, false)); if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) { LOG.debug("Major compaction triggered on store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); return true; } LOG.debug("Skipping major compaction of " + this + ", because the files are already major compacted"); return false; }
@Override public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact) throws IOException { boolean result = false; long mcTime = getNextMajorCompactTime(filesToCompact); if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) { return result; } // TODO: Use better method for determining stamp of last major (HBASE-2990) long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); long now = EnvironmentEdgeManager.currentTime(); if (lowTimestamp > 0L && lowTimestamp < (now - mcTime)) { String regionInfo; if (this.storeConfigInfo != null && this.storeConfigInfo instanceof HStore) { regionInfo = ((HStore)this.storeConfigInfo).getRegionInfo().getRegionNameAsString(); } else { regionInfo = this.toString(); } // Major compaction time has elapsed. long cfTTL = HConstants.FOREVER; if (this.storeConfigInfo != null) { cfTTL = this.storeConfigInfo.getStoreFileTtl(); } if (filesToCompact.size() == 1) { // Single file HStoreFile sf = filesToCompact.iterator().next(); OptionalLong minTimestamp = sf.getMinimumTimestamp(); long oldest = minTimestamp.isPresent() ? now - minTimestamp.getAsLong() : Long.MIN_VALUE; if (sf.isMajorCompactionResult() && (cfTTL == Long.MAX_VALUE || oldest < cfTTL)) { float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex( RSRpcServices.getHostname(comConf.conf, false)); if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) { LOG.debug("Major compaction triggered on only store " + regionInfo + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); result = true; } else { LOG.debug("Skipping major compaction of " + regionInfo + " because one (major) compacted file only, oldestTime " + oldest + "ms is < TTL=" + cfTTL + " and blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")"); } } else if (cfTTL != HConstants.FOREVER && oldest > cfTTL) { LOG.debug("Major compaction triggered on store " + regionInfo + ", because keyvalues outdated; time since last major compaction " + (now - lowTimestamp) + "ms"); result = true; } } else { LOG.debug("Major compaction triggered on store " + regionInfo + "; time since last major compaction " + (now - lowTimestamp) + "ms"); result = true; } } return result; }
@Override public void setRsRpcServices(RSRpcServices rsRpcServices) { this.rsRpcServices = rsRpcServices; }
@Test public void testMetaUpdatesGoToPriorityQueue() throws Exception { // This test has to be end-to-end, and do the verification from the server side Configuration c = UTIL.getConfiguration(); c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS, SpyingRpcSchedulerFactory.class.getName()); // restart so that new config takes place afterClass(); beforeClass(); final TableName tableName = TableName.valueOf(name.getMethodName()); try (Admin admin = connection.getAdmin(); RegionLocator rl = connection.getRegionLocator(tableName)) { // create a table and prepare for a manual split UTIL.createTable(tableName, "cf1"); HRegionLocation loc = rl.getAllRegionLocations().get(0); RegionInfo parent = loc.getRegionInfo(); long rid = 1000; byte[] splitKey = Bytes.toBytes("a"); RegionInfo splitA = RegionInfoBuilder.newBuilder(parent.getTable()) .setStartKey(parent.getStartKey()) .setEndKey(splitKey) .setSplit(false) .setRegionId(rid) .build(); RegionInfo splitB = RegionInfoBuilder.newBuilder(parent.getTable()) .setStartKey(splitKey) .setEndKey(parent.getEndKey()) .setSplit(false) .setRegionId(rid) .build(); // find the meta server MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster(); int rsIndex = cluster.getServerWithMeta(); HRegionServer rs; if (rsIndex >= 0) { rs = cluster.getRegionServer(rsIndex); } else { // it is in master rs = cluster.getMaster(); } SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler(); long prevCalls = scheduler.numPriorityCalls; MetaTableAccessor.splitRegion(connection, parent, splitA, splitB,loc.getServerName(),1,false); assertTrue(prevCalls < scheduler.numPriorityCalls); } }
@Override protected RSRpcServices createRpcServices() throws IOException { this.rsRpcServices = new FakeRSRpcServices(this); return rsRpcServices; }
@Override protected RSRpcServices createRpcServices() throws IOException { return new RSRpcServicesWithScanTimeout(this); }
protected RSRpcServices createRpcServices() throws IOException { return new MasterRpcServices(this); }
void setRsRpcServices(RSRpcServices rsRpcServices);