Java 类org.apache.hadoop.hbase.procedure2.ProcedureExecutor 实例源码

private void startProcedureExecutor() throws IOException {
  final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
  final Path logDir = new Path(fileSystemManager.getRootDir(),
      MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);

  procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
      new MasterProcedureEnv.WALStoreLeaseRecovery(this));
  procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
  procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
      procEnv.getProcedureQueue());

  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
      Math.max(Runtime.getRuntime().availableProcessors(),
        MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption = conf.getBoolean(
      MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  procedureStore.start(numThreads);
  procedureExecutor.start(numThreads, abortOnCorruption);
}

public static byte[] waitForProcedureToComplete(ProcedureExecutor<MasterProcedureEnv> procExec,
    final long procId) throws IOException {
  while (!procExec.isFinished(procId) && procExec.isRunning()) {
    // TODO: add a config to make it tunable
    // Dev Consideration: are we waiting forever, or we can set up some timeout value?
    Threads.sleepWithoutInterrupt(250);
  }
  ProcedureInfo result = procExec.getResult(procId);
  if (result != null) {
    if (result.isFailed()) {
      // If the procedure fails, we should always have an exception captured. Throw it.
      throw RemoteProcedureException.fromProto(
        result.getForeignExceptionMessage()).unwrapRemoteException();
    }
    return result.getResult();
  } else {
    if (procExec.isRunning()) {
      throw new IOException("Procedure " + procId + "not found");
    } else {
      throw new IOException("The Master is Aborting");
    }
  }
}

@Test
public void testAbortProcedure() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedure");
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER.getShortName());
  final long procId = procExec.submitProcedure(proc);

  AccessTestAction abortProcedureAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      ACCESS_CONTROLLER
      .preAbortProcedure(ObserverContext.createAndPrepare(CP_ENV, null), procExec, procId);
     return null;
    }
  };

  verifyAllowed(abortProcedureAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(abortProcedureAction, USER_OWNER);
  verifyDenied(
    abortProcedureAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Test(timeout=60000)
public void testDeleteNonExistingColumnFamily() throws Exception {
  final TableName tableName = TableName.valueOf("testDeleteNonExistingColumnFamily");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final String cf3 = "cf3";

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // delete the column family that does not exist
  long procId1 = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf3.getBytes()),
    nonceGroup,
    nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);

  ProcedureInfo result = procExec.getResult(procId1);
  assertTrue(result.isFailed());
  LOG.debug("Delete failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
}

@Test(timeout = 60000)
public void testRecoveryAndDoubleExecutionOnline() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
  final String cf5 = "cf5";

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  // create the table
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3", cf5);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Delete procedure && kill the executor
  long procId = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf5.getBytes()),
    nonceGroup,
    nonce);

  // Restart the executor and execute the step twice
  int numberOfSteps = DeleteColumnFamilyState.values().length;
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
    DeleteColumnFamilyState.values());

  MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
    tableName, cf5);
}

@Test(timeout = 60000)
public void testDisableTable() throws Exception {
  final TableName tableName = TableName.valueOf("testDisableTable");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // Disable the table
  long procId = procExec.submitProcedure(
    new DisableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout = 60000)
public void testDisableTableTwiceWithSameNonce() throws Exception {
  final TableName tableName = TableName.valueOf("testDisableTableTwiceWithSameNonce");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // Disable the table
  long procId1 = procExec.submitProcedure(new DisableTableProcedure(
      procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  long procId2 = procExec.submitProcedure(new DisableTableProcedure(
    procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);

  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
  assertTrue(procId1 == procId2);
}

@Test(timeout=60000)
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2");

  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Disable procedure && kill the executor
  long procId = procExec.submitProcedure(
    new DisableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);

  // Restart the executor and execute the step twice
  int numberOfSteps = DisableTableState.values().length;
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
    procExec,
    procId,
    numberOfSteps,
    DisableTableState.values());
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout = 60000)
public void testRecoveryAndDoubleExecutionOnline() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
  final String cf5 = "cf5";
  final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf5);
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  // create the table
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3");

  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the AddColumnFamily procedure && kill the executor
  long procId = procExec.submitProcedure(
    new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup,
    nonce);

  // Restart the executor and execute the step twice
  int numberOfSteps = AddColumnFamilyState.values().length;
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
    AddColumnFamilyState.values());

  MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
    tableName, cf5);
}

@Test(timeout = 60000)
public void testRollbackAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
  final String cf6 = "cf6";
  final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf6);
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  // create the table
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the AddColumnFamily procedure && kill the executor
  long procId = procExec.submitProcedure(
    new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup,
    nonce);

  int numberOfSteps = AddColumnFamilyState.values().length - 2; // failing in the middle of proc
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps,
    AddColumnFamilyState.values());

  MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
    tableName, cf6);
}

@Test(timeout=60000)
public void testAbortProcedureSuccess() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedureSuccess");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
  // Submit an abortable procedure
  long procId = procExec.submitProcedure(
      new DisableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);

  boolean abortResult = procExec.abort(procId, true);
  assertTrue(abortResult);

  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
  ProcedureTestingUtility.restart(procExec);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  // Validate the disable table procedure was aborted successfully
  MasterProcedureTestingUtility.validateTableIsEnabled(
    UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout=60000)
public void testAbortProcedureFailure() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedureFailure");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  HRegionInfo[] regions =
      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
  UTIL.getHBaseAdmin().disableTable(tableName);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
  // Submit an un-abortable procedure
  long procId = procExec.submitProcedure(
      new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

  boolean abortResult = procExec.abort(procId, true);
  assertFalse(abortResult);

  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
  ProcedureTestingUtility.restart(procExec);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  // Validate the delete table procedure was not aborted
  MasterProcedureTestingUtility.validateTableDeletion(
    UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");
}

@Test(timeout=60000, expected=TableExistsException.class)
public void testCreateExisting() throws Exception {
  final TableName tableName = TableName.valueOf("testCreateExisting");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
  final HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, null);

  // create the table
  long procId1 = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions), nonceGroup, nonce);

  // create another with the same name
  ProcedurePrepareLatch latch2 = new ProcedurePrepareLatch.CompatibilityLatch();
  long procId2 = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions, latch2),
    nonceGroup + 1,
    nonce + 1);

  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));

  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  latch2.await();
}

@Test(timeout=60000)
public void testCreateTwiceWithSameNonce() throws Exception {
  final TableName tableName = TableName.valueOf("testCreateTwiceWithSameNonce");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f");
  final HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, null);

  // create the table
  long procId1 = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions), nonceGroup, nonce);

  // create another with the same name
  long procId2 = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions), nonceGroup, nonce);

  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));

  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
  assertTrue(procId1 == procId2);
}

@Test(timeout=60000)
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");

  // create the table
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Create procedure && kill the executor
  byte[][] splitKeys = null;
  HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
  HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
  long procId = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions), nonceGroup, nonce);

  // Restart the executor and execute the step twice
  // NOTE: the 6 (number of CreateTableState steps) is hardcoded,
  //       so you have to look at this test at least once when you add a new step.
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
    procExec, procId, 6, CreateTableState.values());

  MasterProcedureTestingUtility.validateTableCreation(
    UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
}

@Test(timeout=60000)
public void testTruncateNotDisabledTable() throws Exception {
  final TableName tableName = TableName.valueOf("testTruncateNotDisabledTable");

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");

  long procId = ProcedureTestingUtility.submitAndWait(procExec,
      new TruncateTableProcedure(procExec.getEnvironment(), tableName, false));

  // Second delete should fail with TableNotDisabled
  ProcedureInfo result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Truncate failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);
}

@Test(timeout = 60000)
public void testEnableTable() throws Exception {
  final TableName tableName = TableName.valueOf("testEnableTable");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
  UTIL.getHBaseAdmin().disableTable(tableName);

  // Enable the table
  long procId = procExec.submitProcedure(
    new EnableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout = 60000)
public void testEnableTableTwiceWithSameNonce() throws Exception {
  final TableName tableName = TableName.valueOf("testEnableTableTwiceWithSameNonce");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
  UTIL.getHBaseAdmin().disableTable(tableName);

  // Enable the table
  long procId1 = procExec.submitProcedure(
    new EnableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  long procId2 = procExec.submitProcedure(
    new EnableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);

  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
  // The second proc should succeed too - because it is the same proc.
  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
  assertTrue(procId1 == procId2);
}

@Test(timeout = 60000)
public void testRollbackAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2");
  UTIL.getHBaseAdmin().disableTable(tableName);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Enable procedure && kill the executor
  long procId = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);

  int numberOfSteps = EnableTableState.values().length - 2; // failing in the middle of proc
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
    procExec,
    procId,
    numberOfSteps,
    EnableTableState.values());
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);
}

private void startProcedureExecutor() throws IOException {
  final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);

  procedureStore = new WALProcedureStore(conf,
      new MasterProcedureEnv.WALStoreLeaseRecovery(this));
  procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
  MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler();
  procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler);
  configurationManager.registerObserver(procEnv);

  int cpus = Runtime.getRuntime().availableProcessors();
  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
      Math.max((cpus > 0? cpus/4: 0),
          MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption = conf.getBoolean(
      MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  procedureStore.start(numThreads);
  procedureExecutor.start(numThreads, abortOnCorruption);
  procEnv.getRemoteDispatcher().start();
}

public static byte[] waitForProcedureToComplete(
    final ProcedureExecutor<MasterProcedureEnv> procExec,
    final Procedure<?> proc, final long timeout)
    throws IOException {
  waitFor(procExec.getEnvironment(), "pid=" + proc.getProcId(),
    new ProcedureSyncWait.Predicate<Boolean>() {
      @Override
      public Boolean evaluate() throws IOException {
        return !procExec.isRunning() || procExec.isFinished(proc.getProcId());
      }
    }
  );

  if (!procExec.isRunning()) {
    throw new IOException("The Master is Aborting");
  }

  if (proc.hasException()) {
    // If the procedure fails, we should always have an exception captured. Throw it.
    throw proc.getException().unwrapRemoteIOException();
  } else {
    return proc.getResult();
  }
}

/**
 * An ugly utility to be removed when refactor TableNamespaceManager.
 * @throws TimeoutIOException
 */
private static void block(final MasterServices services, final long procId)
throws TimeoutIOException {
  int timeoutInMillis = services.getConfiguration().
      getInt(ClusterSchema.HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY,
          ClusterSchema.DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT);
  long deadlineTs = EnvironmentEdgeManager.currentTime() + timeoutInMillis;
  ProcedureExecutor<MasterProcedureEnv> procedureExecutor =
      services.getMasterProcedureExecutor();
  while(EnvironmentEdgeManager.currentTime() < deadlineTs) {
    if (procedureExecutor.isFinished(procId)) return;
    // Sleep some
    Threads.sleep(10);
  }
  throw new TimeoutIOException("Procedure pid=" + procId + " is still running");
}

@Test
public void testGetProcedures() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER);
  procExec.submitProcedure(proc);
  final List<Procedure<?>> procList = procExec.getProcedures();

  AccessTestAction getProceduresAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      ACCESS_CONTROLLER
      .postGetProcedures(ObserverContextImpl.createAndPrepare(CP_ENV));
     return null;
    }
  };

  verifyAllowed(getProceduresAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(getProceduresAction, USER_OWNER);
  verifyIfNull(
    getProceduresAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Test
public void testRace() throws Exception {
  ProcedureExecutor<?> executor =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
  DummyProcedure p = new DummyProcedure();
  long procId = executor.submitProcedure(p);
  p.failureSet.await();
  assertEquals(GetProcedureResultResponse.State.RUNNING, getState(procId));
  p.canRollback.countDown();
  UTIL.waitFor(30000, new Waiter.ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return getState(procId) == GetProcedureResultResponse.State.FINISHED;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Procedure pid=" + procId + " is still in " + getState(procId) +
        " state, expected " + GetProcedureResultResponse.State.FINISHED;
    }
  });
}

@Test
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  List<RegionInfo> tableRegions = createTable(tableName);

  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  RegionInfo[] regionsToMerge = new RegionInfo[2];
  regionsToMerge[0] = tableRegions.get(0);
  regionsToMerge[1] = tableRegions.get(1);

  long procId = procExec.submitProcedure(
    new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));

  // Restart the executor and execute the step twice
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);

  assertRegionCount(tableName, initialRegionCount - 1);
}

@Test(timeout = 60000)
public void testEnableTable() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
  UTIL.getAdmin().disableTable(tableName);

  // Enable the table
  long procId = procExec.submitProcedure(
    new EnableTableProcedure(procExec.getEnvironment(), tableName, false));
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  MasterProcedureTestingUtility.validateTableIsEnabled(getMaster(), tableName);
}

@Test(timeout=60000)
public void testDeleteNonEmptyNamespace() throws Exception {
  final String namespaceName = "testDeleteNonExistNamespace";
  final TableName tableName = TableName.valueOf("testDeleteNonExistNamespace:" + name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  // create namespace
  createNamespaceForTesting(namespaceName);
  // create the table under the new namespace
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");

  long procId = procExec.submitProcedure(
    new DeleteNamespaceProcedure(procExec.getEnvironment(), namespaceName));
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  Procedure<?> result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Delete namespace failed with exception: " + result.getException());
  assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException);
}

private void startProcedureExecutor(final RSProcedureDispatcher remoteDispatcher)
    throws IOException {
  final Configuration conf = getConfiguration();
  final Path logDir = new Path(fileSystemManager.getRootDir(),
      WALProcedureStore.MASTER_PROCEDURE_LOGDIR);

  this.procedureStore = new NoopProcedureStore();
  this.procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));

  this.procedureEnv = new MasterProcedureEnv(this,
     remoteDispatcher != null ? remoteDispatcher : new RSProcedureDispatcher(this));

  this.procedureExecutor = new ProcedureExecutor(conf, procedureEnv, procedureStore,
      procedureEnv.getProcedureScheduler());

  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
      Math.max(Runtime.getRuntime().availableProcessors(),
        MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption = conf.getBoolean(
      MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  this.procedureStore.start(numThreads);
  this.procedureExecutor.start(numThreads, abortOnCorruption);
  this.procedureEnv.getRemoteDispatcher().start();
}

@Test(timeout = 60000)
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2");
  UTIL.getAdmin().disableTable(tableName);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Enable procedure && kill the executor
  long procId = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, false));

  // Restart the executor and execute the step twice
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);

  MasterProcedureTestingUtility.validateTableIsEnabled(getMaster(), tableName);
}

@Test(timeout=60000)
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());

  // create the table
  byte[][] splitKeys = null;
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
    getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
  UTIL.getAdmin().disableTable(tableName);

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Delete procedure && kill the executor
  long procId = procExec.submitProcedure(
    new DeleteTableProcedure(procExec.getEnvironment(), tableName));

  // Restart the executor and execute the step twice
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);

  MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);
}

@Test(timeout=60000)
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2");

  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Disable procedure && kill the executor
  long procId = procExec.submitProcedure(
    new DisableTableProcedure(procExec.getEnvironment(), tableName, false));

  // Restart the executor and execute the step twice
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);

  MasterProcedureTestingUtility.validateTableIsDisabled(getMaster(), tableName);
}

private void testCreateWithFailoverAtStep(final int step) throws Exception {
  final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step);

  // create the table
  ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
  ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);

  // Start the Create procedure && kill the executor
  byte[][] splitKeys = null;
  TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
  RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
  long procId = procExec.submitProcedure(
      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
  testRecoveryAndDoubleExecution(UTIL, procId, step);

  MasterProcedureTestingUtility.validateTableCreation(
      UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
}

private void testDeleteWithFailoverAtStep(final int step) throws Exception {
  final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step);

  // create the table
  byte[][] splitKeys = null;
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
  Path tableDir = FSUtils.getTableDir(getRootDir(), tableName);
  MasterProcedureTestingUtility.validateTableCreation(
      UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  UTIL.getAdmin().disableTable(tableName);

  ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
  ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);

  // Start the Delete procedure && kill the executor
  long procId = procExec.submitProcedure(
      new DeleteTableProcedure(procExec.getEnvironment(), tableName));
  testRecoveryAndDoubleExecution(UTIL, procId, step);

  MasterProcedureTestingUtility.validateTableDeletion(
      UTIL.getHBaseCluster().getMaster(), tableName);
}

private void testDisableTableWithFailoverAtStep(final int step) throws Exception {
  final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step);

  // create the table
  final byte[][] splitKeys = new byte[][] {
      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(
      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");

  ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Delete procedure && kill the executor
  long procId = procExec.submitProcedure(
      new DisableTableProcedure(procExec.getEnvironment(), tableName, false));
  testRecoveryAndDoubleExecution(UTIL, procId, step);

  MasterProcedureTestingUtility.validateTableIsDisabled(
      UTIL.getHBaseCluster().getMaster(), tableName);
}

private void testEnableTableWithFailoverAtStep(final int step) throws Exception {
  final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step);

  // create the table
  final byte[][] splitKeys = new byte[][] {
      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  MasterProcedureTestingUtility.createTable(
      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
  UTIL.getAdmin().disableTable(tableName);

  ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Delete procedure && kill the executor
  long procId = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, false));
  testRecoveryAndDoubleExecution(UTIL, procId, step);

  MasterProcedureTestingUtility.validateTableIsEnabled(
      UTIL.getHBaseCluster().getMaster(), tableName);
}

public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil,
    final long procId, final int lastStepBeforeFailover) throws Exception {
  ProcedureExecutor<MasterProcedureEnv> procExec =
      testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  ProcedureTestingUtility.waitProcedure(procExec, procId);

  final Procedure proc = procExec.getProcedure(procId);
  for (int i = 0; i < lastStepBeforeFailover; ++i) {
    LOG.info("Restart "+ i +" exec state: " + proc);
    ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
    ProcedureTestingUtility.waitProcedure(procExec, procId);
  }
  ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);

  LOG.info("Trigger master failover");
  MasterProcedureTestingUtility.masterFailover(testUtil);

  procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
}

@Test
public void testTruncateNotDisabledTable() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");

  long procId = ProcedureTestingUtility.submitAndWait(procExec,
      new TruncateTableProcedure(procExec.getEnvironment(), tableName, false));

  // Second delete should fail with TableNotDisabled
  Procedure<?> result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Truncate failed with exception: " + result.getException());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);
}

@Test(timeout=60000)
public void testAbortProcedureSuccess() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
  // Submit an abortable procedure
  long procId = procExec.submitProcedure(
      new DisableTableProcedure(procExec.getEnvironment(), tableName, false));
  // Wait for one step to complete
  ProcedureTestingUtility.waitProcedure(procExec, procId);

  boolean abortResult = procExec.abort(procId, true);
  assertTrue(abortResult);

  MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId);
  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  // Validate the disable table procedure was aborted successfully
  MasterProcedureTestingUtility.validateTableIsEnabled(
    UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout = 60000)
public void testRollbackAndDoubleExecution() throws Exception {
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution");
  final HTableDescriptor htd = createHTableDescriptor(clonedTableName, CF);

  // take the snapshot
  SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();

  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Clone snapshot procedure && kill the executor
  long procId = procExec.submitProcedure(
    new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));

  int numberOfSteps = 0; // failing at pre operation
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);

  MasterProcedureTestingUtility.validateTableDeletion(
    UTIL.getHBaseCluster().getMaster(), clonedTableName);
}

@Test(timeout=60000)
public void testModifyNamespaceWithInvalidRegionCount() throws Exception {
  final NamespaceDescriptor nsd =
      NamespaceDescriptor.create("testModifyNamespaceWithInvalidRegionCount").build();
  final String nsKey = "hbase.namespace.quota.maxregions";
  final String nsValue = "-1";
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  createNamespaceForTesting(nsd);

  // Modify
  nsd.setConfiguration(nsKey, nsValue);

  long procId = procExec.submitProcedure(
    new ModifyNamespaceProcedure(procExec.getEnvironment(), nsd));
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  Procedure<?> result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Modify namespace failed with exception: " + result.getException());
  assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException);
}