Java 类org.apache.hadoop.hbase.ProcedureInfo 实例源码

@Override
public void postListProcedures(
    ObserverContext<MasterCoprocessorEnvironment> ctx,
    List<ProcedureInfo> procInfoList) throws IOException {
  if (procInfoList.isEmpty()) {
    return;
  }

  // Retains only those which passes authorization checks, as the checks weren't done as part
  // of preListProcedures.
  Iterator<ProcedureInfo> itr = procInfoList.iterator();
  User user = getActiveUser();
  while (itr.hasNext()) {
    ProcedureInfo procInfo = itr.next();
    try {
      if (!ProcedureInfo.isProcedureOwner(procInfo, user)) {
        // If the user is not the procedure owner, then we should further probe whether
        // he can see the procedure.
        requirePermission("listProcedures", Action.ADMIN);
      }
    } catch (AccessDeniedException e) {
      itr.remove();
    }
  }
}

public static byte[] waitForProcedureToComplete(ProcedureExecutor<MasterProcedureEnv> procExec,
    final long procId) throws IOException {
  while (!procExec.isFinished(procId) && procExec.isRunning()) {
    // TODO: add a config to make it tunable
    // Dev Consideration: are we waiting forever, or we can set up some timeout value?
    Threads.sleepWithoutInterrupt(250);
  }
  ProcedureInfo result = procExec.getResult(procId);
  if (result != null) {
    if (result.isFailed()) {
      // If the procedure fails, we should always have an exception captured. Throw it.
      throw RemoteProcedureException.fromProto(
        result.getForeignExceptionMessage()).unwrapRemoteException();
    }
    return result.getResult();
  } else {
    if (procExec.isRunning()) {
      throw new IOException("Procedure " + procId + "not found");
    } else {
      throw new IOException("The Master is Aborting");
    }
  }
}

@Test(timeout=60000)
public void testDeleteNonExistingColumnFamily() throws Exception {
  final TableName tableName = TableName.valueOf("testDeleteNonExistingColumnFamily");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final String cf3 = "cf3";

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // delete the column family that does not exist
  long procId1 = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf3.getBytes()),
    nonceGroup,
    nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);

  ProcedureInfo result = procExec.getResult(procId1);
  assertTrue(result.isFailed());
  LOG.debug("Delete failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
}

@Test(timeout=60000)
public void testTruncateNotDisabledTable() throws Exception {
  final TableName tableName = TableName.valueOf("testTruncateNotDisabledTable");

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");

  long procId = ProcedureTestingUtility.submitAndWait(procExec,
      new TruncateTableProcedure(procExec.getEnvironment(), tableName, false));

  // Second delete should fail with TableNotDisabled
  ProcedureInfo result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Truncate failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);
}

/**
 * List procedures.
 * @return the procedures in a list
 */
public List<ProcedureInfo> listProcedures() {
  List<ProcedureInfo> procedureLists =
      new ArrayList<ProcedureInfo>(procedures.size() + completed.size());
  for (java.util.Map.Entry<Long, Procedure> p: procedures.entrySet()) {
    procedureLists.add(Procedure.createProcedureInfo(p.getValue(), null));
  }
  for (java.util.Map.Entry<Long, ProcedureInfo> e: completed.entrySet()) {
    // Note: The procedure could show up twice in the list with different state, as
    // it could complete after we walk through procedures list and insert into
    // procedureList - it is ok, as we will use the information in the ProcedureInfo
    // to figure it out; to prevent this would increase the complexity of the logic.
    procedureLists.add(e.getValue());
  }
  return procedureLists;
}

/**
 * Check if the user is this procedure's owner
 * @param procId the target procedure
 * @param user the user
 * @return true if the user is the owner of the procedure,
 *   false otherwise or the owner is unknown.
 */
public boolean isProcedureOwner(final long procId, final User user) {
  if (user == null) {
    return false;
  }

  Procedure proc = procedures.get(procId);
  if (proc != null) {
    return proc.getOwner().equals(user.getShortName());
  }
  ProcedureInfo procInfo = completed.get(procId);
  if (procInfo == null) {
    // Procedure either does not exist or has already completed and got cleaned up.
    // At this time, we cannot check the owner of the procedure
    return false;
  }
  return ProcedureInfo.isProcedureOwner(procInfo, user);
}

/**
 * Helper to create the ProcedureInfo from Procedure.
 */
@InterfaceAudience.Private
public static ProcedureInfo createProcedureInfo(final Procedure proc, final NonceKey nonceKey) {
  RemoteProcedureException exception = proc.hasException() ? proc.getException() : null;
  return new ProcedureInfo(
    proc.getProcId(),
    proc.toStringClass(),
    proc.getOwner(),
    proc.getState(),
    proc.hasParent() ? proc.getParentProcId() : -1,
    nonceKey,
    exception != null ?
        RemoteProcedureException.toProto(exception.getSource(), exception.getCause()) : null,
    proc.getLastUpdate(),
    proc.getStartTime(),
    proc.getResult());
}

@Test(timeout=30000)
public void testSingleStepProcRecovery() throws Exception {
  Procedure proc = new TestSingleStepProcedure();
  procExecutor.testing.killBeforeStoreUpdate = true;
  long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
  assertFalse(procExecutor.isRunning());
  procExecutor.testing.killBeforeStoreUpdate = false;

  // Restart and verify that the procedures restart
  long restartTs = EnvironmentEdgeManager.currentTime();
  restart();
  waitProcedure(procId);
  ProcedureInfo result = procExecutor.getResult(procId);
  assertTrue(result.getLastUpdate() > restartTs);
  ProcedureTestingUtility.assertProcNotFailed(result);
  assertEquals(1, Bytes.toInt(result.getResult()));
  long resultTs = result.getLastUpdate();

  // Verify that after another restart the result is still there
  restart();
  result = procExecutor.getResult(procId);
  ProcedureTestingUtility.assertProcNotFailed(result);
  assertEquals(resultTs, result.getLastUpdate());
  assertEquals(1, Bytes.toInt(result.getResult()));
}

@Test(timeout=30000)
public void testCompletedProcWithSameNonce() throws Exception {
  final long nonceGroup = 123;
  final long nonce = 2222;
  Procedure proc = new TestSingleStepProcedure();
  // Submit a proc and wait for its completion
  long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc, nonceGroup, nonce);

  // Restart
  restart();
  waitProcedure(procId);

  Procedure proc2 = new TestSingleStepProcedure();
  // Submit a procedure with the same nonce and expect the same procedure would return.
  long procId2 = ProcedureTestingUtility.submitAndWait(procExecutor, proc2, nonceGroup, nonce);
  assertTrue(procId == procId2);

  ProcedureInfo result = procExecutor.getResult(procId2);
  ProcedureTestingUtility.assertProcNotFailed(result);
}

@Test(timeout=30000)
public void testBadSubprocList() {
  List<String> state = new ArrayList<String>();
  Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
  Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2, NULL_PROC);
  Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
  long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);

  // subProc1 has a "null" subprocedure which is catched as InvalidArgument
  // failed state with 2 execute and 2 rollback
  LOG.info(state);
  ProcedureInfo result = procExecutor.getResult(rootId);
  assertTrue(state.toString(), result.isFailed());
  ProcedureTestingUtility.assertIsIllegalArgumentException(result);

  assertEquals(state.toString(), 4, state.size());
  assertEquals("rootProc-execute", state.get(0));
  assertEquals("subProc1-execute", state.get(1));
  assertEquals("subProc1-rollback", state.get(2));
  assertEquals("rootProc-rollback", state.get(3));
}

@Test(timeout=30000)
public void testSingleSequentialProcRollback() {
  List<String> state = new ArrayList<String>();
  Procedure subProc2 = new TestSequentialProcedure("subProc2", state,
                                                   new TestProcedureException("fail test"));
  Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
  Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
  long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);

  // the 3rd proc fail, rollback after 2 successful execution
  LOG.info(state);
  ProcedureInfo result = procExecutor.getResult(rootId);
  assertTrue(state.toString(), result.isFailed());
  LOG.info(result.getExceptionFullMessage());
  Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
  assertTrue("expected TestProcedureException, got " + cause,
    cause instanceof TestProcedureException);

  assertEquals(state.toString(), 6, state.size());
  assertEquals("rootProc-execute", state.get(0));
  assertEquals("subProc1-execute", state.get(1));
  assertEquals("subProc2-execute", state.get(2));
  assertEquals("subProc2-rollback", state.get(3));
  assertEquals("subProc1-rollback", state.get(4));
  assertEquals("rootProc-rollback", state.get(5));
}

@Test(timeout=30000)
public void testAbortTimeout() {
  final int PROC_TIMEOUT_MSEC = 2500;
  List<String> state = new ArrayList<String>();
  Procedure proc = new TestWaitingProcedure("wproc", state, false);
  proc.setTimeout(PROC_TIMEOUT_MSEC);
  long startTime = EnvironmentEdgeManager.currentTime();
  long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
  long execTime = EnvironmentEdgeManager.currentTime() - startTime;
  LOG.info(state);
  assertTrue("we didn't wait enough execTime=" + execTime, execTime >= PROC_TIMEOUT_MSEC);
  ProcedureInfo result = procExecutor.getResult(rootId);
  assertTrue(state.toString(), result.isFailed());
  ProcedureTestingUtility.assertIsTimeoutException(result);
  assertEquals(state.toString(), 2, state.size());
  assertEquals("wproc-execute", state.get(0));
  assertEquals("wproc-rollback", state.get(1));
}

@Test(timeout=30000)
public void testAbortTimeoutWithChildren() {
  List<String> state = new ArrayList<String>();
  Procedure proc = new TestWaitingProcedure("wproc", state, true);
  proc.setTimeout(2500);
  long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
  LOG.info(state);
  ProcedureInfo result = procExecutor.getResult(rootId);
  assertTrue(state.toString(), result.isFailed());
  ProcedureTestingUtility.assertIsTimeoutException(result);
  assertEquals(state.toString(), 4, state.size());
  assertEquals("wproc-execute", state.get(0));
  assertEquals("wproc-child-execute", state.get(1));
  assertEquals("wproc-child-rollback", state.get(2));
  assertEquals("wproc-rollback", state.get(3));
}

/**
 * List procedures
 * @return procedure list
 * @throws IOException
 */
@Override
public ProcedureInfo[] listProcedures() throws IOException {
  return
      executeCallable(new MasterCallable<ProcedureInfo[]>(getConnection()) {
        @Override
        public ProcedureInfo[] call(int callTimeout) throws Exception {
          PayloadCarryingRpcController controller = rpcControllerFactory.newController();
          controller.setCallTimeout(callTimeout);
          List<ProcedureProtos.Procedure> procList = master.listProcedures(
            controller, ListProceduresRequest.newBuilder().build()).getProcedureList();
          ProcedureInfo[] procInfoList = new ProcedureInfo[procList.size()];
          for (int i = 0; i < procList.size(); i++) {
            procInfoList[i] = ProcedureInfo.convert(procList.get(i));
          }
          return procInfoList;
        }
      });
}

public void postListProcedures(final List<ProcedureInfo> procInfoList) throws IOException {
  execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
    @Override
    public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
        throws IOException {
      oserver.postListProcedures(ctx, procInfoList);
    }
  });
}

@Override
public List<ProcedureInfo> listProcedures() throws IOException {
  if (cpHost != null) {
    cpHost.preListProcedures();
  }

  final List<ProcedureInfo> procInfoList = this.procedureExecutor.listProcedures();

  if (cpHost != null) {
    cpHost.postListProcedures(procInfoList);
  }

  return procInfoList;
}

@Override
public GetProcedureResultResponse getProcedureResult(RpcController controller,
    GetProcedureResultRequest request) throws ServiceException {
  LOG.debug("Checking to see if procedure is done procId=" + request.getProcId());
  try {
    master.checkInitialized();
    GetProcedureResultResponse.Builder builder = GetProcedureResultResponse.newBuilder();

    Pair<ProcedureInfo, Procedure> v = master.getMasterProcedureExecutor()
        .getResultOrProcedure(request.getProcId());
    if (v.getFirst() != null) {
      ProcedureInfo result = v.getFirst();
      builder.setState(GetProcedureResultResponse.State.FINISHED);
      builder.setStartTime(result.getStartTime());
      builder.setLastUpdate(result.getLastUpdate());
      if (result.isFailed()) {
        builder.setException(result.getForeignExceptionMessage());
      }
      if (result.hasResultData()) {
        builder.setResult(ByteStringer.wrap(result.getResult()));
      }
      master.getMasterProcedureExecutor().removeResult(request.getProcId());
    } else {
      Procedure proc = v.getSecond();
      if (proc == null) {
        builder.setState(GetProcedureResultResponse.State.NOT_FOUND);
      } else {
        builder.setState(GetProcedureResultResponse.State.RUNNING);
        builder.setStartTime(proc.getStartTime());
        builder.setLastUpdate(proc.getLastUpdate());
      }
    }
    return builder.build();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
}

@Override
public ListProceduresResponse listProcedures(
    RpcController rpcController,
    ListProceduresRequest request) throws ServiceException {
  try {
    ListProceduresResponse.Builder response =
        ListProceduresResponse.newBuilder();
    for(ProcedureInfo p: master.listProcedures()) {
      response.addProcedure(ProcedureInfo.convertToProcedureProto(p));
    }
    return response.build();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
}

@Test
public void testListProcedures() throws Exception {
  final TableName tableName = TableName.valueOf("testAbortProcedure");
  final ProcedureExecutor<MasterProcedureEnv> procExec =
      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
  Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName);
  proc.setOwner(USER_OWNER.getShortName());
  final long procId = procExec.submitProcedure(proc);
  final List<ProcedureInfo> procInfoList = procExec.listProcedures();

  AccessTestAction listProceduresAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      List<ProcedureInfo> procInfoListClone = new ArrayList<ProcedureInfo>(procInfoList.size());
      for(ProcedureInfo pi : procInfoList) {
        procInfoListClone.add(pi.clone());
      }
      ACCESS_CONTROLLER
      .postListProcedures(ObserverContext.createAndPrepare(CP_ENV, null), procInfoListClone);
     return null;
    }
  };

  verifyAllowed(listProceduresAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN);
  verifyAllowed(listProceduresAction, USER_OWNER);
  verifyIfNull(
    listProceduresAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}

@Test(timeout=60000)
public void testDeleteDeletedTable() throws Exception {
  final TableName tableName = TableName.valueOf("testDeleteDeletedTable");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
    procExec, tableName, null, "f");
  UTIL.getHBaseAdmin().disableTable(tableName);

  // delete the table (that exists)
  long procId1 = procExec.submitProcedure(
      new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
  // delete the table (that will no longer exist)
  long procId2 = procExec.submitProcedure(
      new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup + 1, nonce + 1);

  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.waitProcedure(procExec, procId2);

  // First delete should succeed
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
  MasterProcedureTestingUtility.validateTableDeletion(
    UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

  // Second delete should fail with TableNotFound
  ProcedureInfo result = procExec.getResult(procId2);
  assertTrue(result.isFailed());
  LOG.debug("Delete failed with exception: " + result.getExceptionFullMessage());
  assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotFoundException);
}

@Test(timeout=60000)
public void testModifyNonExistingColumnFamily() throws Exception {
  final TableName tableName = TableName.valueOf("testModifyExistingColumnFamily");
  final String cf2 = "cf2";
  final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf2);
  int oldBlockSize = columnDescriptor.getBlocksize();
  int newBlockSize = 2 * oldBlockSize;

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");

  // Modify the column family that does not exist
  columnDescriptor.setBlocksize(newBlockSize);
  long procId1 = procExec.submitProcedure(
    new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup,
    nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);

  ProcedureInfo result = procExec.getResult(procId1);
  assertTrue(result.isFailed());
  LOG.debug("Modify failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
}

@Test(timeout=60000)
public void testTruncateNotExistentTable() throws Exception {
  final TableName tableName = TableName.valueOf("testTruncateNotExistentTable");

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  long procId = ProcedureTestingUtility.submitAndWait(procExec,
      new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));

  // Second delete should fail with TableNotFound
  ProcedureInfo result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Truncate failed with exception: " + result.getExceptionFullMessage());
  assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotFoundException);
}

@Test(timeout=60000, expected=TableNotDisabledException.class)
public void testEnableNonDisabledTable() throws Exception {
  final TableName tableName = TableName.valueOf("testEnableNonExistingTable");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // Enable the table - expect failure
  long procId1 = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  ProcedureTestingUtility.waitProcedure(procExec, procId1);

  ProcedureInfo result = procExec.getResult(procId1);
  assertTrue(result.isFailed());
  LOG.debug("Enable failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotDisabledException);

  // Enable the table with skipping table state check flag (simulate recovery scenario)
  long procId2 = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, true),
      nonceGroup + 1,
      nonce + 1);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);

  // Enable the table - expect failure from ProcedurePrepareLatch
  final ProcedurePrepareLatch prepareLatch = new ProcedurePrepareLatch.CompatibilityLatch();
  long procId3 = procExec.submitProcedure(
      new EnableTableProcedure(procExec.getEnvironment(), tableName, false, prepareLatch),
      nonceGroup + 2,
      nonce + 2);
  prepareLatch.await();
  Assert.fail("Enable should throw exception through latch.");
}

public CompletedProcedureCleaner(final Configuration conf, final ProcedureStore store,
    final Map<Long, ProcedureInfo> completedMap,
    final Map<NonceKey, Long> nonceKeysToProcIdsMap) {
  // set the timeout interval that triggers the periodic-procedure
  setTimeout(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));
  this.completed = completedMap;
  this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;
  this.store = store;
  this.conf = conf;
}

public void periodicExecute(final TEnvironment env) {
  if (completed.isEmpty()) {
    if (LOG.isTraceEnabled()) {
      LOG.trace("No completed procedures to cleanup.");
    }
    return;
  }

  final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);
  final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);

  final long now = EnvironmentEdgeManager.currentTime();
  final Iterator<Map.Entry<Long, ProcedureInfo>> it = completed.entrySet().iterator();
  final boolean isDebugEnabled = LOG.isDebugEnabled();
  while (it.hasNext() && store.isRunning()) {
    final Map.Entry<Long, ProcedureInfo> entry = it.next();
    final ProcedureInfo procInfo = entry.getValue();

    // TODO: Select TTL based on Procedure type
    if ((procInfo.hasClientAckTime() && (now - procInfo.getClientAckTime()) >= evictAckTtl) ||
        (now - procInfo.getLastUpdate()) >= evictTtl) {
      if (isDebugEnabled) {
        LOG.debug("Evict completed procedure: " + procInfo);
      }
      store.delete(entry.getKey());
      it.remove();

      NonceKey nonceKey = procInfo.getNonceKey();
      if (nonceKey != null) {
        nonceKeysToProcIdsMap.remove(nonceKey);
      }
    }
  }
}

public Pair<ProcedureInfo, Procedure> getResultOrProcedure(final long procId) {
  ProcedureInfo result = completed.get(procId);
  Procedure proc = null;
  if (result == null) {
    proc = procedures.get(procId);
    if (proc == null) {
      result = completed.get(procId);
    }
  }
  return new Pair(result, proc);
}

@Override
public ProcedureInfo nextAsProcedureInfo() {
  try {
    return current.convertToInfo();
  } finally {
    current = current.replayNext;
  }
}

@Test(timeout=30000)
public void testMultiStepProcRecovery() throws Exception {
  // Step 0 - kill
  Procedure proc = new TestMultiStepProcedure();
  long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
  assertFalse(procExecutor.isRunning());

  // Step 0 exec && Step 1 - kill
  restart();
  waitProcedure(procId);
  ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
  assertFalse(procExecutor.isRunning());

  // Step 1 exec && step 2 - kill
  restart();
  waitProcedure(procId);
  ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
  assertFalse(procExecutor.isRunning());

  // Step 2 exec
  restart();
  waitProcedure(procId);
  assertTrue(procExecutor.isRunning());

  // The procedure is completed
  ProcedureInfo result = procExecutor.getResult(procId);
  ProcedureTestingUtility.assertProcNotFailed(result);
}

@Test(timeout=30000)
public void testStateMachineRecovery() throws Exception {
  ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
  ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);

  // Step 1 - kill
  Procedure proc = new TestStateMachineProcedure();
  long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
  assertFalse(procExecutor.isRunning());

  // Step 1 exec && Step 2 - kill
  restart();
  waitProcedure(procId);
  ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
  assertFalse(procExecutor.isRunning());

  // Step 2 exec && step 3 - kill
  restart();
  waitProcedure(procId);
  ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
  assertFalse(procExecutor.isRunning());

  // Step 3 exec
  restart();
  waitProcedure(procId);
  assertTrue(procExecutor.isRunning());

  // The procedure is completed
  ProcedureInfo result = procExecutor.getResult(procId);
  ProcedureTestingUtility.assertProcNotFailed(result);
  assertEquals(15, Bytes.toInt(result.getResult()));
}

@Test(timeout=30000)
public void testSingleSequentialProc() {
  List<String> state = new ArrayList<String>();
  Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
  Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
  Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
  long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);

  // successful state, with 3 execute
  LOG.info(state);
  ProcedureInfo result = procExecutor.getResult(rootId);
  ProcedureTestingUtility.assertProcNotFailed(result);
  assertEquals(state.toString(), 3, state.size());
}

@Test(timeout=30000)
public void testRollbackRetriableFailure() {
  long procId = ProcedureTestingUtility.submitAndWait(procExecutor, new TestFaultyRollback());

  ProcedureInfo result = procExecutor.getResult(procId);
  assertTrue("expected a failure", result.isFailed());
  LOG.info(result.getExceptionFullMessage());
  Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
  assertTrue("expected TestProcedureException, got " + cause,
    cause instanceof TestProcedureException);
}

public static void assertIsAbortException(final ProcedureInfo result) {
  assertEquals(true, result.isFailed());
  LOG.info(result.getExceptionFullMessage());
  Throwable cause = getExceptionCause(result);
  assertTrue("expected abort exception, got " + cause,
    cause instanceof ProcedureAbortedException);
}

public static void assertIsIllegalArgumentException(final ProcedureInfo result) {
  assertEquals(true, result.isFailed());
  LOG.info(result.getExceptionFullMessage());
  Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
  assertTrue("expected IllegalArgumentIOException, got " + cause,
    cause instanceof IllegalArgumentIOException);
}

@Override
public void postListProcedures(
    ObserverContext<MasterCoprocessorEnvironment> ctx,
    List<ProcedureInfo> procInfoList) throws IOException {
}

@Override
public void postListProcedures(
    ObserverContext<MasterCoprocessorEnvironment> ctx,
    List<ProcedureInfo> procInfoList) throws IOException {
}

@Test(timeout = 300000)
public void testListProcedures() throws Exception {
  ProcedureInfo[] procList = admin.listProcedures();
  assertTrue(procList.length >= 0);
}

@Override
public List<ProcedureInfo> listProcedures() throws IOException {
  return null;  //To change body of implemented methods use File | Settings | File Templates.
}

@Test(timeout=60000)
public void testDeleteColumnFamilyTwice() throws Exception {
  final TableName tableName = TableName.valueOf("testDeleteColumnFamilyTwice");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  final String cf2 = "cf2";

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", cf2);

  // delete the column family that exists
  long procId1 = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
    nonceGroup,
    nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  // First delete should succeed
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);

  MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
    tableName, cf2);

  // delete the column family that does not exist
  long procId2 = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
    nonceGroup + 1,
    nonce + 1);

  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId2);

  // Second delete should fail with InvalidFamilyOperationException
  ProcedureInfo result = procExec.getResult(procId2);
  assertTrue(result.isFailed());
  LOG.debug("Delete online failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);

  // Try again, this time with table disabled.
  UTIL.getHBaseAdmin().disableTable(tableName);
  long procId3 = procExec.submitProcedure(
    new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
    nonceGroup + 2,
    nonce + 2);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId3);
  // Expect fail with InvalidFamilyOperationException
  result = procExec.getResult(procId2);
  assertTrue(result.isFailed());
  LOG.debug("Delete offline failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
}

@Test(timeout = 60000)
public void testDisableTableMultipleTimes() throws Exception {
  final TableName tableName = TableName.valueOf("testDisableTableMultipleTimes");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");

  // Disable the table
  long procId1 = procExec.submitProcedure(new DisableTableProcedure(
      procExec.getEnvironment(), tableName, false), nonceGroup, nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);

  // Disable the table again - expect failure
  long procId2 = procExec.submitProcedure(new DisableTableProcedure(
      procExec.getEnvironment(), tableName, false), nonceGroup + 1, nonce + 1);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId2);
  ProcedureInfo result = procExec.getResult(procId2);
  assertTrue(result.isFailed());
  LOG.debug("Disable failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableNotEnabledException);

  // Disable the table - expect failure from ProcedurePrepareLatch
  try {
    final ProcedurePrepareLatch prepareLatch = new ProcedurePrepareLatch.CompatibilityLatch();

    long procId3 = procExec.submitProcedure(new DisableTableProcedure(
        procExec.getEnvironment(), tableName, false, prepareLatch), nonceGroup + 2, nonce + 2);
    prepareLatch.await();
    Assert.fail("Disable should throw exception through latch.");
  } catch (TableNotEnabledException tnee) {
    // Expected
    LOG.debug("Disable failed with expected exception.");
  }

  // Disable the table again with skipping table state check flag (simulate recovery scenario)
  long procId4 = procExec.submitProcedure(new DisableTableProcedure(
      procExec.getEnvironment(), tableName, true));
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId4);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId4);
  MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
    tableName);
}

@Test(timeout=60000)
public void testAddSameColumnFamilyTwice() throws Exception {
  final TableName tableName = TableName.valueOf("testAddColumnFamilyTwice");
  final String cf2 = "cf2";
  final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf2);

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");

  // add the column family
  long procId1 = procExec.submitProcedure(
    new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup,
    nonce);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId1);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
  MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
    tableName, cf2);

  // add the column family that exists
  long procId2 = procExec.submitProcedure(
    new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup + 1,
    nonce + 1);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId2);

  // Second add should fail with InvalidFamilyOperationException
  ProcedureInfo result = procExec.getResult(procId2);
  assertTrue(result.isFailed());
  LOG.debug("Add failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);

  // Do the same add the existing column family - this time offline
  UTIL.getHBaseAdmin().disableTable(tableName);
  long procId3 = procExec.submitProcedure(
    new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
    nonceGroup + 2,
    nonce + 2);
  // Wait the completion
  ProcedureTestingUtility.waitProcedure(procExec, procId3);

  // Second add should fail with InvalidFamilyOperationException
  result = procExec.getResult(procId3);
  assertTrue(result.isFailed());
  LOG.debug("Add failed with exception: " + result.getExceptionFullMessage());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
}