Java 类org.apache.hadoop.hbase.snapshot.RestoreSnapshotException 实例源码

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
 * <p>
 * Only a single snapshot should be restored at a time, or results may be undefined.
 * @param snapshot snapshot to restore
 * @return response from the server indicating the max time to wait for the snapshot
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
    throws IOException, RestoreSnapshotException {
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();

  // run the snapshot restore on the master
  return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
    @Override
    public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
      PayloadCarryingRpcController controller = rpcControllerFactory.newController();
      controller.setCallTimeout(callTimeout);
      return master.restoreSnapshot(controller, request);
    }
  });
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
 * <p>
 * Only a single snapshot should be restored at a time, or results may be undefined.
 * @param snapshot snapshot to restore
 * @return response from the server indicating the max time to wait for the snapshot
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
    throws IOException, RestoreSnapshotException {
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();

  // run the snapshot restore on the master
  return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
    @Override
    public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
      return master.restoreSnapshot(null, request);
    }
  });
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
 * <p>
 * Only a single snapshot should be restored at a time, or results may be undefined.
 * @param snapshot snapshot to restore
 * @return response from the server indicating the max time to wait for the snapshot
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
    throws IOException, RestoreSnapshotException {
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();

  // run the snapshot restore on the master
  return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
    @Override
    public RestoreSnapshotResponse call() throws ServiceException {
      return master.restoreSnapshot(null, request);
    }
  });
}

private TableName getTableNameBeforeRestoreSnapshot(final String snapshotName)
    throws IOException, RestoreSnapshotException {
  TableName tableName = null;
  for (SnapshotDescription snapshotInfo: listSnapshots()) {
    if (snapshotInfo.getName().equals(snapshotName)) {
      tableName = snapshotInfo.getTableName();
      break;
    }
  }

  if (tableName == null) {
    throw new RestoreSnapshotException(
      "Unable to find the table name for snapshot=" + snapshotName);
  }
  return tableName;
}

@Override
public Future<Void> restoreSnapshotAsync(final String snapshotName)
    throws IOException, RestoreSnapshotException {
  TableName tableName = getTableNameBeforeRestoreSnapshot(snapshotName);

  // The table does not exists, switch to clone.
  if (!tableExists(tableName)) {
    return cloneSnapshotAsync(snapshotName, tableName);
  }

  // Check if the table is disabled
  if (!isTableDisabled(tableName)) {
    throw new TableNotDisabledException(tableName);
  }

  return internalRestoreSnapshotAsync(snapshotName, tableName, false);
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
 * <p>
 * Only a single snapshot should be restored at a time, or results may be undefined.
 * @param snapshot snapshot to restore
 * @return response from the server indicating the max time to wait for the snapshot
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
    throws IOException, RestoreSnapshotException {
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();

  // run the snapshot restore on the master
  return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
    @Override
    public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
      return master.restoreSnapshot(null, request);
    }
  });
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
 * <p>
 * Only a single snapshot should be restored at a time, or results may be undefined.
 * @param snapshot snapshot to restore
 * @return response from the server indicating the max time to wait for the snapshot
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
    throws IOException, RestoreSnapshotException {
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
      .build();

  // run the snapshot restore on the master
  return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
    @Override
    public RestoreSnapshotResponse call() throws ServiceException {
      return master.restoreSnapshot(null, request);
    }
  });
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

@Test
public void testRestoreSnapshotQuotaExceed() throws Exception {
  String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
  NamespaceDescriptor nspDesc =
      NamespaceDescriptor.create(nsp)
          .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
  ADMIN.createNamespace(nspDesc);
  NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
  assertNotNull("Namespace descriptor found null.", ndesc);
  TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
  HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
  ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);

  NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
  assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());

  String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
  // snapshot has 4 regions
  ADMIN.snapshot(snapshot, tableName1);
  // recreate table with 1 region and set max regions to 3 for namespace
  ADMIN.disableTable(tableName1);
  ADMIN.deleteTable(tableName1);
  ADMIN.createTable(tableDescOne);
  ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
  ADMIN.modifyNamespace(ndesc);

  ADMIN.disableTable(tableName1);
  try {
    ADMIN.restoreSnapshot(snapshot);
    fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
        + " wraps IOException into RestoreSnapshotException");
  } catch (RestoreSnapshotException ignore) {
    assertTrue(ignore.getCause() instanceof QuotaExceededException);
  }
  assertEquals(1, getNamespaceState(nsp).getRegionCount());
  ADMIN.enableTable(tableName1);
  ADMIN.deleteSnapshot(snapshot);
}

@Override
public void restoreSnapshot(byte[] name)
        throws IOException, RestoreSnapshotException, UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.restoreSnapshot(name);
}

@Override
public void restoreSnapshot(String string)
        throws IOException, RestoreSnapshotException, UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.restoreSnapshot(string);
}

@Override
public void restoreSnapshot(byte[] name, boolean bln)
        throws IOException, RestoreSnapshotException, UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.restoreSnapshot(name, bln);
}

@Override
public void restoreSnapshot(String string, boolean bln)
        throws IOException, RestoreSnapshotException, UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.restoreSnapshot(string, bln);
}

@Override
public void cloneSnapshot(byte[] name, TableName tn)
        throws IOException, TableExistsException, RestoreSnapshotException,
        UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.cloneSnapshot(name, tn);
}

@Override
public void cloneSnapshot(String string, TableName tn)
        throws IOException, TableExistsException, RestoreSnapshotException,
        UnsupportedOperationException {
  if (repository.isActivated()) {
    throw new UnsupportedOperationException(UNSUPPORTED_WHEN_COLMANAGER_ACTIVATED_MSG);
  }

  wrappedHbaseAdmin.cloneSnapshot(string, tn);
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
 * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
 * create an HTable instance to this table before it is available.
 * @param snapshotName snapshot to restore
 * @param tableName table name to restore the snapshot on
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private void internalRestoreSnapshot(final String snapshotName, final TableName
    tableName)
    throws IOException, RestoreSnapshotException {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder()
      .setName(snapshotName).setTable(tableName.getNameAsString()).build();

  // actually restore the snapshot
  internalRestoreSnapshotAsync(snapshot);

  final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
      .setSnapshot(snapshot).build();
  IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
      .setDone(false).buildPartial();
  final long maxPauseTime = 5000;
  int tries = 0;
  while (!done.getDone()) {
    try {
      // sleep a backoff <= pauseTime amount
      long sleep = getPauseTime(tries++);
      sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
      LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
      Thread.sleep(sleep);
    } catch (InterruptedException e) {
      throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
    }
    LOG.debug("Getting current status of snapshot restore from master...");
    done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
        getConnection()) {
      @Override
      public IsRestoreSnapshotDoneResponse call(int callTimeout) throws ServiceException {
        return master.isRestoreSnapshotDone(null, request);
      }
    });
  }
  if (!done.getDone()) {
    throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
 * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
 * create an HTable instance to this table before it is available.
 * @param snapshotName snapshot to restore
 * @param tableName table name to restore the snapshot on
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private void internalRestoreSnapshot(final String snapshotName, final TableName
    tableName)
    throws IOException, RestoreSnapshotException {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder()
      .setName(snapshotName).setTable(tableName.getNameAsString()).build();

  // actually restore the snapshot
  internalRestoreSnapshotAsync(snapshot);

  final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
      .setSnapshot(snapshot).build();
  IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
      .setDone(false).buildPartial();
  final long maxPauseTime = 5000;
  int tries = 0;
  while (!done.getDone()) {
    try {
      // sleep a backoff <= pauseTime amount
      long sleep = getPauseTime(tries++);
      sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
      LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
      Thread.sleep(sleep);
    } catch (InterruptedException e) {
      LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
      Thread.currentThread().interrupt();
    }
    LOG.debug("Getting current status of snapshot restore from master...");
    done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
        getConnection()) {
      @Override
      public IsRestoreSnapshotDoneResponse call() throws ServiceException {
        return master.isRestoreSnapshotDone(null, request);
      }
    });
  }
  if (!done.getDone()) {
    throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param tableDescriptor Table Descriptor of the table to create
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @return procId the ID of the clone snapshot procedure
 */
synchronized long cloneSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl)
    throws HBaseSnapshotException {
  TableName tableName = tableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    long procId = master.getMasterProcedureExecutor().submitProcedure(
      new CloneSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(),
              tableDescriptor, snapshot, restoreAcl),
      nonceKey);
    this.restoreTableToProcIdMap.put(tableName, procId);
    return procId;
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot="
      + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot. The restore will fail if the destination table has a snapshot
 * or restore in progress.
 * @param snapshot Snapshot Descriptor
 * @param tableDescriptor Table Descriptor
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @param restoreAcl true to restore acl of snapshot
 * @return procId the ID of the restore snapshot procedure
 */
private synchronized long restoreSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor tableDescriptor, final NonceKey nonceKey, final boolean restoreAcl)
    throws HBaseSnapshotException {
  final TableName tableName = tableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    long procId = master.getMasterProcedureExecutor().submitProcedure(
      new RestoreSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(),
              tableDescriptor, snapshot, restoreAcl),
      nonceKey);
    this.restoreTableToProcIdMap.put(tableName, procId);
    return procId;
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

@Override
public void restoreSnapshot(final String snapshotName)
    throws IOException, RestoreSnapshotException {
  boolean takeFailSafeSnapshot =
      conf.getBoolean(HConstants.SNAPSHOT_RESTORE_TAKE_FAILSAFE_SNAPSHOT,
        HConstants.DEFAULT_SNAPSHOT_RESTORE_TAKE_FAILSAFE_SNAPSHOT);
  restoreSnapshot(snapshotName, takeFailSafeSnapshot);
}

@Override
public void cloneSnapshot(String snapshotName, TableName tableName, boolean restoreAcl)
    throws IOException, TableExistsException, RestoreSnapshotException {
  if (tableExists(tableName)) {
    throw new TableExistsException(tableName);
  }
  get(
    internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl),
    Integer.MAX_VALUE,
    TimeUnit.MILLISECONDS);
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
 * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
 * create an HTable instance to this table before it is available.
 * @param snapshotName snapshot to restore
 * @param tableName table name to restore the snapshot on
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private Future<Void> internalRestoreSnapshotAsync(final String snapshotName,
    final TableName tableName, final boolean restoreAcl)
    throws IOException, RestoreSnapshotException {
  final SnapshotProtos.SnapshotDescription snapshot =
      SnapshotProtos.SnapshotDescription.newBuilder()
      .setName(snapshotName).setTable(tableName.getNameAsString()).build();

  // actually restore the snapshot
  ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

  RestoreSnapshotResponse response = executeCallable(
      new MasterCallable<RestoreSnapshotResponse>(getConnection(), getRpcControllerFactory()) {
    @Override
    protected RestoreSnapshotResponse rpcCall() throws Exception {
      final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder()
          .setSnapshot(snapshot)
          .setNonceGroup(ng.getNonceGroup())
          .setNonce(ng.newNonce())
          .setRestoreACL(restoreAcl)
          .build();
      return master.restoreSnapshot(getRpcController(), request);
    }
  });

  return new RestoreSnapshotFuture(this, snapshot, tableName, response);
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
 * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
 * create an HTable instance to this table before it is available.
 * @param snapshotName snapshot to restore
 * @param tableName table name to restore the snapshot on
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private void internalRestoreSnapshot(final String snapshotName, final TableName
    tableName)
    throws IOException, RestoreSnapshotException {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder()
      .setName(snapshotName).setTable(tableName.getNameAsString()).build();

  // actually restore the snapshot
  internalRestoreSnapshotAsync(snapshot);

  final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
      .setSnapshot(snapshot).build();
  IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
      .setDone(false).buildPartial();
  final long maxPauseTime = 5000;
  int tries = 0;
  while (!done.getDone()) {
    try {
      // sleep a backoff <= pauseTime amount
      long sleep = getPauseTime(tries++);
      sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
      LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
      Thread.sleep(sleep);
    } catch (InterruptedException e) {
      throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
    }
    LOG.debug("Getting current status of snapshot restore from master...");
    done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
        getConnection()) {
      @Override
      public IsRestoreSnapshotDoneResponse call(int callTimeout) throws ServiceException {
        return master.isRestoreSnapshotDone(null, request);
      }
    });
  }
  if (!done.getDone()) {
    throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  TableName tableName = hTableDescriptor.getTableName();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
    this.executorService.submit(handler);
    restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
 * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
 * create an HTable instance to this table before it is available.
 * @param snapshotName snapshot to restore
 * @param tableName table name to restore the snapshot on
 * @throws IOException if a remote or network exception occurs
 * @throws RestoreSnapshotException if snapshot failed to be restored
 * @throws IllegalArgumentException if the restore request is formatted incorrectly
 */
private void internalRestoreSnapshot(final String snapshotName, final TableName
    tableName)
    throws IOException, RestoreSnapshotException {
  SnapshotDescription snapshot = SnapshotDescription.newBuilder()
      .setName(snapshotName).setTable(tableName.getNameAsString()).build();

  // actually restore the snapshot
  internalRestoreSnapshotAsync(snapshot);

  final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
      .setSnapshot(snapshot).build();
  IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
      .setDone(false).buildPartial();
  final long maxPauseTime = 5000;
  int tries = 0;
  while (!done.getDone()) {
    try {
      // sleep a backoff <= pauseTime amount
      long sleep = getPauseTime(tries++);
      sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
      LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
      Thread.sleep(sleep);
    } catch (InterruptedException e) {
      LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
      Thread.currentThread().interrupt();
    }
    LOG.debug("Getting current status of snapshot restore from master...");
    done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
        getConnection()) {
      @Override
      public IsRestoreSnapshotDoneResponse call() throws ServiceException {
        return master.isRestoreSnapshotDone(null, request);
      }
    });
  }
  if (!done.getDone()) {
    throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Restore the specified snapshot.
 * The restore will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor
 */
private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    RestoreSnapshotHandler handler =
      new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    restoreHandlers.put(hTableDescriptor.getNameAsString(), handler);
  } catch (Exception e) {
    String msg = "Couldn't restore the snapshot=" + SnapshotDescriptionUtils.toString(
        snapshot)  +
        " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}

/**
 * Clone the specified snapshot into a new table.
 * The operation will fail if the destination table has a snapshot or restore in progress.
 *
 * @param snapshot Snapshot Descriptor
 * @param hTableDescriptor Table Descriptor of the table to create
 */
synchronized void cloneSnapshot(final SnapshotDescription snapshot,
    final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
  String tableName = hTableDescriptor.getNameAsString();

  // make sure we aren't running a snapshot on the same table
  if (isTakingSnapshot(tableName)) {
    throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
  }

  // make sure we aren't running a restore on the same table
  if (isRestoringTable(tableName)) {
    throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
  }

  try {
    CloneSnapshotHandler handler =
      new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster);
    this.executorService.submit(handler);
    this.restoreHandlers.put(tableName, handler);
  } catch (Exception e) {
    String msg = "Couldn't clone the snapshot=" + SnapshotDescriptionUtils.toString(snapshot) +
      " on table=" + tableName;
    LOG.error(msg, e);
    throw new RestoreSnapshotException(msg, e);
  }
}