@Override public void start(CoprocessorEnvironment env) { this.env = (RegionCoprocessorEnvironment)env; random = new SecureRandom(); conf = env.getConfiguration(); baseStagingDir = SecureBulkLoadUtil.getBaseStagingDir(conf); this.userProvider = UserProvider.instantiate(conf); try { fs = FileSystem.get(conf); fs.mkdirs(baseStagingDir, PERM_HIDDEN); fs.setPermission(baseStagingDir, PERM_HIDDEN); //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN); FileStatus status = fs.getFileStatus(baseStagingDir); if(status == null) { throw new IllegalStateException("Failed to create staging directory"); } if(!status.getPermission().equals(PERM_HIDDEN)) { throw new IllegalStateException( "Directory already exists but permissions aren't set to '-rwx--x--x' "); } } catch (IOException e) { throw new IllegalStateException("Failed to get FileSystem instance",e); } }
private void runTest(String testName, HTableDescriptor htd, BloomType bloomType, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception { for (boolean managed : new boolean[] { true, false }) { Path dir = util.getDataTestDirOnTestFS(testName); FileSystem fs = util.getTestFileSystem(); dir = dir.makeQualified(fs); Path familyDir = new Path(dir, Bytes.toString(FAMILY)); int hfileIdx = 0; for (byte[][] range : hfileRanges) { byte[] from = range[0]; byte[] to = range[1]; HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000); } int expectedRows = hfileIdx * 1000; if (preCreateTable) { util.getHBaseAdmin().createTable(htd, tableSplitKeys); } final TableName tableName = htd.getTableName(); if (!util.getHBaseAdmin().tableExists(tableName)) { util.getHBaseAdmin().createTable(htd); } LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration()); if (managed) { try (HTable table = new HTable(util.getConfiguration(), tableName)) { loader.doBulkLoad(dir, table); assertEquals(expectedRows, util.countRows(table)); } } else { try (Connection conn = ConnectionFactory.createConnection(util.getConfiguration()); HTable table = (HTable) conn.getTable(tableName)) { loader.doBulkLoad(dir, table); } } // verify staging folder has been cleaned up Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration()); if (fs.exists(stagingBasePath)) { FileStatus[] files = fs.listStatus(stagingBasePath); for (FileStatus file : files) { assertTrue("Folder=" + file.getPath() + " is not cleaned up.", file.getPath().getName() != "DONOTERASE"); } } util.deleteTable(tableName); } }
public Path getStagingPath(String bulkToken, byte[] family) throws IOException { return SecureBulkLoadUtil.getStagingPath(table.getConfiguration(), bulkToken, family); }
private void runTest(String testName, HTableDescriptor htd, BloomType bloomType, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception { Path dir = util.getDataTestDirOnTestFS(testName); FileSystem fs = util.getTestFileSystem(); dir = dir.makeQualified(fs); Path familyDir = new Path(dir, Bytes.toString(FAMILY)); int hfileIdx = 0; for (byte[][] range : hfileRanges) { byte[] from = range[0]; byte[] to = range[1]; HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_" + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000); } int expectedRows = hfileIdx * 1000; if (preCreateTable) { util.getHBaseAdmin().createTable(htd, tableSplitKeys); } final TableName tableName = htd.getTableName(); LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration()); String [] args= {dir.toString(), tableName.toString()}; loader.run(args); Table table = new HTable(util.getConfiguration(), tableName); try { assertEquals(expectedRows, util.countRows(table)); } finally { table.close(); } // verify staging folder has been cleaned up Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration()); if(fs.exists(stagingBasePath)) { FileStatus[] files = fs.listStatus(stagingBasePath); for(FileStatus file : files) { assertTrue("Folder=" + file.getPath() + " is not cleaned up.", file.getPath().getName() != "DONOTERASE"); } } util.deleteTable(tableName); }