/** * Execute compaction, using a Map-Reduce job. */ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, final boolean compactOnce, final boolean major) throws Exception { Configuration conf = getConf(); conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); conf.setBoolean(CONF_COMPACT_MAJOR, major); Job job = new Job(conf); job.setJobName("CompactionTool"); job.setJarByClass(CompactionTool.class); job.setMapperClass(CompactionMapper.class); job.setInputFormatClass(CompactionInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); // add dependencies (including HBase ones) TableMapReduceUtil.addDependencyJars(job); Path stagingDir = JobUtil.getStagingDir(conf); try { // Create input file with the store dirs Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTime()); CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); CompactionInputFormat.addInputPath(job, inputPath); // Initialize credential for secure cluster TableMapReduceUtil.initCredentials(job); // Start the MR Job and wait return job.waitForCompletion(true) ? 0 : 1; } finally { fs.delete(stagingDir, true); } }
/** * Execute compaction, using a Map-Reduce job. */ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, final boolean compactOnce, final boolean major) throws Exception { Configuration conf = getConf(); conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); conf.setBoolean(CONF_COMPACT_MAJOR, major); Job job = new Job(conf); job.setJobName("CompactionTool"); job.setJarByClass(CompactionTool.class); job.setMapperClass(CompactionMapper.class); job.setInputFormatClass(CompactionInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); // add dependencies (including HBase ones) TableMapReduceUtil.addDependencyJars(job); // This job instantiates HRegions, which requires the Counter class from the high_scale library TableMapReduceUtil.addDependencyJars(job.getConfiguration(), org.cliffc.high_scale_lib.Counter.class); Path stagingDir = JobUtil.getStagingDir(conf); try { // Create input file with the store dirs Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTimeMillis()); CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); CompactionInputFormat.addInputPath(job, inputPath); // Initialize credential for secure cluster TableMapReduceUtil.initCredentials(job); // Start the MR Job and wait return job.waitForCompletion(true) ? 0 : 1; } finally { fs.delete(stagingDir, true); } }
/** * Execute compaction, using a Map-Reduce job. */ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, final boolean compactOnce, final boolean major) throws Exception { Configuration conf = getConf(); conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); conf.setBoolean(CONF_COMPACT_MAJOR, major); Job job = new Job(conf); job.setJobName("CompactionTool"); job.setJarByClass(CompactionTool.class); job.setMapperClass(CompactionMapper.class); job.setInputFormatClass(CompactionInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); // add dependencies (including HBase ones) TableMapReduceUtil.addDependencyJars(job); Path stagingDir = JobUtil.getStagingDir(conf); try { // Create input file with the store dirs Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTimeMillis()); CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); CompactionInputFormat.addInputPath(job, inputPath); // Initialize credential for secure cluster TableMapReduceUtil.initCredentials(job); // Start the MR Job and wait return job.waitForCompletion(true) ? 0 : 1; } finally { fs.delete(stagingDir, true); } }
/** * Execute compaction, using a Map-Reduce job. */ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, final boolean compactOnce, final boolean major) throws Exception { Configuration conf = getConf(); conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); conf.setBoolean(CONF_COMPACT_MAJOR, major); Job job = new Job(conf); job.setJobName("CompactionTool"); job.setJarByClass(CompactionTool.class); job.setMapperClass(CompactionMapper.class); job.setInputFormatClass(CompactionInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); Path stagingDir = JobUtil.getStagingDir(conf); try { // Create input file with the store dirs Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTimeMillis()); CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); CompactionInputFormat.addInputPath(job, inputPath); // Initialize credential for secure cluster TableMapReduceUtil.initCredentials(job); // Start the MR Job and wait return job.waitForCompletion(true) ? 0 : 1; } finally { fs.delete(stagingDir, true); } }
/** * Execute compaction, using a Map-Reduce job. */ private int doMapReduce(final FileSystem fs, final Set<Path> toCompactDirs, final boolean compactOnce) throws Exception { Configuration conf = getConf(); conf.setBoolean(CONF_COMPACT_ONCE, compactOnce); Job job = new Job(conf); job.setJobName("CompactionTool"); job.setJarByClass(CompactionTool.class); job.setMapperClass(CompactionMapper.class); job.setInputFormatClass(CompactionInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); Path stagingDir = JobUtil.getStagingDir(conf); try { // Create input file with the store dirs Path inputPath = new Path(stagingDir, "compact-"+ EnvironmentEdgeManager.currentTimeMillis()); CompactionInputFormat.createInputFile(fs, inputPath, toCompactDirs); CompactionInputFormat.addInputPath(job, inputPath); // Initialize credential for secure cluster TableMapReduceUtil.initCredentials(job); // Start the MR Job and wait return job.waitForCompletion(true) ? 0 : 1; } finally { fs.delete(stagingDir, true); } }
private static Path getInputFolderPath(Configuration conf) throws IOException, InterruptedException { Path stagingDir = JobUtil.getStagingDir(conf); return new Path(stagingDir, INPUT_FOLDER_PREFIX + String.valueOf(EnvironmentEdgeManager.currentTimeMillis())); }
