/** * Configure a waiting job */ static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException { jobConf.setJobName(jobName); jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); jobConf.setInputFormat(RandomInputFormat.class); jobConf.setNumMapTasks(numMaps); jobConf.setNumReduceTasks(numRed); jobConf.setJar("build/test/mapred/testjar/testjob.jar"); jobConf.set(getTaskSignalParameter(true), mapSignalFilename); jobConf.set(getTaskSignalParameter(false), redSignalFilename); }
public static RunningJob runJobSucceed(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-succeed"); conf.setMapperClass(IdentityMapper.class); conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); long sleepCount = 0; while (!job.isComplete()) { try { if (sleepCount > 300) { // 30 seconds throw new IOException("Job didn't finish in 30 seconds"); } Thread.sleep(100); sleepCount++; } catch (InterruptedException e) { break; } } return job; }
public static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-fail"); conf.setMapperClass(FailMapper.class); conf.setReducerClass(IdentityReducer.class); conf.setMaxMapAttempts(1); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); long sleepCount = 0; while (!job.isComplete()) { try { if (sleepCount > 300) { // 30 seconds throw new IOException("Job didn't finish in 30 seconds"); } Thread.sleep(100); sleepCount++; } catch (InterruptedException e) { break; } } return job; }
public void testEmptyJoin() throws Exception { JobConf job = new JobConf(); Path base = cluster.getFileSystem().makeQualified(new Path("/empty")); Path[] src = { new Path(base,"i0"), new Path("i1"), new Path("i2") }; job.set("mapreduce.join.expr", CompositeInputFormat.compose("outer", Fake_IF.class, src)); job.setInputFormat(CompositeInputFormat.class); FileOutputFormat.setOutputPath(job, new Path(base, "out")); job.setMapperClass(IdentityMapper.class); job.setReducerClass(IdentityReducer.class); job.setOutputKeyClass(IncomparableKey.class); job.setOutputValueClass(NullWritable.class); JobClient.runJob(job); base.getFileSystem(job).delete(base, true); }
static RunningJob launchJob(JobConf jobConf, Path inDir, Path outputPath, int numMaps, String jobName) throws IOException { jobConf.setJobName(jobName); jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setMapperClass(IdentityMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); jobConf.setNumMapTasks(numMaps); jobConf.setNumReduceTasks(0); jobConf.setJar("build/test/mapred/testjar/testjob.jar"); return JobClient.runJob(jobConf); }
@Override public void run(String[] args) throws Exception { Flags flags = new Flags(); flags.add("input"); flags.add("output"); flags.add("prob"); flags.parseAndCheck(args); prob = flags.getDouble("prob"); JobConf job = new JobConf(this.getClass()); job.set("prob", flags.getString("prob")); MapReduceHelper.runTextSeqFileMapReduce( job, SamplingMapper.class, IdentityReducer.class, flags.getString("input"), flags.getString("output")); }
/** * Tests task's temp directory. * * In this test, we give different values to mapreduce.task.tmp.dir * both relative and absolute. And check whether the temp directory * is created. We also check whether java.io.tmpdir value is same as * the directory specified. We create a temp file and check if is is * created in the directory specified. */ @Test public void testTaskTempDir(){ try { JobConf conf = new JobConf(mr.getConfig()); // intialize input, output directories Path inDir = new Path("testing/wc/input"); Path outDir = new Path("testing/wc/output"); String input = "The input"; configure(conf, inDir, outDir, input, MapClass.class, IdentityReducer.class); launchTest(conf, inDir, outDir, input); } catch(Exception e) { e.printStackTrace(); fail("Exception in testing temp dir"); tearDown(); } }
/** * Configure a waiting job */ static void configureWaitingJobConf(JobConf jobConf, Path inDir, Path outputPath, int numMaps, int numRed, String jobName, String mapSignalFilename, String redSignalFilename) throws IOException { jobConf.setJobName(jobName); jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); jobConf.setInputFormat(RandomInputFormat.class); jobConf.setNumMapTasks(numMaps); jobConf.setNumReduceTasks(numRed); jobConf.setJar("build/test/testjar/testjob.jar"); jobConf.set(getTaskSignalParameter(true), mapSignalFilename); jobConf.set(getTaskSignalParameter(false), redSignalFilename); }
static RunningJob runJobSucceed(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-succeed"); conf.setMapperClass(IdentityMapper.class); conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); while (!job.isComplete()) { try { Thread.sleep(100); } catch (InterruptedException e) { break; } } return job; }
static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-fail"); conf.setMapperClass(FailMapper.class); conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); while (!job.isComplete()) { try { Thread.sleep(100); } catch (InterruptedException e) { break; } } return job; }
static RunningJob runJobKill(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-kill"); conf.setMapperClass(KillMapper.class); conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); while (job.getJobState() != JobStatus.RUNNING) { try { Thread.sleep(100); } catch (InterruptedException e) { break; } } job.killJob(); while (job.cleanupProgress() == 0.0f) { try { Thread.sleep(10); } catch (InterruptedException ie) { break; } } return job; }
/** * Launch tests * @param conf Configuration of the mapreduce job. * @param inDir input path * @param outDir output path * @param input Input text * @throws IOException */ public void launchTest(JobConf conf, Path inDir, Path outDir, String input) throws IOException { configure(conf, inDir, outDir, input, MapClass.class, IdentityReducer.class); FileSystem outFs = outDir.getFileSystem(conf); // Launch job with default option for temp dir. // i.e. temp dir is ./tmp JobClient.runJob(conf); outFs.delete(outDir, true); // Launch job by giving relative path to temp dir. conf.set("mapred.child.tmp", "../temp"); JobClient.runJob(conf); outFs.delete(outDir, true); // Launch job by giving absolute path to temp dir conf.set("mapred.child.tmp", "/tmp"); JobClient.runJob(conf); outFs.delete(outDir, true); }
@Test public void testEmptyJoin() throws Exception { JobConf job = new JobConf(); Path base = cluster.getFileSystem().makeQualified(new Path("/empty")); Path[] src = { new Path(base,"i0"), new Path("i1"), new Path("i2") }; job.set("mapreduce.join.expr", CompositeInputFormat.compose("outer", Fake_IF.class, src)); job.setInputFormat(CompositeInputFormat.class); FileOutputFormat.setOutputPath(job, new Path(base, "out")); job.setMapperClass(IdentityMapper.class); job.setReducerClass(IdentityReducer.class); job.setOutputKeyClass(IncomparableKey.class); job.setOutputValueClass(NullWritable.class); JobClient.runJob(job); base.getFileSystem(job).delete(base, true); }
void runTestTaskEnv(JobConf conf, Path inDir, Path outDir) throws IOException { String input = "The input"; configure(conf, inDir, outDir, input, EnvCheckMapper.class, IdentityReducer.class); // test // - new SET of new var (MY_PATH) // - set of old var (HOME) // - append to an old var from modified env (LD_LIBRARY_PATH) // - append to an old var from tt's env (PATH) // - append to a new var (NEW_PATH) conf.set("mapred.child.env", "MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp"); conf.set("path", System.getenv("PATH")); RunningJob job = JobClient.runJob(conf); assertTrue("The environment checker job failed.", job.isSuccessful()); }
static RunningJob launchJob(JobConf jobConf, Path inDir, Path outputPath, int numMaps, String jobName) throws IOException { jobConf.setJobName(jobName); jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setMapperClass(IdentityMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); jobConf.setNumMapTasks(numMaps); jobConf.setNumReduceTasks(0); jobConf.setJar("build/test/testjar/testjob.jar"); return JobClient.runJob(jobConf); }
@SuppressWarnings({ "deprecation", "unchecked" }) public static TaskCompletionEvent[] runJob(JobConf conf, Class mapperClass, boolean enableNoFetchEmptyMapOutputs) throws Exception { conf.setMapperClass(mapperClass); conf.setReducerClass(IdentityReducer.class); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); conf.setNumMapTasks(3); conf.setNumReduceTasks(1); conf.setInputFormat(FakeIF.class); conf.setBoolean("mapred.enable.no.fetch.map.outputs", enableNoFetchEmptyMapOutputs); FileInputFormat.setInputPaths(conf, new Path("/in")); final Path outp = new Path("/out"); FileOutputFormat.setOutputPath(conf, outp); RunningJob job = null; job = JobClient.runJob(conf); assertTrue(job.isSuccessful()); return job.getTaskCompletionEvents(0); }
