Java 类org.apache.hadoop.mapred.lib.LazyOutputFormat 实例源码

项目：hadoop 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：aliyun-oss-hadoop-fs 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：big-c 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hadoop-2.6.0-cdh5.4.3 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hadoop-plus 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：FlexMap 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hops 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hadoop-TCP 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hardfs 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：hadoop-on-lustre2 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}

项目：mapreduce-fork 文件：TestLazyOutput.java

private static void runTestLazyOutput(JobConf job, Path output,
    int numReducers, boolean createLazily) 
throws Exception {

  job.setJobName("test-lazy-output");

  FileInputFormat.setInputPaths(job, INPUT);
  FileOutputFormat.setOutputPath(job, output);
  job.setInputFormat(TextInputFormat.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapperClass(TestMapper.class);        
  job.setReducerClass(TestReducer.class);

  JobClient client = new JobClient(job);
  job.setNumReduceTasks(numReducers);
  if (createLazily) {
    LazyOutputFormat.setOutputFormatClass
      (job, TextOutputFormat.class);
  } else {
    job.setOutputFormat(TextOutputFormat.class);
  }

  JobClient.runJob(job);
}