Java 类org.apache.hadoop.mapred.gridmix.GenerateData.GenSplit 实例源码

项目：hadoop 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：aliyun-oss-hadoop-fs 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：big-c 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hadoop-2.6.0-cdh5.4.3 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hadoop-plus 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hops 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hadoop-TCP 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hardfs 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：hadoop-on-lustre2 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}

项目：mapreduce-fork 文件：TestCompressionEmulationUtils.java

@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  // get the total data to be generated
  long toGen =
    jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
  if (toGen < 0) {
    throw new IOException("Invalid/missing generation bytes: " + toGen);
  }
  // get the total number of mappers configured
  int totalMappersConfigured =
    jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
  if (totalMappersConfigured < 0) {
    throw new IOException("Invalid/missing num mappers: " 
                          + totalMappersConfigured);
  }

  final long bytesPerTracker = toGen / totalMappersConfigured;
  final ArrayList<InputSplit> splits = 
    new ArrayList<InputSplit>(totalMappersConfigured);
  for (int i = 0; i < totalMappersConfigured; ++i) {
    splits.add(new GenSplit(bytesPerTracker, 
               new String[] { "tracker_local" }));
  }
  return splits;
}