private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); tt.setLocalDirAllocator(localDirAllocator); LocalStorage localStorage = new LocalStorage(conf.getLocalDirs()); LocalFileSystem localFs = FileSystem.getLocal(conf); localStorage.checkDirs(localFs, true); tt.setLocalStorage(localStorage); localizer = new Localizer(FileSystem.get(conf), conf .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, localStorage); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
private void setUpCluster(int numRacks, int numNodesPerRack, boolean assignMultiple) throws IOException { resetMetrics(); conf = new JobConf(); conf.set("mapred.fairscheduler.allocation.file", ALLOC_FILE); conf.set("mapred.fairscheduler.poolnameproperty", POOL_PROPERTY); conf.setBoolean("mapred.fairscheduler.assignmultiple", assignMultiple); // Manually set locality delay because we aren't using a JobTracker so // we can't auto-compute it from the heartbeat interval. conf.setLong("mapred.fairscheduler.locality.delay", 10000); taskTrackerManager = new FakeTaskTrackerManager(numRacks, numNodesPerRack); clock = new FakeClock(); scheduler = new FairScheduler(clock, true); scheduler.waitForMapsBeforeLaunchingReduces = false; scheduler.setConf(conf); scheduler.setTaskTrackerManager(taskTrackerManager); scheduler.start(); }
private void startTT(JobConf conf) throws IOException, InterruptedException { myClock = new FakeClock(); // clock is reset. String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1"; conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs); tt = new TaskTracker(); tt.setConf(new JobConf(conf)); LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir"); localizer = new Localizer(FileSystem.get(conf), conf .getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); tt.setLocalizer(localizer); userLogManager = new UtilsForTests.InLineUserLogManager(conf); TaskController taskController = userLogManager.getTaskController(); taskController.setup(localDirAllocator, new LocalStorage(conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY))); tt.setTaskController(taskController); userLogCleaner = userLogManager.getUserLogCleaner(); userLogCleaner.setClock(myClock); tt.setUserLogManager(userLogManager); userLogManager.clearOldUserLogs(conf); }
public HFSPFakeJobInProgress(JobConf jobConf, FakeClock clock, HFSPFakeTaskTrackerManager taskTrackerManager, JobTracker jobTracker, String[][] mapInputLocations, JobTracker jt) throws IOException { super(new JobID("test", ++(HFSPFakeJobInProgress.jobCounter)), jobConf, jt); this.taskTrackerManager = taskTrackerManager; this.jobTracker = jobTracker; this.clock = clock; this.startTime = clock.time; this.mapInputLocations = mapInputLocations; this.startTime = System.currentTimeMillis(); this.status = new JobStatus(); this.status.setRunState(JobStatus.PREP); // this.nonLocalMaps = new LinkedList<TaskInProgress>(); this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>(); this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>(); this.nonRunningReduces = new HashSet<TaskInProgress>(); this.runningReduces = new LinkedHashSet<TaskInProgress>(); // this.jobHistory = new FakeJobHistory(); initTasks(); }
public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestRackAwareTaskPlacement.class)) { protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setClass("topology.node.switch.mapping.impl", StaticMapping.class, DNSToSwitchMapping.class); conf.set(JTConfig.JT_INSTRUMENTATION, FakeJobTrackerMetricsInst.class.getName()); jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); fakeInst = (FakeJobTrackerMetricsInst) jobTracker.getInstrumentation(); // Set up the Topology Information for (int i = 0; i < allHosts.length; i++) { StaticMapping.addNodeToRack(allHosts[i], allRacks[i]); } for (String tracker : trackers) { FakeObjectUtilities.establishFirstContact(jobTracker, tracker); } } }; return setup; }
public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestJobTrackerInstrumentation.class)) { protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1); conf.setClass(JTConfig.JT_TASK_SCHEDULER, FakeTaskScheduler.class, TaskScheduler.class); conf.set(JTConfig.JT_INSTRUMENTATION, FakeJobTrackerMetricsInst.class.getName()); jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); mi = (FakeJobTrackerMetricsInst) jobTracker.getInstrumentation(); for (String tracker : trackers) { FakeObjectUtilities.establishFirstContact(jobTracker, tracker); } } protected void tearDown() throws Exception { } }; return setup; }
public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestJobInProgress.class)) { protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setClass("topology.node.switch.mapping.impl", StaticMapping.class, DNSToSwitchMapping.class); jobTracker = new FakeJobTracker(conf, new FakeClock(), trackers); // Set up the Topology Information for (int i = 0; i < hosts.length; i++) { StaticMapping.addNodeToRack(hosts[i], racks[i]); } for (String s: trackers) { FakeObjectUtilities.establishFirstContact(jobTracker, s); } } }; return setup; }
private void setUpCluster(int numRacks, int numNodesPerRack, boolean assignMultiple) { conf = new JobConf(); conf.set("mapred.fairscheduler.allocation.file", ALLOC_FILE); conf.set("mapred.fairscheduler.poolnameproperty", POOL_PROPERTY); conf.setBoolean("mapred.fairscheduler.assignmultiple", assignMultiple); // Manually set locality delay because we aren't using a JobTracker so // we can't auto-compute it from the heartbeat interval. conf.setLong("mapred.fairscheduler.locality.delay.node", 5000); conf.setLong("mapred.fairscheduler.locality.delay.rack", 10000); taskTrackerManager = new FakeTaskTrackerManager(numRacks, numNodesPerRack); clock = new FakeClock(); scheduler = new FairScheduler(clock, true); scheduler.waitForMapsBeforeLaunchingReduces = false; scheduler.setConf(conf); scheduler.setTaskTrackerManager(taskTrackerManager); scheduler.start(); }
private void setUpCluster(int numRacks, int numNodesPerRack, boolean assignMultiple) throws IOException { resetMetrics(); conf = new JobConf(); conf.set("mapred.fairscheduler.allocation.file", ALLOC_FILE); conf.set("mapred.fairscheduler.poolnameproperty", POOL_PROPERTY); conf.setBoolean("mapred.fairscheduler.assignmultiple", assignMultiple); // Manually set locality delay because we aren't using a JobTracker so // we can't auto-compute it from the heartbeat interval. conf.setLong("mapred.fairscheduler.locality.delay.node", 5000); conf.setLong("mapred.fairscheduler.locality.delay.rack", 10000); conf.set("mapred.job.tracker", "localhost:0"); conf.set("mapred.job.tracker.http.address", "0.0.0.0:0"); taskTrackerManager = new FakeTaskTrackerManager(numRacks, numNodesPerRack); clock = new FakeClock(); try { jobTracker = new JobTracker(conf, clock); } catch (Exception e) { throw new RuntimeException("Could not start JT", e); } scheduler = new FairScheduler(clock, true); scheduler.waitForMapsBeforeLaunchingReduces = false; scheduler.setConf(conf); scheduler.setTaskTrackerManager(taskTrackerManager); scheduler.start(); // TaskStatus complains if a task's start time is 0, so advance it a bit advanceTime(100); }
HFSPFakeTaskInProgress(JobID jId, JobTracker jobTracker, boolean isMap, int id, JobConf jobConf, HFSPFakeJobInProgress job, String[] inputLocations, JobSplit.TaskSplitMetaInfo split, FakeClock clock) { super(jId, "", split, jobTracker, jobConf, job, id, 1); this.clock = clock; this.isMap = isMap; this.fakeJob = job; this.inputLocations = inputLocations; activeTasks = new TreeMap<TaskAttemptID, String>(); taskStatus = TaskStatus.createTaskStatus(isMap); taskStatus.setRunState(TaskStatus.State.UNASSIGNED); }
private void setUpCluster(int numRacks, int numNodesPerRack, boolean assignMultiple) throws IOException, IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException, InterruptedException { this.conf = new JobConf(); this.conf.setInt(HFSPScheduler.TRAINER_MIN_MAPS_KEYNAME, 0); this.conf.setInt(CompletedTasksTrainer.NUM_MAP_COMPLETED_KEY, 1); this.conf.setInt(HFSPScheduler.TRAINER_MIN_REDUCES_KEYNAME, 0); this.conf.setInt(SojournTrainer.getSojournConfKeyname( HFSPScheduler.PREFIX_KEYNAME, TaskType.REDUCE), 1); this.conf.set("mapred.job.tracker", "localhost:0"); this.conf.set("mapred.job.tracker.http.address", "0.0.0.0:0"); this.taskTrackerManager = new HFSPFakeTaskTrackerManager(numRacks, numNodesPerRack); this.clock = new FakeClock(); try { this.jobTracker = new JobTracker(this.conf, this.clock); } catch (Exception e) { throw new RuntimeException("Could not start JT", e); } this.scheduler = new HFSPScheduler(clock, true); this.scheduler.setConf(conf); this.scheduler.setTaskTrackerManager(taskTrackerManager); this.scheduler.start(); this.advanceTime(100); }
@Override protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); clock = new FakeClock(); // We use a "partial mock" of JobTracker which lets us see when certain // methods are called. If we were writing JobTracker from scratch then // we would make it call another object which we would mock out instead // (and use a real JobTracker) so we could perform assertions on the mock. // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#16 jobTracker = spy(new JobTracker(conf, clock)); }
public void testJobTrackerStartConfig() throws Exception { JobConf conf = new JobConf(); conf = MiniMRCluster.configureJobConf(conf, "file:///", 0, 0, null); // test JobTracker's default clock Clock c = JobTracker.getClock(); assertNotNull(c); assertEquals(c, JobTracker.DEFAULT_CLOCK); // test with default values JobTracker jt = JobTracker.startTracker(conf); c = JobTracker.getClock(); // test clock assertNotNull(c); assertEquals(c, JobTracker.DEFAULT_CLOCK); // test identifier assertEquals(12, jt.getTrackerIdentifier().length()); // correct upto mins jt.stopTracker(); // test with special clock FakeClock myClock = new FakeClock(); jt = JobTracker.startTracker(conf, myClock); c = JobTracker.getClock(); assertNotNull(c); assertEquals(c, myClock); jt.stopTracker(); // test with special identifier String identifier = "test-identifier"; jt = JobTracker.startTracker(conf, JobTracker.DEFAULT_CLOCK, identifier); assertEquals(identifier, jt.getTrackerIdentifier()); jt.stopTracker(); }
@Override protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); conf.set(JTConfig.JT_MAX_TRACKER_BLACKLISTS, "1"); jobTracker = new FakeJobTracker(conf, (clock = new FakeClock()), trackers); jobTracker.startExpireTrackersThread(); }
@Override protected void setUp() throws Exception { JobConf conf = new JobConf(); conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0"); conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0"); conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000); jobTracker = new FakeJobTracker(conf, (clock = new FakeClock()), trackers); }
