/** * Return the job conf configured with the priorities and mappers as passed. * @param conf The default conf * @param priorities priorities for the jobs * @param numMaps number of maps for the jobs * @param numReds number of reducers for the jobs * @param outputDir output dir * @param inDir input dir * @param mapSignalFile filename thats acts as a signal for maps * @param reduceSignalFile filename thats acts as a signal for reducers * @return a array of jobconfs configured as needed * @throws IOException */ private static JobConf[] getJobs(JobConf conf, JobPriority[] priorities, int[] numMaps, int[] numReds, Path outputDir, Path inDir, String mapSignalFile, String reduceSignalFile) throws IOException { JobConf[] jobs = new JobConf[priorities.length]; for (int i = 0; i < jobs.length; ++i) { jobs[i] = new JobConf(conf); Path newOutputDir = outputDir.suffix(String.valueOf(numJobsSubmitted++)); UtilsForTests.configureWaitingJobConf(jobs[i], inDir, newOutputDir, numMaps[i], numReds[i], "jt restart test job", mapSignalFile, reduceSignalFile); jobs[i].setJobPriority(priorities[i]); } return jobs; }
/** * Will verify that given task tracker is not blacklisted * @param client tasktracker info * @param conf modified configuration object * @param cluster mrcluster instance * @throws IOException thrown if verification fails */ public void verifyTTNotBlackListed(TTClient client, Configuration conf, MRCluster cluster) throws IOException { int interval = conf.getInt("mapred.healthChecker.interval",0); Assert.assertTrue("Interval cannot be zero.",interval != 0); UtilsForTests.waitFor(interval+2000); String defaultHealthScript = conf.get("mapred.healthChecker.script.path"); Assert.assertTrue("Task tracker is not healthy", nodeHealthStatus(client, true) == true); TaskTrackerStatus status = client.getStatus(); JTClient jclient = cluster.getJTClient(); Assert.assertTrue("Failed to move task tracker to healthy list", jclient.getProxy().isBlackListed(status.getTrackerName()) == false); Assert.assertTrue("Health script was not set",defaultHealthScript != null); }
/** * Verifies that the given task tracker is blacklisted * @param conf modified Configuration object * @param client tasktracker info * @param errorMessage that needs to be asserted * @param cluster mr cluster instance * @throws IOException is thrown when verification fails */ public void verifyTTBlackList(Configuration conf, TTClient client, String errorMessage, MRCluster cluster) throws IOException{ int interval = conf.getInt("mapred.healthChecker.interval",0); Assert.assertTrue("Interval cannot be zero.",interval != 0); UtilsForTests.waitFor(interval+2000); //TaskTrackerStatus status = client.getStatus(); Assert.assertTrue("Task tracker was never blacklisted ", nodeHealthStatus(client, false) == true); TaskTrackerStatus status = client.getStatus(); Assert.assertTrue("The custom error message did not appear", status.getHealthStatus().getHealthReport().trim(). equals(errorMessage)); JTClient jClient = cluster.getJTClient(); Assert.assertTrue("Failed to move task tracker to blacklisted list", jClient.getProxy().isBlackListed(status.getTrackerName()) == true); }
/** * The method return true from the task tracker if it is unhealthy/healthy * depending the blacklisted status * @param client the tracker tracker instance * @param health status information. * @return status of task tracker * @throws IOException failed to get the status of task tracker */ public boolean nodeHealthStatus(TTClient client,boolean hStatus) throws IOException { int counter = 0; TaskTrackerStatus status = client.getStatus(); while (counter < 60) { LOG.info("isNodeHealthy "+status.getHealthStatus().isNodeHealthy()); if (status.getHealthStatus().isNodeHealthy() == hStatus) { break; } else { UtilsForTests.waitFor(3000); status = client.getStatus(); Assert.assertNotNull("Task tracker status is null",status); } counter++; } if(counter != 60) { return true; } return false; }
private void waitTillRunState(JobInfo jInfo, JobID jobID, JTProtocol remoteJTClient) throws Exception { int count = 0; while (jInfo != null && jInfo.getStatus().getRunState() != JobStatus.RUNNING) { UtilsForTests.waitFor(10000); count++; jInfo = remoteJTClient.getJobInfo(jobID); //If the count goes beyond 100 seconds, then break; This is to avoid //infinite loop. if (count > 10) { Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point"); } } }
/** * The method provides the information on the job has stopped or not * @return indicates true if the job has stopped false otherwise. * @param job id has the information of the running job. * @throw IOException is thrown if the job info cannot be fetched. */ public boolean isJobStopped(JobID id) throws IOException{ int counter = 0; JobInfo jInfo = getProxy().getJobInfo(id); if(jInfo != null ) { while (counter < 60) { if (jInfo.getStatus().isJobComplete()) { break; } UtilsForTests.waitFor(1000); jInfo = getProxy().getJobInfo(id); counter ++; } } return (counter != 60)? true : false; }
/** * It uses to check whether job is started or not. * @param id job id * @return true if job is running. * @throws IOException if an I/O error occurs. */ public boolean isJobStarted(JobID id) throws IOException { JobInfo jInfo = getJobInfo(id); int counter = 0; while (counter < 60) { if (jInfo.getStatus().getRunState() == JobStatus.RUNNING) { break; } else { UtilsForTests.waitFor(1000); jInfo = getJobInfo(jInfo.getID()); Assert.assertNotNull("Job information is null",jInfo); } counter++; } return (counter != 60)? true : false ; }
/** * It uses to check whether task is started or not. * @param taskInfo task information * @return true if task is running. * @throws IOException if an I/O error occurs. */ public boolean isTaskStarted(TaskInfo taskInfo) throws IOException { JTProtocol wovenClient = getProxy(); int counter = 0; while (counter < 60) { if (taskInfo.getTaskStatus().length > 0) { if (taskInfo.getTaskStatus()[0].getRunState() == TaskStatus.State.RUNNING) { break; } } UtilsForTests.waitFor(1000); taskInfo = wovenClient.getTaskInfo(taskInfo.getTaskID()); counter++; } return (counter != 60)? true : false; }
/** * This methods provides the information on the particular task managed * by a task tracker has stopped or not. * @param TaskID is id of the task to get the status. * @throws IOException if there is an error. * @return true is stopped. */ public boolean isTaskStopped(TaskID tID) throws IOException { int counter = 0; if(tID != null && proxy.getTask(tID) != null) { TaskStatus.State tState= proxy.getTask(tID).getTaskStatus().getRunState(); while ( counter < 60) { if(tState != TaskStatus.State.RUNNING && tState != TaskStatus.State.UNASSIGNED) { break; } UtilsForTests.waitFor(1000); tState= proxy.getTask(tID).getTaskStatus().getRunState(); counter++; } } return (counter != 60)? true : false; }
/** * Get a TTClient Instance from a running task <br/> * @param Task Information of the running task * @return TTClient instance * @throws IOException */ public TTClient getTTClientInstance(TaskInfo taskInfo) throws IOException { JTProtocol remoteJTClient = getJTClient().getProxy(); String [] taskTrackers = taskInfo.getTaskTrackers(); int counter = 0; TTClient ttClient = null; while (counter < 60) { if (taskTrackers.length != 0) { break; } UtilsForTests.waitFor(100); taskInfo = remoteJTClient.getTaskInfo(taskInfo.getTaskID()); taskTrackers = taskInfo.getTaskTrackers(); counter ++; } if ( taskTrackers.length != 0 ) { String hostName = taskTrackers[0].split("_")[1]; hostName = hostName.split(":")[0]; ttClient = getTTClient(hostName); } return ttClient; }
