/** Get a host for the given parameters. This method need not be thread-safe. */ public HttpHost getHost(String host, int port, String scheme) { if(scheme == null) { scheme = "http"; } Protocol protocol = protocols.get(scheme); if(protocol == null) { protocol = Protocol.getProtocol("http"); if(protocol == null) { throw new IllegalArgumentException("Unrecognised scheme parameter"); } } return new HttpHost(host, port, protocol); }
public MesosTracker(HttpHost host, TaskID taskId, long mapSlots, long reduceSlots, MesosScheduler scheduler) { this.host = host; this.taskId = taskId; this.mapSlots = mapSlots; this.reduceSlots = reduceSlots; this.scheduler = scheduler; if (scheduler.metrics != null) { this.context = scheduler.metrics.trackerTimer.time(); } this.idleCheckInterval = scheduler.conf.getLong("mapred.mesos.tracker.idle.interval", MesosScheduler.DEFAULT_IDLE_CHECK_INTERVAL); this.idleCheckMax = scheduler.conf.getLong("mapred.mesos.tracker.idle.checks", MesosScheduler.DEFAULT_IDLE_REVOCATION_CHECKS); scheduleStartupTimer(); if (this.idleCheckInterval > 0 && this.idleCheckMax > 0) { scheduleIdleCheck(); } }
private String getRequest() { if (!Server.serving) return ""; CloseableHttpClient httpclient = HttpClients.createDefault(); HttpHost host = new HttpHost("localhost", Server.frontPort, Protocol.getProtocol("http")); HttpGet request = new HttpGet(host.toURI().concat("/" + Server.RESPONSES)); ResponseHandler<String> responseHandler = new ResponseHandler<String>() { public String handleResponse(final HttpResponse response) throws IOException { int status = response.getStatusLine().getStatusCode(); if (status >= 200 && status < 300) { HttpEntity entity = response.getEntity(); return entity != null ? EntityUtils.toString(entity) : null; } else throw new ClientProtocolException("Unexpected response status: " + status); } }; String responseBody = null; try { responseBody = httpclient.execute(request, responseHandler); } catch (IOException e) { Runner.LOGGER.warn(e); } return responseBody; }
@Override public List<Task> assignTasks(TaskTracker taskTracker) throws IOException { HttpHost tracker = new HttpHost(taskTracker.getStatus().getHost(), taskTracker.getStatus().getHttpPort()); if (!mesosTrackers.containsKey(tracker)) { LOG.info("Unknown/exited TaskTracker: " + tracker + ". "); return null; } MesosTracker mesosTracker = mesosTrackers.get(tracker); // Make sure we're not asked to assign tasks to any task trackers that have // been stopped. This could happen while the task tracker has not been // removed from the cluster e.g still in the heartbeat timeout period. synchronized (this) { if (mesosTracker.stopped) { LOG.info("Asked to assign tasks to stopped tracker " + tracker + "."); return null; } } // Let the underlying task scheduler do the actual task scheduling. List<Task> tasks = taskScheduler.assignTasks(taskTracker); // The Hadoop Fair Scheduler is known to return null. if (tasks == null) { return null; } // Keep track of which TaskTracker contains which tasks. for (Task task : tasks) { mesosTracker.jobs.add(task.getJobID()); } return tasks; }
protected void scheduleStartupTimer() { scheduler.scheduleTimer(new Runnable() { @Override public void run() { if (MesosTracker.this.active) { // If the tracker activated while we were awaiting to acquire the // lock, start the periodic cleanup timer and return. schedulePeriodic(); return; } // When the scheduler is busy or doesn't receive offers, it may // fail to mark some TaskTrackers as active even though they are. // Here we do a final check with the JobTracker to make sure this // TaskTracker is really not there before we kill it. final Collection<TaskTrackerStatus> taskTrackers = MesosTracker.this.scheduler.jobTracker.taskTrackers(); for (TaskTrackerStatus status : taskTrackers) { HttpHost host = new HttpHost(status.getHost(), status.getHttpPort()); if (status.getHealthStatus().isNodeHealthy() && MesosTracker.this.host.equals(host)) { schedulePeriodic(); return; } } if (MesosTracker.this.scheduler.metrics != null) { MesosTracker.this.scheduler.metrics.launchTimeout.mark(); } LOG.warn("Tracker " + MesosTracker.this.host + " failed to launch within " + MesosScheduler.LAUNCH_TIMEOUT_MS / 1000 + " seconds, killing it"); MesosTracker.this.scheduler.killTracker(MesosTracker.this); } }, MesosScheduler.LAUNCH_TIMEOUT_MS, TimeUnit.MILLISECONDS); }
public synchronized void setHost(String host, int port, String scheme) { setHost(new HttpHost(host, port, getNewProtocol(host, port, scheme))); }
/** * Execute a method in a new HttpClient instance. * If the auth failed, authenticate then retry the method. * * @param method methot to exec * @param <M> Method type * @return the status code * @throws IOException on any failure */ private <M extends HttpMethod> int exec(M method) throws IOException { final HttpClient client = new HttpClient(); if (proxyHost != null) { client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(proxyHost, proxyPort)); } int statusCode = execWithDebugOutput(method, client); if ((statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == HttpStatus.SC_BAD_REQUEST) && method instanceof AuthPostMethod && !useKeystoneAuthentication) { if (LOG.isDebugEnabled()) { LOG.debug("Operation failed with status " + method.getStatusCode() + " attempting keystone auth"); } //if rackspace key authentication failed - try custom Keystone authentication useKeystoneAuthentication = true; final AuthPostMethod authentication = (AuthPostMethod) method; //replace rackspace auth with keystone one authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest)); statusCode = execWithDebugOutput(method, client); } if (statusCode == HttpStatus.SC_UNAUTHORIZED ) { //unauthed -or the auth uri rejected it. if (method instanceof AuthPostMethod) { //unauth response from the AUTH URI itself. throw new SwiftAuthenticationFailedException(authRequest.toString(), "auth", authUri, method); } //any other URL: try again if (LOG.isDebugEnabled()) { LOG.debug("Reauthenticating"); } //re-auth, this may recurse into the same dir authenticate(); if (LOG.isDebugEnabled()) { LOG.debug("Retrying original request"); } statusCode = execWithDebugOutput(method, client); } return statusCode; }
/** Get a host for the given parameters. This method need not be thread-safe. */ public HttpHost getHost(HostConfiguration old, String scheme, String host, int port) { return new HttpHost(host, port, getProtocol(old, scheme, host, port)); }
/** * Execute a method in a new HttpClient instance. * If the auth failed, authenticate then retry the method. * * @param method method to exec * @param <M> Method type * @return the status code * @throws IOException on any failure */ private <M extends HttpMethod> int exec(M method) throws IOException { final HttpClient client = new HttpClient(); if (proxyHost != null) { client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(proxyHost, proxyPort)); } int statusCode = execWithDebugOutput(method, client); if ((statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == HttpStatus.SC_BAD_REQUEST) && method instanceof AuthPostMethod && !useKeystoneAuthentication) { if (LOG.isDebugEnabled()) { LOG.debug("Operation failed with status " + method.getStatusCode() + " attempting keystone auth"); } //if rackspace key authentication failed - try custom Keystone authentication useKeystoneAuthentication = true; final AuthPostMethod authentication = (AuthPostMethod) method; //replace rackspace auth with keystone one authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest)); statusCode = execWithDebugOutput(method, client); } if (statusCode == HttpStatus.SC_UNAUTHORIZED ) { //unauthed -or the auth uri rejected it. if (method instanceof AuthPostMethod) { //unauth response from the AUTH URI itself. throw new SwiftAuthenticationFailedException(authRequest.toString(), "auth", authUri, method); } //any other URL: try again if (LOG.isDebugEnabled()) { LOG.debug("Reauthenticating"); } //re-auth, this may recurse into the same dir authenticate(); if (LOG.isDebugEnabled()) { LOG.debug("Retrying original request"); } statusCode = execWithDebugOutput(method, client); } return statusCode; }
@Override public synchronized void statusUpdate(SchedulerDriver schedulerDriver, Protos.TaskStatus taskStatus) { LOG.info("Status update of " + taskStatus.getTaskId().getValue() + " to " + taskStatus.getState().name() + " with message " + taskStatus.getMessage()); // Remove the TaskTracker if the corresponding Mesos task has reached a // terminal state. switch (taskStatus.getState()) { case TASK_FINISHED: case TASK_FAILED: case TASK_KILLED: case TASK_LOST: // Make a copy to iterate over keys and delete values. Set<HttpHost> trackers = new HashSet<HttpHost>(mesosTrackers.keySet()); // Remove the task from the map. for (HttpHost tracker : trackers) { if (mesosTrackers.get(tracker).taskId.equals(taskStatus.getTaskId())) { LOG.info("Removing terminated TaskTracker: " + tracker); mesosTrackers.get(tracker).stop(); mesosTrackers.remove(tracker); } } break; case TASK_STAGING: case TASK_STARTING: case TASK_RUNNING: break; default: LOG.error("Unexpected TaskStatus: " + taskStatus.getState().name()); break; } if (metrics != null) { Meter meter = metrics.taskStateMeter.get(taskStatus.getState()); if (meter != null) { meter.mark(); } } }
public synchronized void setHost(String host, int port, String scheme) { setHost(new HttpHost(host, port, keepProtocol(host, port, scheme))); }
/** * Execute a method in a new HttpClient instance. * If the auth failed, authenticate then retry the method. * * @param method methot to exec * @param <M> Method type * @return the status code * @throws IOException on any failure */ private <M extends HttpMethod> int exec(M method) throws IOException { final HttpClient client = new HttpClient(); if (proxyHost != null) { client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(proxyHost, proxyPort)); } int statusCode = execWithDebugOutput(method, client); if ((statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == HttpStatus.SC_BAD_REQUEST) && method instanceof AuthPostMethod && !useKeystoneAuthentication) { if (LOG.isDebugEnabled()) { LOG.debug("Operation failed with status " + method.getStatusCode() + " attempting keystone auth"); } //if rackspace key authentication failed - try custom Keystone authentication useKeystoneAuthentication = true; final AuthPostMethod authentication = (AuthPostMethod) method; //replace rackspace auth with keystone one authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest)); statusCode = execWithDebugOutput(method, client); } if (statusCode == HttpStatus.SC_UNAUTHORIZED ) { //unauthed -or the auth uri rejected it. if (method instanceof AuthPostMethod) { //unauth response from the AUTH URI itself. throw new SwiftAuthenticationFailedException(authRequest.toString(), "auth", authUri, method); } //any other URL: try again if (LOG.isDebugEnabled()) { LOG.debug("Reauthenticating"); } //re-auth, this may recurse into the same dir setAuthToken(method, authenticate()); if (LOG.isDebugEnabled()) { LOG.debug("Retrying original request"); } statusCode = execWithDebugOutput(method, client); } return statusCode; }
