private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString()); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext)); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
public static Token createContainerToken(ContainerId cId, long rmIdentifier, NodeId nodeId, String user, NMContainerTokenSecretManager containerTokenSecretManager, LogAggregationContext logAggregationContext) throws IOException { Resource r = BuilderUtils.newResource(1024, 1); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), user, r, System.currentTimeMillis() + 100000L, 123, rmIdentifier, Priority.newInstance(0), 0, logAggregationContext); Token containerToken = BuilderUtils .newContainerToken(nodeId, containerTokenSecretManager .retrievePassword(containerTokenIdentifier), containerTokenIdentifier); return containerToken; }
private LogAggregationContext getLogAggregationContextFromContainerToken( MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext) throws Exception { RMApp app2 = rm1.submitApp(200, logAggregationContext); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(true); // request a container. am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED); // acquire the container. List<Container> containers = am2.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); Assert.assertEquals(containerId, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); ContainerTokenIdentifier token = BuilderUtils.newContainerTokenIdentifier(containers.get(0) .getContainerToken()); return token.getLogAggregationContext(); }
@Override protected RMContainerTokenSecretManager createContainerTokenSecretManager( Configuration conf) { return new RMContainerTokenSecretManager(conf) { @Override public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext); } }; }
@Test(timeout=20000) public void testStopAfterError() throws Exception { DeletionService delSrvc = mock(DeletionService.class); // get the AppLogAggregationImpl thread to crash LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class); when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException()); LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc); logAggregationService.init(this.conf); logAggregationService.start(); ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1); LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class); contextWithAllContainers.setLogAggregationPolicyClassName( AllContainerLogAggregationPolicy.class.getName()); logAggregationService.handle(new LogHandlerAppStartedEvent( application1, this.user, null, this.acls, contextWithAllContainers)); logAggregationService.stop(); assertEquals(0, logAggregationService.getNumAggregators()); logAggregationService.close(); }
private LogAggregationService createLogAggregationService( ApplicationId appId, String className, String parameters, boolean createLogAggContext) { ConcurrentHashMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>(); LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); logAggregationService.start(); LogAggregationContext logAggContext = null; if (createLogAggContext) { logAggContext = Records.newRecord(LogAggregationContext.class); logAggContext.setLogAggregationPolicyClassName(className); if (parameters != null) { logAggContext.setLogAggregationPolicyParameters(parameters); } } logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, logAggContext)); return logAggregationService; }
@Override protected RMContainerTokenSecretManager createContainerTokenSecretManager( Configuration conf) { return new RMContainerTokenSecretManager(conf) { @Override public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, String nodeLabelExp, ContainerType containerType) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext, nodeLabelExp, containerType); } }; }
@Test public void testLogAggregationContextPassedIntoContainerToken() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); MockNM nm2 = rm1.registerNode("127.0.0.1:2345", 8000); // LogAggregationContext is set as null Assert .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null)); // create a not-null LogAggregationContext LogAggregationContext logAggregationContext = LogAggregationContext.newInstance( "includePattern", "excludePattern"); LogAggregationContext returned = getLogAggregationContextFromContainerToken(rm1, nm2, logAggregationContext); Assert.assertEquals("includePattern", returned.getIncludePattern()); Assert.assertEquals("excludePattern", returned.getExcludePattern()); rm1.stop(); }
public LogValue(List<String> rootLogDirs, ContainerId containerId, String user, LogAggregationContext logAggregationContext, Set<String> alreadyUploadedLogFiles, boolean appFinished, boolean containerFinished, String userFolder) { this.rootLogDirs = new ArrayList<String>(rootLogDirs); this.containerId = containerId; this.user = user; this.userFolder = userFolder; // Ensure logs are processed in lexical order Collections.sort(this.rootLogDirs); this.logAggregationContext = logAggregationContext; this.alreadyUploadedLogFiles = alreadyUploadedLogFiles; this.appFinished = appFinished; this.containerFinished = containerFinished; }
private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString(), false); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext, user.getShortUserName() + "Folder")); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
@Test(timeout=20000) public void testStopAfterError() throws Exception { DeletionService delSrvc = mock(DeletionService.class); // get the AppLogAggregationImpl thread to crash LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class); when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException()); LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc); logAggregationService.init(this.conf); logAggregationService.start(); ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1); LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class); contextWithAllContainers.setLogAggregationPolicyClassName( AllContainerLogAggregationPolicy.class.getName()); logAggregationService.handle(new LogHandlerAppStartedEvent( application1, this.user, null, this.acls, contextWithAllContainers, this.userFolder)); logAggregationService.stop(); assertEquals(0, logAggregationService.getNumAggregators()); logAggregationService.close(); }
private LogAggregationService createLogAggregationService( ApplicationId appId, String className, String parameters, boolean createLogAggContext) { ConcurrentHashMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>(); LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); logAggregationService.start(); LogAggregationContext logAggContext = null; if (createLogAggContext) { logAggContext = Records.newRecord(LogAggregationContext.class); logAggContext.setLogAggregationPolicyClassName(className); if (parameters != null) { logAggContext.setLogAggregationPolicyParameters(parameters); } } logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, logAggContext, this.userFolder)); return logAggregationService; }
@Override protected RMContainerTokenSecretManager createContainerTokenSecretManager( Configuration conf) { return new RMContainerTokenSecretManager(conf) { @Override public Token createContainerToken(ContainerId containerId, int containerVersion, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, String nodeLabelExp, ContainerType containerType, String appSubmitterFolder) { numRetries++; return super.createContainerToken(containerId, containerVersion, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext, nodeLabelExp, containerType, appSubmitterFolder); } }; }
public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext) { ContainerTokenIdentifierProto.Builder builder = ContainerTokenIdentifierProto.newBuilder(); if (containerID != null) { builder.setContainerId(((ContainerIdPBImpl)containerID).getProto()); } builder.setNmHostAddr(hostName); builder.setAppSubmitter(appSubmitter); if (r != null) { builder.setResource(((ResourcePBImpl)r).getProto()); } builder.setExpiryTimeStamp(expiryTimeStamp); builder.setMasterKeyId(masterKeyId); builder.setRmIdentifier(rmIdentifier); if (priority != null) { builder.setPriority(((PriorityPBImpl)priority).getProto()); } builder.setCreationTime(creationTime); if (logAggregationContext != null) { builder.setLogAggregationContext( ((LogAggregationContextPBImpl)logAggregationContext).getProto()); } proto = builder.build(); }
@Override public LogAggregationContext getLogAggregationContext() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; if (this.logAggregationContext != null) { return this.logAggregationContext; } // Else via proto if (!p.hasLogAggregationContext()) { return null; } logAggregationContext = convertFromProtoFormat(p.getLogAggregationContext()); return logAggregationContext; }
@Override public void setLogAggregationContext( LogAggregationContext logAggregationContext) { maybeInitBuilder(); if (logAggregationContext == null) builder.clearLogAggregationContext(); this.logAggregationContext = logAggregationContext; }
public LogValue(List<String> rootLogDirs, ContainerId containerId, String user, LogAggregationContext logAggregationContext, Set<String> alreadyUploadedLogFiles, boolean appFinished) { this.rootLogDirs = new ArrayList<String>(rootLogDirs); this.containerId = containerId; this.user = user; // Ensure logs are processed in lexical order Collections.sort(this.rootLogDirs); this.logAggregationContext = logAggregationContext; this.alreadyUploadedLogFiles = alreadyUploadedLogFiles; this.appFinished = appFinished; }
public ContainerTokenIdentifierForTest(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext) { ContainerTokenIdentifierForTestProto.Builder builder = ContainerTokenIdentifierForTestProto.newBuilder(); if (containerID != null) { builder.setContainerId(((ContainerIdPBImpl)containerID).getProto()); } builder.setNmHostAddr(hostName); builder.setAppSubmitter(appSubmitter); if (r != null) { builder.setResource(((ResourcePBImpl)r).getProto()); } builder.setExpiryTimeStamp(expiryTimeStamp); builder.setMasterKeyId(masterKeyId); builder.setRmIdentifier(rmIdentifier); if (priority != null) { builder.setPriority(((PriorityPBImpl)priority).getProto()); } builder.setCreationTime(creationTime); if (logAggregationContext != null) { builder.setLogAggregationContext( ((LogAggregationContextPBImpl)logAggregationContext).getProto()); } proto = builder.build(); }
public ApplicationInitEvent(ApplicationId appId, Map<ApplicationAccessType, String> acls, LogAggregationContext logAggregationContext) { super(appId, ApplicationEventType.INIT_APPLICATION); this.applicationACLs = acls; this.logAggregationContext = logAggregationContext; }
@VisibleForTesting public LogAggregationContext getLogAggregationContext() { try { this.readLock.lock(); return this.logAggregationContext; } finally { this.readLock.unlock(); } }
public LogHandlerAppStartedEvent(ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy retentionPolicy, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) { super(LogHandlerEventType.APPLICATION_STARTED); this.applicationId = appId; this.user = user; this.credentials = credentials; this.retentionPolicy = retentionPolicy; this.appAcls = appAcls; this.logAggregationContext = logAggregationContext; }
private ContainerManagerApplicationProto buildAppProto(ApplicationId appId, String user, Credentials credentials, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) { ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder(); builder.setId(((ApplicationIdPBImpl) appId).getProto()); builder.setUser(user); if (logAggregationContext != null) { builder.setLogAggregationContext(( (LogAggregationContextPBImpl)logAggregationContext).getProto()); } builder.clearCredentials(); if (credentials != null) { DataOutputBuffer dob = new DataOutputBuffer(); try { credentials.writeTokenStorageToStream(dob); builder.setCredentials(ByteString.copyFrom(dob.getData())); } catch (IOException e) { // should not occur LOG.error("Cannot serialize credentials", e); } } builder.clearAcls(); if (appAcls != null) { for (Map.Entry<ApplicationAccessType, String> acl : appAcls.entrySet()) { ApplicationACLMapProto p = ApplicationACLMapProto.newBuilder() .setAccessType(ProtoUtils.convertToProtoFormat(acl.getKey())) .setAcl(acl.getValue()) .build(); builder.addAcls(p); } } return builder.build(); }
public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, logAggregationContext, true); }
@Test public void testLogAggregationContextPassedIntoContainerToken() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); MockNM nm2 = rm1.registerNode("127.0.0.1:2345", 8000); // LogAggregationContext is set as null Assert .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null)); // create a not-null LogAggregationContext LogAggregationContext logAggregationContext = LogAggregationContext.newInstance( "includePattern", "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern"); LogAggregationContext returned = getLogAggregationContextFromContainerToken(rm1, nm2, logAggregationContext); Assert.assertEquals("includePattern", returned.getIncludePattern()); Assert.assertEquals("excludePattern", returned.getExcludePattern()); Assert.assertEquals("rolledLogsIncludePattern", returned.getRolledLogsIncludePattern()); Assert.assertEquals("rolledLogsExcludePattern", returned.getRolledLogsExcludePattern()); rm1.stop(); }
public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext, String nodeLabelExpression) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, logAggregationContext, nodeLabelExpression, ContainerType.TASK); }
public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext, String nodeLabelExpression, ContainerType containerType) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, logAggregationContext, nodeLabelExpression, containerType, ExecutionType.GUARANTEED); }
public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext, String nodeLabelExpression, ContainerType containerType, ExecutionType executionType) { ContainerTokenIdentifierProto.Builder builder = ContainerTokenIdentifierProto.newBuilder(); if (containerID != null) { builder.setContainerId(((ContainerIdPBImpl)containerID).getProto()); } builder.setNmHostAddr(hostName); builder.setAppSubmitter(appSubmitter); if (r != null) { builder.setResource(((ResourcePBImpl)r).getProto()); } builder.setExpiryTimeStamp(expiryTimeStamp); builder.setMasterKeyId(masterKeyId); builder.setRmIdentifier(rmIdentifier); if (priority != null) { builder.setPriority(((PriorityPBImpl)priority).getProto()); } builder.setCreationTime(creationTime); if (logAggregationContext != null) { builder.setLogAggregationContext( ((LogAggregationContextPBImpl)logAggregationContext).getProto()); } if (nodeLabelExpression != null) { builder.setNodeLabelExpression(nodeLabelExpression); } builder.setContainerType(convertToProtoFormat(containerType)); builder.setExecutionType(convertToProtoFormat(executionType)); proto = builder.build(); }
