Java 类com.amazonaws.services.elasticmapreduce.model.ClusterSummary 实例源码

@Test
public void testGetActiveEmrClusterByName() throws Exception
{
    // Get the EMR cluster definition object
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);

    // check cluster summary before creation
    ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrHelper.getAwsParamsDto());
    assertNull(clusterSummary);

    // Create the cluster
    String clusterId = emrDao.createEmrCluster(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrClusterDefinition, emrHelper.getAwsParamsDto());

    // check cluster summary after creation
    clusterSummary = emrDao.getActiveEmrClusterByName(MockEmrOperationsImpl.MOCK_CLUSTER_NAME, emrHelper.getAwsParamsDto());
    assertNotNull(clusterSummary);
    assertEquals(clusterId, clusterSummary.getId());
}

@Test
public void testGetActiveEmrClusterIdNoIdSpecifiedAssertReturnActualClusterId()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = null;
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getActiveEmrClusterByName(any(), any())).thenReturn(new ClusterSummary().withId(expectedEmrClusterId).withName(emrClusterName));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getActiveEmrClusterByName(eq(emrClusterName), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

@Test
public void terminateEmrCluster() throws Exception
{
    String clusterName = "clusterName";
    boolean overrideTerminationProtection = false;
    String clusterId = "clusterId";

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(clusterId);
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    emrDao.terminateEmrCluster(clusterId, overrideTerminationProtection, new AwsParamsDto());

    // Assert that terminateEmrCluster was called with these parameters ONCE
    verify(mockEmrOperations).terminateEmrCluster(any(), eq(clusterId), eq(overrideTerminationProtection));
}

@Test
public void testGetActiveEmrClusterByNameWhenClusterNameIsBlank()
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByName(BLANK_TEXT, awsParamsDto);

    // Verify the external calls.
    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertNull(result);
}

/**
 * Collect data for ElasticMapReduce.
 *
 * @param stats
 *            current statistics object.
 * @param account
 *            currently used credentials object.
 * @param region
 *            currently used aws region.
 */
public static void scanElasticMapReduce(AwsStats stats, AwsAccount account, Regions region) {
    LOG.debug("Scan for MapReduce in region " + region.getName() + " in account " + account.getAccountId());

    try {
        AmazonElasticMapReduce elasticMapReduce = new AmazonElasticMapReduceClient(account.getCredentials());
        elasticMapReduce.setRegion(Region.getRegion(region));

        List<ClusterSummary> list = elasticMapReduce.listClusters().getClusters();

        int totalItems = list.size();
        for (ClusterSummary cs : list) {
            stats.add(new AwsResource(cs.getName(), account.getAccountId(), AwsResourceType.ElasticMapReduce, region));
        }

        LOG.info(totalItems + " ElasticMapReduce clusters in region " + region.getName() + " in account " + account.getAccountId());
    } catch (AmazonServiceException ase) {
        if (ase.getErrorCode().contains("AccessDenied")) {
            LOG.info("Access denied for ElasticMapReduce in region " + region.getName() + " in account " + account.getAccountId());
        } else {
            LOG.error("Exception of ElasticMapReduce: " + ase.getMessage());
        }
    }
}

protected List<String> getActiveTaggedClusters() throws Exception{
    AmazonElasticMapReduceClient emrClient = new AmazonElasticMapReduceClient();
    List<String> waitingClusters = new ArrayList<String>();
    ListClustersResult clusterResult = emrClient.listClusters(new ListClustersRequest().withClusterStates(ClusterState.WAITING));

    DescribeClusterRequest specifcTagDescribe = new DescribeClusterRequest();
    specifcTagDescribe.putCustomQueryParameter("Cluster.Tags",null);
     for( ClusterSummary cluster : clusterResult.getClusters()){
            System.out.println("list cluster id "+cluster.getId());
            List<Tag> tagList = emrClient.describeCluster(specifcTagDescribe.withClusterId(cluster.getId())).getCluster().getTags();
            for(Tag tag:tagList){
                if(tag.getKey().equals(props.getProperty("edba.cluster.tag.key"))){
                    waitingClusters.add(cluster.getId());
                }
            }

    }
    return waitingClusters;

}

@Test
public void addEmrMasterSecurityGroupsCallsEc2AddSecurityGroup() throws Exception
{
    String clusterName = "clusterName";
    List<String> securityGroups = Arrays.asList("securityGroup");
    AwsParamsDto awsParams = new AwsParamsDto();
    String ec2InstanceId = "ec2InstanceId";

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("clusterId");
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    ListInstancesResult listInstancesResult = new ListInstancesResult();
    listInstancesResult.setInstances(new ArrayList<>());
    Instance instance = new Instance();
    instance.setEc2InstanceId(ec2InstanceId);
    listInstancesResult.getInstances().add(instance);
    when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(listInstancesResult);

    emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams);

    verify(mockEc2Dao).addSecurityGroupsToEc2Instance(eq(ec2InstanceId), eq(securityGroups), any());
    verifyNoMoreInteractions(mockEc2Dao);
}

@Test
public void addEmrMasterSecurityGroupsThrowWhenNoInstancesFound() throws Exception
{
    String clusterName = "clusterName";
    List<String> securityGroups = Arrays.asList("securityGroup");
    AwsParamsDto awsParams = new AwsParamsDto();

    ListClustersResult listClustersResult = new ListClustersResult();
    listClustersResult.setClusters(new ArrayList<>());
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("clusterId");
    clusterSummary.setName(clusterName);
    listClustersResult.getClusters().add(clusterSummary);
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);

    when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(new ListInstancesResult());

    try
    {
        emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams);
        fail();
    }
    catch (Exception e)
    {
        assertEquals(IllegalArgumentException.class, e.getClass());
        assertEquals("No master instances found for the cluster \"" + clusterName + "\".", e.getMessage());
    }
}

@Override
public ListClustersResult listEmrClusters(AmazonElasticMapReduceClient emrClient, ListClustersRequest listClustersRequest)
{
    List<ClusterSummary> clusterSummaryList = new ArrayList<>();
    for (MockEmrJobFlow cluster : emrClusters.values())
    {
        if (!listClustersRequest.getClusterStates().isEmpty() && listClustersRequest.getClusterStates().contains(cluster.getStatus()))
        {
            ClusterSummary clusterSummary = new ClusterSummary();
            clusterSummary.withId(cluster.getJobFlowId()).withName(cluster.getJobFlowName()).withStatus(new ClusterStatus().withState(cluster.getStatus())
                .withStateChangeReason(new ClusterStateChangeReason().withCode(cluster.getStatusChangeReason().getCode())
                    .withMessage(cluster.getStatusChangeReason().getMessage())).withTimeline(new ClusterTimeline().withCreationDateTime(
                    cluster.getStatusTimeline().getCreationTime() != null ? cluster.getStatusTimeline().getCreationTime().toGregorianCalendar().getTime() :
                        null).withEndDateTime(
                    cluster.getStatusTimeline().getEndTime() != null ? cluster.getStatusTimeline().getEndTime().toGregorianCalendar().getTime() : null)
                    .withReadyDateTime(
                        cluster.getStatusTimeline().getReadyTime() != null ? cluster.getStatusTimeline().getReadyTime().toGregorianCalendar().getTime() :
                            null)));
            clusterSummaryList.add(clusterSummary);
        }
    }
    if (StringUtils.isBlank(listClustersRequest.getMarker()))
    {
        return new ListClustersResult().withClusters(clusterSummaryList).withMarker(MOCK_EMR_MAKER);
    }
    else
    {
        return new ListClustersResult().withClusters(clusterSummaryList);
    }
}

/**
 * Helper method to determine if an Amazon EMR cluster exists
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to check
 * @return true if the Amazon EMR cluster exists, otherwise false
 */
public static boolean clusterExists(AmazonElasticMapReduce client, String clusterIdentifier) {
    if (clusterIdentifier != null && !clusterIdentifier.isEmpty()) {
        ListClustersResult clustersList = client.listClusters();
        ListIterator<ClusterSummary> iterator = clustersList.getClusters().listIterator();
        ClusterSummary summary;
        for (summary = iterator.next() ; iterator.hasNext();summary = iterator.next()) {
            if (summary.getId().equals(clusterIdentifier)) {
                DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);  
                DescribeClusterResult result = client.describeCluster(describeClusterRequest);  
                if (result != null) {
                    Cluster cluster = result.getCluster();
                    //check if HBase is installed on this cluster
                    if (isHBaseInstalled(client, cluster.getId())) return false;
                    String state = cluster.getStatus().getState();
                    LOG.info(clusterIdentifier + " is " + state + ". ");
                    if (state.equalsIgnoreCase("RUNNING") ||state.equalsIgnoreCase("WAITING"))  {
                        LOG.info("The cluster with id " + clusterIdentifier + " exists and is " + state);   
                        return true;
                    }
                }
            }       
        }                   
    }
    LOG.info("The cluster with id " + clusterIdentifier + " does not exist");
    return false;  
}

@Override
public ClusterSummary getActiveEmrClusterByName(String clusterName, AwsParamsDto awsParams)
{
    if (StringUtils.isNotBlank(clusterName))
    {
        /**
         * Call AWSOperations for ListClusters API. Need to list all the active clusters that are in
         * BOOTSTRAPPING/RUNNING/STARTING/WAITING states
         */
        ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(getActiveEmrClusterStates());

        /**
         * ListClusterRequest returns only 50 clusters at a time. However, this returns a marker
         * that can be used for subsequent calls to listClusters to get all the clusters
         */
        String markerForListClusters = listClustersRequest.getMarker();

        // Loop through all the available clusters and look for the given cluster id
        do
        {
            /**
             * Call AWSOperations for ListClusters API.
             * Need to include the Marker returned by the previous iteration
             */
            ListClustersResult clusterResult =
                emrOperations.listEmrClusters(getEmrClient(awsParams), listClustersRequest.withMarker(markerForListClusters));

            // Loop through all the active clusters returned by AWS
            for (ClusterSummary clusterInstance : clusterResult.getClusters())
            {
                // If the cluster name matches, then return the status
                if (StringUtils.isNotBlank(clusterInstance.getName()) && clusterInstance.getName().equalsIgnoreCase(clusterName))
                {
                    return clusterInstance;
                }
            }
            markerForListClusters = clusterResult.getMarker();
        }
        while (markerForListClusters != null);
    }

    return null;
}

@Test
public void getActiveEmrClusterByNameAssertUsesListMarker() throws Exception
{
    String clusterName = "clusterName";
    String expectedClusterId = "clusterId";

    when(mockEmrOperations.listEmrClusters(any(), any())).then(new Answer<ListClustersResult>()
    {
        @Override
        public ListClustersResult answer(InvocationOnMock invocation) throws Throwable
        {
            ListClustersRequest listClustersRequest = invocation.getArgument(1);
            String marker = listClustersRequest.getMarker();

            ListClustersResult listClustersResult = new ListClustersResult();
            listClustersResult.setClusters(new ArrayList<>());

            /*
             * When no marker is given, this is the request for the first page.
             * Return a known marker. The expectation is that the next call to this method should have a request with this expected marker.
             */
            if (marker == null)
            {
                listClustersResult.setMarker("pagination_marker");
            }
            /*
             * When a marker is given, this is expected to be the subsequent call.
             */
            else
            {
                // Assert that the correct marker is passed in
                assertEquals("pagination_marker", marker);

                ClusterSummary clusterSummary = new ClusterSummary();
                clusterSummary.setId(expectedClusterId);
                clusterSummary.setName(clusterName);
                listClustersResult.getClusters().add(clusterSummary);
            }
            return listClustersResult;
        }
    });

    ClusterSummary result = emrDao.getActiveEmrClusterByName(clusterName, new AwsParamsDto());
    assertNotNull(result);
    assertEquals(expectedClusterId, result.getId());
}

@Test
public void testGetActiveEmrClusterByName()
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a list cluster request.
    ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(EMR_VALID_STATE);

    // Create a list cluster result with a non-matching cluster and a marker.
    ListClustersResult listClusterResultWithMarker = new ListClustersResult().withClusters(new ClusterSummary().withName(INVALID_VALUE)).withMarker(MARKER);

    // Create a list cluster request with marker.
    ListClustersRequest listClustersRequestWithMarker = new ListClustersRequest().withClusterStates(EMR_VALID_STATE).withMarker(MARKER);

    // Create a cluster summary.
    ClusterSummary clusterSummary = new ClusterSummary().withName(EMR_CLUSTER_NAME);

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Mock the external calls.
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(EMR_VALID_STATE);
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequest)).thenReturn(listClusterResultWithMarker);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequestWithMarker)).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByName(EMR_CLUSTER_NAME, awsParamsDto);

    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
    verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
    verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
    verify(emrOperations, times(2)).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}

/**
 * Get an Active EMR cluster by the cluster name. Cluster only in following states are returned: ClusterState.BOOTSTRAPPING, ClusterState.RUNNING,
 * ClusterState.STARTING, ClusterState.WAITING
 *
 * @param awsParams AWS related parameters for access/secret keys and proxy details.
 * @param clusterName the cluster name value.
 *
 * @return the ClusterSummary object.
 */
public ClusterSummary getActiveEmrClusterByName(String clusterName, AwsParamsDto awsParams);