Namespace Pulumi.Aws.Emr

Classes

Cluster

Provides an Elastic MapReduce Cluster, a web service that makes it easy to process large amounts of data efficiently. See Amazon Elastic MapReduce Documentation for more information.

To configure Instance Groups for task nodes, see the aws.emr.InstanceGroup resource.

Support for Instance Fleets will be made available in an upcoming release.

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var cluster = new Aws.Emr.Cluster("cluster", new Aws.Emr.ClusterArgs
    {
        AdditionalInfo = @"{
""instanceAwsClientConfiguration"": {
""proxyPort"": 8099,
""proxyHost"": ""myproxy.example.com""
}
}

",
        Applications = 
        {
            "Spark",
        },
        BootstrapActions = 
        {
            new Aws.Emr.Inputs.ClusterBootstrapActionArgs
            {
                Args = 
                {
                    "instance.isMaster=true",
                    "echo running on master node",
                },
                Name = "runif",
                Path = "s3://elasticmapreduce/bootstrap-actions/run-if",
            },
        },
        ConfigurationsJson = @"  [
{
  ""Classification"": ""hadoop-env"",
  ""Configurations"": [
    {
      ""Classification"": ""export"",
      ""Properties"": {
        ""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
      }
    }
  ],
  ""Properties"": {}
},
{
  ""Classification"": ""spark-env"",
  ""Configurations"": [
    {
      ""Classification"": ""export"",
      ""Properties"": {
        ""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
      }
    }
  ],
  ""Properties"": {}
}
]

",
        CoreInstanceGroup = new Aws.Emr.Inputs.ClusterCoreInstanceGroupArgs
        {
            AutoscalingPolicy = @"{
""Constraints"": {
""MinCapacity"": 1,
""MaxCapacity"": 2
},
""Rules"": [
{
""Name"": ""ScaleOutMemoryPercentage"",
""Description"": ""Scale out if YARNMemoryAvailablePercentage is less than 15"",
""Action"": {
  ""SimpleScalingPolicyConfiguration"": {
    ""AdjustmentType"": ""CHANGE_IN_CAPACITY"",
    ""ScalingAdjustment"": 1,
    ""CoolDown"": 300
  }
},
""Trigger"": {
  ""CloudWatchAlarmDefinition"": {
    ""ComparisonOperator"": ""LESS_THAN"",
    ""EvaluationPeriods"": 1,
    ""MetricName"": ""YARNMemoryAvailablePercentage"",
    ""Namespace"": ""AWS/ElasticMapReduce"",
    ""Period"": 300,
    ""Statistic"": ""AVERAGE"",
    ""Threshold"": 15.0,
    ""Unit"": ""PERCENT""
  }
}
}
]
}

",
            BidPrice = "0.30",
            EbsConfig = 
            {

                {
                    { "size", "40" },
                    { "type", "gp2" },
                    { "volumesPerInstance", 1 },
                },
            },
            InstanceCount = 1,
            InstanceType = "c4.large",
        },
        EbsRootVolumeSize = 100,
        Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
        {
            EmrManagedMasterSecurityGroup = aws_security_group.Sg.Id,
            EmrManagedSlaveSecurityGroup = aws_security_group.Sg.Id,
            InstanceProfile = aws_iam_instance_profile.Emr_profile.Arn,
            SubnetId = aws_subnet.Main.Id,
        },
        KeepJobFlowAliveWhenNoSteps = true,
        MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
        {
            InstanceType = "m4.large",
        },
        ReleaseLabel = "emr-4.6.0",
        ServiceRole = aws_iam_role.Iam_emr_service_role.Arn,
        Tags = 
        {
            { "env", "env" },
            { "role", "rolename" },
        },
        TerminationProtection = false,
    });
}

}

Multiple Node Master Instance Group

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    // Map public IP on launch must be enabled for public (Internet accessible) subnets
    var exampleSubnet = new Aws.Ec2.Subnet("exampleSubnet", new Aws.Ec2.SubnetArgs
    {
        MapPublicIpOnLaunch = true,
    });
    var exampleCluster = new Aws.Emr.Cluster("exampleCluster", new Aws.Emr.ClusterArgs
    {
        CoreInstanceGroup = ,
        Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
        {
            SubnetId = exampleSubnet.Id,
        },
        MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
        {
            InstanceCount = 3,
        },
        ReleaseLabel = "emr-5.24.1",
        TerminationProtection = true,
    });
}

}

ClusterArgs

ClusterState

InstanceGroup

Provides an Elastic MapReduce Cluster Instance Group configuration. See Amazon Elastic MapReduce Documentation for more information.

NOTE: At this time, Instance Groups cannot be destroyed through the API nor web interface. Instance Groups are destroyed when the EMR Cluster is destroyed. this provider will resize any Instance Group to zero when destroying the resource.

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var task = new Aws.Emr.InstanceGroup("task", new Aws.Emr.InstanceGroupArgs
    {
        ClusterId = aws_emr_cluster.Tf_test_cluster.Id,
        InstanceCount = 1,
        InstanceType = "m5.xlarge",
    });
}

}

InstanceGroupArgs

InstanceGroupState

SecurityConfiguration

Provides a resource to manage AWS EMR Security Configurations

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var foo = new Aws.Emr.SecurityConfiguration("foo", new Aws.Emr.SecurityConfigurationArgs
    {
        Configuration = @"{
""EncryptionConfiguration"": {
""AtRestEncryptionConfiguration"": {
  ""S3EncryptionConfiguration"": {
    ""EncryptionMode"": ""SSE-S3""
  },
  ""LocalDiskEncryptionConfiguration"": {
    ""EncryptionKeyProviderType"": ""AwsKms"",
    ""AwsKmsKey"": ""arn:aws:kms:us-west-2:187416307283:alias/tf_emr_test_key""
  }
},
""EnableInTransitEncryption"": false,
""EnableAtRestEncryption"": true
}
}

",
    });
}

}

SecurityConfigurationArgs

SecurityConfigurationState