Namespace Pulumi.Gcp.Dataproc

Classes

AutoscalingPolicy

Describes an autoscaling policy for Dataproc cluster autoscaler.

AutoscalingPolicyArgs

AutoscalingPolicyState

Cluster

Manages a Cloud Dataproc cluster resource within GCP. For more information see the official dataproc documentation.

!> Warning: Due to limitations of the API, all arguments except labels,cluster_config.worker_config.num_instances and cluster_config.preemptible_worker_config.num_instances are non-updatable. Changing others will cause recreation of the whole cluster!

Example Usage - Basic

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var simplecluster = new Gcp.Dataproc.Cluster("simplecluster", new Gcp.Dataproc.ClusterArgs
    {
        Region = "us-central1",
    });
}

}

Example Usage - Advanced

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var mycluster = new Gcp.Dataproc.Cluster("mycluster", new Gcp.Dataproc.ClusterArgs
    {
        ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
        {
            GceClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigGceClusterConfigArgs
            {
                ServiceAccountScopes = 
                {
                    "https://www.googleapis.com/auth/monitoring",
                    "useraccounts-ro",
                    "storage-rw",
                    "logging-write",
                },
                Tags = 
                {
                    "foo",
                    "bar",
                },
            },
            InitializationAction = 
            {

                {
                    { "script", "gs://dataproc-initialization-actions/stackdriver/stackdriver.sh" },
                    { "timeoutSec", 500 },
                },
            },
            MasterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigArgs
            {
                DiskConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigDiskConfigArgs
                {
                    BootDiskSizeGb = 15,
                    BootDiskType = "pd-ssd",
                },
                MachineType = "n1-standard-1",
                NumInstances = 1,
            },
            PreemptibleWorkerConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigPreemptibleWorkerConfigArgs
            {
                NumInstances = 0,
            },
            SoftwareConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigSoftwareConfigArgs
            {
                ImageVersion = "1.3.7-deb9",
                OverrideProperties = 
                {
                    { "dataproc:dataproc.allow.zero.workers", "true" },
                },
            },
            StagingBucket = "dataproc-staging-bucket",
            WorkerConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigWorkerConfigArgs
            {
                DiskConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigWorkerConfigDiskConfigArgs
                {
                    BootDiskSizeGb = 15,
                    NumLocalSsds = 1,
                },
                MachineType = "n1-standard-1",
                MinCpuPlatform = "Intel Skylake",
                NumInstances = 2,
            },
        },
        Labels = 
        {
            { "foo", "bar" },
        },
        Region = "us-central1",
    });
}

}

Example Usage - Using a GPU accelerator

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var acceleratedCluster = new Gcp.Dataproc.Cluster("acceleratedCluster", new Gcp.Dataproc.ClusterArgs
    {
        ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
        {
            GceClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigGceClusterConfigArgs
            {
                Zone = "us-central1-a",
            },
            MasterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigArgs
            {
                Accelerators = 
                {
                    new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigAcceleratorArgs
                    {
                        AcceleratorCount = "1",
                        AcceleratorType = "nvidia-tesla-k80",
                    },
                },
            },
        },
        Region = "us-central1",
    });
}

}

ClusterArgs

ClusterIAMBinding

Three different resources help you manage IAM policies on dataproc clusters. Each of these resources serves a different use case:

• gcp.dataproc.ClusterIAMPolicy: Authoritative. Sets the IAM policy for the cluster and replaces any existing policy already attached.
• gcp.dataproc.ClusterIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the cluster are preserved.
• gcp.dataproc.ClusterIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the cluster are preserved.

Note: gcp.dataproc.ClusterIAMPolicy cannot be used in conjunction with gcp.dataproc.ClusterIAMBinding and gcp.dataproc.ClusterIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the cluster as gcp.dataproc.ClusterIAMPolicy replaces the entire policy.

Note: gcp.dataproc.ClusterIAMBinding resources can be used in conjunction with gcp.dataproc.ClusterIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.ClusterIAMPolicy("editor", new Gcp.Dataproc.ClusterIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        Cluster = "your-dataproc-cluster",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMBinding("editor", new Gcp.Dataproc.ClusterIAMBindingArgs
    {
        Cluster = "your-dataproc-cluster",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMMember("editor", new Gcp.Dataproc.ClusterIAMMemberArgs
    {
        Cluster = "your-dataproc-cluster",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

ClusterIAMBindingArgs

ClusterIAMBindingState

ClusterIAMMember

Three different resources help you manage IAM policies on dataproc clusters. Each of these resources serves a different use case:

• gcp.dataproc.ClusterIAMPolicy: Authoritative. Sets the IAM policy for the cluster and replaces any existing policy already attached.
• gcp.dataproc.ClusterIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the cluster are preserved.
• gcp.dataproc.ClusterIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the cluster are preserved.

Note: gcp.dataproc.ClusterIAMPolicy cannot be used in conjunction with gcp.dataproc.ClusterIAMBinding and gcp.dataproc.ClusterIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the cluster as gcp.dataproc.ClusterIAMPolicy replaces the entire policy.

Note: gcp.dataproc.ClusterIAMBinding resources can be used in conjunction with gcp.dataproc.ClusterIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.ClusterIAMPolicy("editor", new Gcp.Dataproc.ClusterIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        Cluster = "your-dataproc-cluster",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMBinding("editor", new Gcp.Dataproc.ClusterIAMBindingArgs
    {
        Cluster = "your-dataproc-cluster",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMMember("editor", new Gcp.Dataproc.ClusterIAMMemberArgs
    {
        Cluster = "your-dataproc-cluster",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

ClusterIAMMemberArgs

ClusterIAMMemberState

ClusterIAMPolicy

Three different resources help you manage IAM policies on dataproc clusters. Each of these resources serves a different use case:

• gcp.dataproc.ClusterIAMPolicy: Authoritative. Sets the IAM policy for the cluster and replaces any existing policy already attached.
• gcp.dataproc.ClusterIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the cluster are preserved.
• gcp.dataproc.ClusterIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the cluster are preserved.

Note: gcp.dataproc.ClusterIAMPolicy cannot be used in conjunction with gcp.dataproc.ClusterIAMBinding and gcp.dataproc.ClusterIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the cluster as gcp.dataproc.ClusterIAMPolicy replaces the entire policy.

Note: gcp.dataproc.ClusterIAMBinding resources can be used in conjunction with gcp.dataproc.ClusterIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.ClusterIAMPolicy("editor", new Gcp.Dataproc.ClusterIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        Cluster = "your-dataproc-cluster",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMBinding("editor", new Gcp.Dataproc.ClusterIAMBindingArgs
    {
        Cluster = "your-dataproc-cluster",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.ClusterIAMMember("editor", new Gcp.Dataproc.ClusterIAMMemberArgs
    {
        Cluster = "your-dataproc-cluster",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

ClusterIAMPolicyArgs

ClusterIAMPolicyState

ClusterState

Job

Manages a job resource within a Dataproc cluster within GCE. For more information see the official dataproc documentation.

!> Note: This resource does not support 'update' and changing any attributes will cause the resource to be recreated.

JobArgs

JobIAMBinding

Three different resources help you manage IAM policies on dataproc jobs. Each of these resources serves a different use case:

• gcp.dataproc.JobIAMPolicy: Authoritative. Sets the IAM policy for the job and replaces any existing policy already attached.
• gcp.dataproc.JobIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the job are preserved.
• gcp.dataproc.JobIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the job are preserved.

Note: gcp.dataproc.JobIAMPolicy cannot be used in conjunction with gcp.dataproc.JobIAMBinding and gcp.dataproc.JobIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the job as gcp.dataproc.JobIAMPolicy replaces the entire policy.

Note: gcp.dataproc.JobIAMBinding resources can be used in conjunction with gcp.dataproc.JobIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.JobIAMPolicy("editor", new Gcp.Dataproc.JobIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        JobId = "your-dataproc-job",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMBinding("editor", new Gcp.Dataproc.JobIAMBindingArgs
    {
        JobId = "your-dataproc-job",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMMember("editor", new Gcp.Dataproc.JobIAMMemberArgs
    {
        JobId = "your-dataproc-job",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

JobIAMBindingArgs

JobIAMBindingState

JobIAMMember

Three different resources help you manage IAM policies on dataproc jobs. Each of these resources serves a different use case:

• gcp.dataproc.JobIAMPolicy: Authoritative. Sets the IAM policy for the job and replaces any existing policy already attached.
• gcp.dataproc.JobIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the job are preserved.
• gcp.dataproc.JobIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the job are preserved.

Note: gcp.dataproc.JobIAMPolicy cannot be used in conjunction with gcp.dataproc.JobIAMBinding and gcp.dataproc.JobIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the job as gcp.dataproc.JobIAMPolicy replaces the entire policy.

Note: gcp.dataproc.JobIAMBinding resources can be used in conjunction with gcp.dataproc.JobIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.JobIAMPolicy("editor", new Gcp.Dataproc.JobIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        JobId = "your-dataproc-job",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMBinding("editor", new Gcp.Dataproc.JobIAMBindingArgs
    {
        JobId = "your-dataproc-job",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMMember("editor", new Gcp.Dataproc.JobIAMMemberArgs
    {
        JobId = "your-dataproc-job",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

JobIAMMemberArgs

JobIAMMemberState

JobIAMPolicy

Three different resources help you manage IAM policies on dataproc jobs. Each of these resources serves a different use case:

• gcp.dataproc.JobIAMPolicy: Authoritative. Sets the IAM policy for the job and replaces any existing policy already attached.
• gcp.dataproc.JobIAMBinding: Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the job are preserved.
• gcp.dataproc.JobIAMMember: Non-authoritative. Updates the IAM policy to grant a role to a new member. Other members for the role for the job are preserved.

Note: gcp.dataproc.JobIAMPolicy cannot be used in conjunction with gcp.dataproc.JobIAMBinding and gcp.dataproc.JobIAMMember or they will fight over what your policy should be. In addition, be careful not to accidentally unset ownership of the job as gcp.dataproc.JobIAMPolicy replaces the entire policy.

Note: gcp.dataproc.JobIAMBinding resources can be used in conjunction with gcp.dataproc.JobIAMMember resources only if they do not grant privilege to the same role.

google_pubsub_subscription_iam_policy

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var admin = Output.Create(Gcp.Organizations.GetIAMPolicy.InvokeAsync(new Gcp.Organizations.GetIAMPolicyArgs
    {
        Binding = 
        {

            {
                { "role", "roles/editor" },
                { "members", 
                {
                    "user:jane@example.com",
                } },
            },
        },
    }));
    var editor = new Gcp.Dataproc.JobIAMPolicy("editor", new Gcp.Dataproc.JobIAMPolicyArgs
    {
        Project = "your-project",
        Region = "your-region",
        JobId = "your-dataproc-job",
        PolicyData = admin.Apply(admin => admin.PolicyData),
    });
}

}

google_pubsub_subscription_iam_binding

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMBinding("editor", new Gcp.Dataproc.JobIAMBindingArgs
    {
        JobId = "your-dataproc-job",
        Members = 
        {
            "user:jane@example.com",
        },
        Role = "roles/editor",
    });
}

}

google_pubsub_subscription_iam_member

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
public MyStack()
{
    var editor = new Gcp.Dataproc.JobIAMMember("editor", new Gcp.Dataproc.JobIAMMemberArgs
    {
        JobId = "your-dataproc-job",
        Member = "user:jane@example.com",
        Role = "roles/editor",
    });
}

}

JobIAMPolicyArgs

JobIAMPolicyState

JobState