Namespace Pulumi.Aws.Glue

Classes

CatalogDatabase

Provides a Glue Catalog Database Resource. You can refer to the Glue Developer Guide for a full explanation of the Glue Data Catalog functionality

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var awsGlueCatalogDatabase = new Aws.Glue.CatalogDatabase("awsGlueCatalogDatabase", new Aws.Glue.CatalogDatabaseArgs
    {
        Name = "MyCatalogDatabase",
    });
}

}

CatalogDatabaseArgs

CatalogDatabaseState

CatalogTable

Provides a Glue Catalog Table Resource. You can refer to the Glue Developer Guide for a full explanation of the Glue Data Catalog functionality.

Example Usage

Basic Table

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var awsGlueCatalogTable = new Aws.Glue.CatalogTable("awsGlueCatalogTable", new Aws.Glue.CatalogTableArgs
    {
        DatabaseName = "MyCatalogDatabase",
        Name = "MyCatalogTable",
    });
}

}

Parquet Table for Athena

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var awsGlueCatalogTable = new Aws.Glue.CatalogTable("awsGlueCatalogTable", new Aws.Glue.CatalogTableArgs
    {
        DatabaseName = "MyCatalogDatabase",
        Name = "MyCatalogTable",
        Parameters = 
        {
            { "EXTERNAL", "TRUE" },
            { "parquet.compression", "SNAPPY" },
        },
        StorageDescriptor = new Aws.Glue.Inputs.CatalogTableStorageDescriptorArgs
        {
            Columns = 
            {
                new Aws.Glue.Inputs.CatalogTableStorageDescriptorColumnArgs
                {
                    Name = "my_string",
                    Type = "string",
                },
                new Aws.Glue.Inputs.CatalogTableStorageDescriptorColumnArgs
                {
                    Name = "my_double",
                    Type = "double",
                },
                new Aws.Glue.Inputs.CatalogTableStorageDescriptorColumnArgs
                {
                    Comment = "",
                    Name = "my_date",
                    Type = "date",
                },
                new Aws.Glue.Inputs.CatalogTableStorageDescriptorColumnArgs
                {
                    Comment = "",
                    Name = "my_bigint",
                    Type = "bigint",
                },
                new Aws.Glue.Inputs.CatalogTableStorageDescriptorColumnArgs
                {
                    Comment = "",
                    Name = "my_struct",
                    Type = "struct<my_nested_string:string>",
                },
            },
            InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
            Location = "s3://my-bucket/event-streams/my-stream",
            OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
            SerDeInfo = new Aws.Glue.Inputs.CatalogTableStorageDescriptorSerDeInfoArgs
            {
                Name = "my-stream",
                Parameters = 
                {
                    { "serialization.format", 1 },
                },
                SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
            },
        },
        TableType = "EXTERNAL_TABLE",
    });
}

}

CatalogTableArgs

CatalogTableState

Classifier

Provides a Glue Classifier resource.

NOTE: It is only valid to create one type of classifier (csv, grok, JSON, or XML). Changing classifier types will recreate the classifier.

Example Usage

Csv Classifier

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Classifier("example", new Aws.Glue.ClassifierArgs
    {
        CsvClassifier = new Aws.Glue.Inputs.ClassifierCsvClassifierArgs
        {
            AllowSingleColumn = false,
            ContainsHeader = "PRESENT",
            Delimiter = ",",
            DisableValueTrimming = false,
            Header = 
            {
                "example1",
                "example2",
            },
            QuoteSymbol = "'",
        },
    });
}

}

Grok Classifier

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Classifier("example", new Aws.Glue.ClassifierArgs
    {
        GrokClassifier = new Aws.Glue.Inputs.ClassifierGrokClassifierArgs
        {
            Classification = "example",
            GrokPattern = "example",
        },
    });
}

}

JSON Classifier

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Classifier("example", new Aws.Glue.ClassifierArgs
    {
        JsonClassifier = new Aws.Glue.Inputs.ClassifierJsonClassifierArgs
        {
            JsonPath = "example",
        },
    });
}

}

XML Classifier

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Classifier("example", new Aws.Glue.ClassifierArgs
    {
        XmlClassifier = new Aws.Glue.Inputs.ClassifierXmlClassifierArgs
        {
            Classification = "example",
            RowTag = "example",
        },
    });
}

}

ClassifierArgs

ClassifierState

Connection

Provides a Glue Connection resource.

Example Usage

Non-VPC Connection

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Connection("example", new Aws.Glue.ConnectionArgs
    {
        ConnectionProperties = 
        {
            { "JDBC_CONNECTION_URL", "jdbc:mysql://example.com/exampledatabase" },
            { "PASSWORD", "examplepassword" },
            { "USERNAME", "exampleusername" },
        },
    });
}

}

VPC Connection

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Connection("example", new Aws.Glue.ConnectionArgs
    {
        ConnectionProperties = 
        {
            { "JDBC_CONNECTION_URL", $"jdbc:mysql://{aws_rds_cluster.Example.Endpoint}/exampledatabase" },
            { "PASSWORD", "examplepassword" },
            { "USERNAME", "exampleusername" },
        },
        PhysicalConnectionRequirements = new Aws.Glue.Inputs.ConnectionPhysicalConnectionRequirementsArgs
        {
            AvailabilityZone = aws_subnet.Example.Availability_zone,
            SecurityGroupIdList = 
            {
                aws_security_group.Example.Id,
            },
            SubnetId = aws_subnet.Example.Id,
        },
    });
}

}

ConnectionArgs

ConnectionState

Crawler

Manages a Glue Crawler. More information can be found in the AWS Glue Developer Guide

Example Usage

DynamoDB Target

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Crawler("example", new Aws.Glue.CrawlerArgs
    {
        DatabaseName = aws_glue_catalog_database.Example.Name,
        DynamodbTargets = 
        {
            new Aws.Glue.Inputs.CrawlerDynamodbTargetArgs
            {
                Path = "table-name",
            },
        },
        Role = aws_iam_role.Example.Arn,
    });
}

}

JDBC Target

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Crawler("example", new Aws.Glue.CrawlerArgs
    {
        DatabaseName = aws_glue_catalog_database.Example.Name,
        JdbcTargets = 
        {
            new Aws.Glue.Inputs.CrawlerJdbcTargetArgs
            {
                ConnectionName = aws_glue_connection.Example.Name,
                Path = "database-name/%",
            },
        },
        Role = aws_iam_role.Example.Arn,
    });
}

}

S3 Target

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Crawler("example", new Aws.Glue.CrawlerArgs
    {
        DatabaseName = aws_glue_catalog_database.Example.Name,
        Role = aws_iam_role.Example.Arn,
        S3Targets = 
        {
            new Aws.Glue.Inputs.CrawlerS3TargetArgs
            {
                Path = $"s3://{aws_s3_bucket.Example.Bucket}",
            },
        },
    });
}

}

Catalog Target

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Crawler("example", new Aws.Glue.CrawlerArgs
    {
        CatalogTargets = 
        {
            new Aws.Glue.Inputs.CrawlerCatalogTargetArgs
            {
                DatabaseName = aws_glue_catalog_database.Example.Name,
                Tables = 
                {
                    aws_glue_catalog_table.Example.Name,
                },
            },
        },
        Configuration = @"{
""Version"":1.0,
""Grouping"": {
""TableGroupingPolicy"": ""CombineCompatibleSchemas""
}
}

",
        DatabaseName = aws_glue_catalog_database.Example.Name,
        Role = aws_iam_role.Example.Arn,
        SchemaChangePolicy = new Aws.Glue.Inputs.CrawlerSchemaChangePolicyArgs
        {
            DeleteBehavior = "LOG",
        },
    });
}

}

CrawlerArgs

CrawlerState

GetScript

GetScriptArgs

GetScriptResult

Job

Provides a Glue Job resource.

Glue functionality, such as monitoring and logging of jobs, is typically managed with the default_arguments argument. See the Special Parameters Used by AWS Glue topic in the Glue developer guide for additional information.

Example Usage

Python Job

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Job("example", new Aws.Glue.JobArgs
    {
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{aws_s3_bucket.Example.Bucket}/example.py",
        },
        RoleArn = aws_iam_role.Example.Arn,
    });
}

}

Scala Job

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Job("example", new Aws.Glue.JobArgs
    {
        Command = new Aws.Glue.Inputs.JobCommandArgs
        {
            ScriptLocation = $"s3://{aws_s3_bucket.Example.Bucket}/example.scala",
        },
        DefaultArguments = 
        {
            { "--job-language", "scala" },
        },
        RoleArn = aws_iam_role.Example.Arn,
    });
}

}

Enabling CloudWatch Logs and Metrics

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var exampleLogGroup = new Aws.CloudWatch.LogGroup("exampleLogGroup", new Aws.CloudWatch.LogGroupArgs
    {
        RetentionInDays = 14,
    });
    var exampleJob = new Aws.Glue.Job("exampleJob", new Aws.Glue.JobArgs
    {
        DefaultArguments = 
        {
            { "--continuous-log-logGroup", exampleLogGroup.Name },
            { "--enable-continuous-cloudwatch-log", "true" },
            { "--enable-continuous-log-filter", "true" },
            { "--enable-metrics", "" },
        },
    });
}

}

JobArgs

JobState

SecurityConfiguration

Manages a Glue Security Configuration.

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.SecurityConfiguration("example", new Aws.Glue.SecurityConfigurationArgs
    {
        EncryptionConfiguration = new Aws.Glue.Inputs.SecurityConfigurationEncryptionConfigurationArgs
        {
            CloudwatchEncryption = new Aws.Glue.Inputs.SecurityConfigurationEncryptionConfigurationCloudwatchEncryptionArgs
            {
                CloudwatchEncryptionMode = "DISABLED",
            },
            JobBookmarksEncryption = new Aws.Glue.Inputs.SecurityConfigurationEncryptionConfigurationJobBookmarksEncryptionArgs
            {
                JobBookmarksEncryptionMode = "DISABLED",
            },
            S3Encryption = new Aws.Glue.Inputs.SecurityConfigurationEncryptionConfigurationS3EncryptionArgs
            {
                KmsKeyArn = data.Aws_kms_key.Example.Arn,
                S3EncryptionMode = "SSE-KMS",
            },
        },
    });
}

}

SecurityConfigurationArgs

SecurityConfigurationState

Trigger

Manages a Glue Trigger resource.

Example Usage

Conditional Trigger

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Trigger("example", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = aws_glue_job.Example1.Name,
            },
        },
        Predicate = new Aws.Glue.Inputs.TriggerPredicateArgs
        {
            Conditions = 
            {
                new Aws.Glue.Inputs.TriggerPredicateConditionArgs
                {
                    JobName = aws_glue_job.Example2.Name,
                    State = "SUCCEEDED",
                },
            },
        },
        Type = "CONDITIONAL",
    });
}

}

On-Demand Trigger

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Trigger("example", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = aws_glue_job.Example.Name,
            },
        },
        Type = "ON_DEMAND",
    });
}

}

Scheduled Trigger

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Trigger("example", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = aws_glue_job.Example.Name,
            },
        },
        Schedule = "cron(15 12 * * ? *)",
        Type = "SCHEDULED",
    });
}

}

Conditional Trigger with Crawler Action

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Trigger("example", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                CrawlerName = aws_glue_crawler.Example1.Name,
            },
        },
        Predicate = new Aws.Glue.Inputs.TriggerPredicateArgs
        {
            Conditions = 
            {
                new Aws.Glue.Inputs.TriggerPredicateConditionArgs
                {
                    JobName = aws_glue_job.Example2.Name,
                    State = "SUCCEEDED",
                },
            },
        },
        Type = "CONDITIONAL",
    });
}

}

Conditional Trigger with Crawler Condition

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Trigger("example", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = aws_glue_job.Example1.Name,
            },
        },
        Predicate = new Aws.Glue.Inputs.TriggerPredicateArgs
        {
            Conditions = 
            {
                new Aws.Glue.Inputs.TriggerPredicateConditionArgs
                {
                    CrawlState = "SUCCEEDED",
                    CrawlerName = aws_glue_crawler.Example2.Name,
                },
            },
        },
        Type = "CONDITIONAL",
    });
}

}

TriggerArgs

TriggerState

Workflow

Provides a Glue Workflow resource. The workflow graph (DAG) can be build using the aws.glue.Trigger resource. See the example below for creating a graph with four nodes (two triggers and two jobs).

Example Usage

using Pulumi;
using Aws = Pulumi.Aws;

class MyStack : Stack
{
public MyStack()
{
    var example = new Aws.Glue.Workflow("example", new Aws.Glue.WorkflowArgs
    {
    });
    var example_start = new Aws.Glue.Trigger("example-start", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = "example-job",
            },
        },
        Type = "ON_DEMAND",
        WorkflowName = example.Name,
    });
    var example_inner = new Aws.Glue.Trigger("example-inner", new Aws.Glue.TriggerArgs
    {
        Actions = 
        {
            new Aws.Glue.Inputs.TriggerActionArgs
            {
                JobName = "another-example-job",
            },
        },
        Predicate = new Aws.Glue.Inputs.TriggerPredicateArgs
        {
            Conditions = 
            {
                new Aws.Glue.Inputs.TriggerPredicateConditionArgs
                {
                    JobName = "example-job",
                    State = "SUCCEEDED",
                },
            },
        },
        Type = "CONDITIONAL",
        WorkflowName = example.Name,
    });
}

}

WorkflowArgs

WorkflowState