# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: MIT-0

#

# Permission is hereby granted, free of charge, to any person

# obtaining a copy of this software and associated documentation files

# (the "Software"), to deal in the Software without restriction,

# including without limitation the rights to use, copy, modify, merge,

# publish, distribute, sublicense, and/or sell copies of the Software,

# and to permit persons to whom the Software is furnished to do so.

#

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

# SOFTWARE.

AWSTemplateFormatVersion: '2010-09-09'

Description: |

  This CloudFormation Template deploys Glue jobs and crawlers for TCGA

  data

Parameters:

  ResourcesBucket:

    Type: String

  ResourcePrefix:

    Type: String

  ResourcePrefixLowercase:

    Type: String

  DatabaseName:

    Type: String

    Default: AUTO

    Description: |

      If not AUTO, references an existing Glue database for crawlers

      to create tables in.

  DataLakeBucket:

    Description: |

      S3 bucket where results will be written. Bucketname needs to be

      unique. The bucket name must respect the S3 bucket naming

      conventions (can contain lowercase letters, numbers, periods and

      hyphens).

    Type: String

    AllowedPattern: "((?=^.{3,63}$)(?!^(\\d+\\.)+\\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])$)|(^.{0}$))"

  ExistingBucket:

    Description: Is this an existing bucket?

    Type: String

    AllowedValues:

      - "Yes"

      - "No"

    Default: "No"

Conditions:

  BucketDoesNotExist:

    Fn::Equals:

      - !Ref ExistingBucket

      - "No"

  NeedsGlueDatabase: !Equals [!Ref DatabaseName, "AUTO"]

Resources:

  TCGAS3Bucket:

    Type: AWS::S3::Bucket

    Condition: BucketDoesNotExist

    DeletionPolicy: Retain

    UpdateReplacePolicy: Retain

    Properties:

      BucketName: !Ref DataLakeBucket

      BucketEncryption:

        ServerSideEncryptionConfiguration:

          - ServerSideEncryptionByDefault:

             SSEAlgorithm: AES256

  GlueJobRole:

    Type: AWS::IAM::Role

    Properties:

      AssumeRolePolicyDocument:

        Version: "2012-10-17"

        Statement:

          - Effect: "Allow"

            Principal:

              Service: "glue.amazonaws.com"

            Action: "sts:AssumeRole"

      Path: "/"

      ManagedPolicyArns:

        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole

      Policies:

        - PolicyName: athena_access

          PolicyDocument:

            Version: 2012-10-17

            Statement:

              - Effect: Allow

                Action:

                  - athena:StartQueryExecution

                  - athena:GetQueryExecution

                  - athena:GetQueryResults

                Resource:

                  - !Sub arn:aws:athena:${AWS::Region}:${AWS::AccountId}:workgroup/primary

        - PolicyName: kms_access

          PolicyDocument:

            Version: 2012-10-17

            Statement:

              - Effect: Allow

                Action:

                  - kms:GenerateDataKey

                  - kms:Decrypt

                  - kms:Encrypt

                Resource:

                  - !ImportValue

                      Fn::Sub: '${ResourcePrefix}-DataCatalogEncryptionKeyArn'

        - PolicyName: "CrawlerAccess"

          PolicyDocument:

            Version: "2012-10-17"

            Statement:

              - Effect: "Allow"

                Action:

                  - s3:PutObject

                  - s3:GetObject

                  - s3:ListBucket

                  - s3:DeleteObject

                Resource:

                  - !Sub 'arn:aws:s3:::${DataLakeBucket}'

                  - !Sub 'arn:aws:s3:::${DataLakeBucket}/*'

              - Effect: "Allow"

                Action:

                  - s3:GetObject

                  - s3:ListBucket

                Resource:

                  - !Sub 'arn:aws:s3:::${ResourcesBucket}'

                  - !Sub 'arn:aws:s3:::${ResourcesBucket}/*'

  TcgaLuadExpressionGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUAD"

        "--data_type": "Gene Expression Quantification"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuadMutationGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command:

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUAD"

        "--data_type": "Masked Somatic Mutation"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuadCnvGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUAD"

        "--data_type": "Gene Level Copy Number"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuadClinicalGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUAD"

        "--data_type": "Clinical Supplement"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuadImagingMetadataGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command:

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/image_api_glue.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUAD"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuscExpressionGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUSC"

        "--data_type": "Gene Expression Quantification"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuscMutationGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command:

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUSC"

        "--data_type": "Masked Somatic Mutation"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuscCnvGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUSC"

        "--data_type": "Gene Level Copy Number"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuscClinicalGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command: 

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/tcga_etl_common_job.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUSC"

        "--data_type": "Clinical Supplement"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaLuscImagingMetadataGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command:

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/image_api_glue.py"

      DefaultArguments:

        "--output_bucket": !Ref 'DataLakeBucket'

        "--project": "TCGA-LUSC"

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TcgaSummaryGlueJob:

    Type: AWS::Glue::Job

    Properties:

      Command:

        Name: glueetl

        ScriptLocation: !Sub "s3://${ResourcesBucket}/scripts/create_tcga_summary.py"

      DefaultArguments:

        "--bucket": !Ref 'DataLakeBucket'

        "--workgroup": "primary"  # does not work when set to the solution wg

        "--database": !If

          - NeedsGlueDatabase

          - !Ref TcgaDb

          - !Ref DatabaseName

      GlueVersion: "2.0"

      ExecutionProperty:

        MaxConcurrentRuns: 2

      MaxRetries: 0

      Role: !Ref GlueJobRole

  TCGAMutationCrawler:

    Type: AWS::Glue::Crawler

    Properties: 

      Name: !Sub ${ResourcePrefixLowercase}-tcga-mut

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA mutation data

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['',['s3://',!Ref 'DataLakeBucket','/tcga-mutation']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"

  TCGACNVCrawler:

    Type: AWS::Glue::Crawler

    Properties: 

      Name: !Sub ${ResourcePrefixLowercase}-tcga-cnv

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA copy number data

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['',['s3://',!Ref 'DataLakeBucket','/tcga-cnv']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"

  TCGAExpressionCrawler:

    Type: AWS::Glue::Crawler

    Properties: 

      Name: !Sub ${ResourcePrefixLowercase}-tcga-exp

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA expression data

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['',['s3://',!Ref 'DataLakeBucket','/tcga-expression']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"

  TCGAClinicalCrawler:

    Type: AWS::Glue::Crawler

    Properties: 

      Name: !Sub ${ResourcePrefixLowercase}-tcga-clin

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA clinical data

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['',['s3://',!Ref 'DataLakeBucket','/tcga-clinical']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"

  TCGAImagingMetadataCrawler:

    Type: AWS::Glue::Crawler

    Properties: 

      Name: !Sub ${ResourcePrefixLowercase}-tcga-img

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA imaging metadata

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['',['s3://',!Ref 'DataLakeBucket','/tcia-metadata']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"

  TCGASummaryCrawler:

    Type: AWS::Glue::Crawler

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-sum

      Role: !Ref GlueJobRole

      Description: AWS Glue crawler to crawl TCGA summary data

      DatabaseName: !If

        - NeedsGlueDatabase

        - !Ref TcgaDb

        - !Ref DatabaseName

      Targets:

        S3Targets:

          - Path: !Join ['', ['s3://', !Ref DataLakeBucket, '/tcga-summary']]

      SchemaChangePolicy:

        UpdateBehavior: "UPDATE_IN_DATABASE"

        DeleteBehavior: "LOG"

      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"      

  TCGAWorkflow:

    Type: AWS::Glue::Workflow

    Properties:

      Description: "Workflow that kicks off exp job and crawler"

  WorkflowStartTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-start

      Type: ON_DEMAND

      Description: Trigger for starting the workflow

      Actions:

        - JobName: !Ref TcgaLuadExpressionGlueJob

        - JobName: !Ref TcgaLuadMutationGlueJob

        - JobName: !Ref TcgaLuadCnvGlueJob

        - JobName: !Ref TcgaLuadClinicalGlueJob

        - JobName: !Ref TcgaLuadImagingMetadataGlueJob

        - JobName: !Ref TcgaLuscExpressionGlueJob

        - JobName: !Ref TcgaLuscMutationGlueJob

        - JobName: !Ref TcgaLuscCnvGlueJob

        - JobName: !Ref TcgaLuscClinicalGlueJob

        - JobName: !Ref TcgaLuscImagingMetadataGlueJob

      WorkflowName: !Ref TCGAWorkflow

  ExpCrawlerTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-exp

      Type: "CONDITIONAL"

      Description: "Description for a conditional job trigger"

      Actions:

        - CrawlerName: !Ref 'TCGAExpressionCrawler' 

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuadExpressionGlueJob'

            State: SUCCEEDED

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuscExpressionGlueJob'

            State: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  MutCrawlerTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-mut

      Type: "CONDITIONAL"

      Description: "Description for a conditional Mutation crawler job trigger"

      Actions:

        - CrawlerName: !Ref 'TCGAMutationCrawler' 

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuadMutationGlueJob'

            State: SUCCEEDED

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuscMutationGlueJob'

            State: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  CnvCrawlerTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-cnv

      Type: "CONDITIONAL"

      Description: "Description for a conditional CNV crawler job trigger"

      Actions:

        - CrawlerName: !Ref 'TCGACNVCrawler' 

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuadCnvGlueJob'

            State: SUCCEEDED

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuscCnvGlueJob'

            State: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  ClinCrawlerTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-clin

      Type: "CONDITIONAL"

      Description: "Description for a conditional Clinical crawler job trigger"

      Actions:

        - CrawlerName: !Ref 'TCGAClinicalCrawler' 

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuadClinicalGlueJob'

            State: SUCCEEDED

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuscClinicalGlueJob'

            State: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  ImMetaCrawlerTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-img

      Type: "CONDITIONAL"

      Description: "Description for a conditional Clinical crawler job trigger"

      Actions:

        - CrawlerName: !Ref 'TCGAImagingMetadataCrawler' 

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuadImagingMetadataGlueJob'

            State: SUCCEEDED

          - LogicalOperator: EQUALS

            JobName: !Ref 'TcgaLuscImagingMetadataGlueJob'

            State: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  RunSummaryJobTrigger:

    Type: AWS::Glue::Trigger

    Properties:

      Name: !Sub ${ResourcePrefixLowercase}-tcga-sum

      Type: "CONDITIONAL"

      Description: "Build TCGA summary"

      Actions:

        - JobName: !Ref TcgaSummaryGlueJob

      StartOnCreation: true

      Predicate:

        Logical: AND

        Conditions:

          - LogicalOperator: EQUALS

            CrawlerName: !Ref TCGAMutationCrawler

            CrawlState: SUCCEEDED

          - LogicalOperator: EQUALS

            CrawlerName: !Ref TCGACNVCrawler

            CrawlState: SUCCEEDED

          - LogicalOperator: EQUALS

            CrawlerName: !Ref TCGAExpressionCrawler

            CrawlState: SUCCEEDED

          - LogicalOperator: EQUALS

            CrawlerName: !Ref TCGAClinicalCrawler

            CrawlState: SUCCEEDED

          - LogicalOperator: EQUALS

            CrawlerName: !Ref TCGAImagingMetadataCrawler

            CrawlState: SUCCEEDED

      WorkflowName: !Ref TCGAWorkflow

  TcgaDb:

    Type: AWS::Glue::Database

    Condition: NeedsGlueDatabase

    Properties:

      CatalogId: !Ref AWS::AccountId

      DatabaseInput:

        Description: "AWS Glue container to hold tables for the TCGA crawlers"

Outputs:

  DataLakeBucket:

    Value: !Ref DataLakeBucket

  TCGAWorkflow:

    Value: !Ref TCGAWorkflow

  CreateQuicksightLink:

    Value: !Sub "https://${AWS::Region}.console.aws.amazon.com/cloudformation/home\

      ?region=${AWS::Region}#/stacks/create/review\

      ?templateURL=https://s3.${AWS::Region}.amazonaws.com/${ResourcesBucket}/quicksight_cfn.yml\

      &stackName=${ResourcePrefix}-Quicksight\

      &param_Project=${ResourcePrefix}"