|
a |
|
b/source/GenomicsAnalysisCode/code_cfn.yml |
|
|
1 |
AWSTemplateFormatVersion: 2010-09-09 |
|
|
2 |
|
|
|
3 |
Description: GenomicsAnalysisCode |
|
|
4 |
|
|
|
5 |
Parameters: |
|
|
6 |
ResourcePrefix: |
|
|
7 |
Type: String |
|
|
8 |
Default: GenomicsAnalysis |
|
|
9 |
ResourcePrefixLowercase: |
|
|
10 |
Type: String |
|
|
11 |
Default: genomicsanalysis |
|
|
12 |
ResourcesBucket: |
|
|
13 |
Type: String |
|
|
14 |
DataLakeBucket: |
|
|
15 |
Type: String |
|
|
16 |
DatabaseAdministrator: |
|
|
17 |
Type: String |
|
|
18 |
|
|
|
19 |
Resources: |
|
|
20 |
|
|
|
21 |
JobRole: |
|
|
22 |
Type: AWS::IAM::Role |
|
|
23 |
Properties: |
|
|
24 |
AssumeRolePolicyDocument: |
|
|
25 |
Version: 2012-10-17 |
|
|
26 |
Statement: |
|
|
27 |
- Effect: Allow |
|
|
28 |
Principal: |
|
|
29 |
Service: |
|
|
30 |
- glue.amazonaws.com |
|
|
31 |
Action: |
|
|
32 |
- sts:AssumeRole |
|
|
33 |
Path: / |
|
|
34 |
ManagedPolicyArns: |
|
|
35 |
- arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole |
|
|
36 |
Policies: |
|
|
37 |
- PolicyName: s3_access |
|
|
38 |
PolicyDocument: |
|
|
39 |
Version: 2012-10-17 |
|
|
40 |
Statement: |
|
|
41 |
- Effect: Allow |
|
|
42 |
Action: |
|
|
43 |
- athena:StartQueryExecution |
|
|
44 |
- athena:GetQueryExecution |
|
|
45 |
- athena:GetQueryResults |
|
|
46 |
Resource: |
|
|
47 |
- !Sub arn:aws:athena:${AWS::Region}:${AWS::AccountId}* |
|
|
48 |
- Effect: Allow |
|
|
49 |
Action: |
|
|
50 |
- s3:GetObject |
|
|
51 |
- s3:ListBucket |
|
|
52 |
Resource: |
|
|
53 |
- !Sub arn:aws:s3:::${ResourcesBucket} |
|
|
54 |
- !Sub arn:aws:s3:::${ResourcesBucket}/* |
|
|
55 |
- Effect: Allow |
|
|
56 |
Action: |
|
|
57 |
- s3:PutObject |
|
|
58 |
- s3:GetObject |
|
|
59 |
- s3:ListBucket |
|
|
60 |
- s3:DeleteObject |
|
|
61 |
Resource: |
|
|
62 |
- !Sub arn:aws:s3:::${DataLakeBucket} |
|
|
63 |
- !Sub arn:aws:s3:::${DataLakeBucket}/* |
|
|
64 |
- PolicyName: kms_access |
|
|
65 |
PolicyDocument: |
|
|
66 |
Version: 2012-10-17 |
|
|
67 |
Statement: |
|
|
68 |
- Effect: Allow |
|
|
69 |
Action: |
|
|
70 |
- kms:GenerateDataKey |
|
|
71 |
- kms:Decrypt |
|
|
72 |
- kms:Encrypt |
|
|
73 |
Resource: |
|
|
74 |
- !GetAtt DataCatalogEncryptionKey.Arn |
|
|
75 |
|
|
|
76 |
RunbookRole: |
|
|
77 |
Type: AWS::IAM::Role |
|
|
78 |
Properties: |
|
|
79 |
AssumeRolePolicyDocument: |
|
|
80 |
Version: 2012-10-17 |
|
|
81 |
Statement: |
|
|
82 |
- Effect: Allow |
|
|
83 |
Principal: |
|
|
84 |
Service: |
|
|
85 |
- sagemaker.amazonaws.com |
|
|
86 |
Action: |
|
|
87 |
- sts:AssumeRole |
|
|
88 |
Path: / |
|
|
89 |
Policies: |
|
|
90 |
- PolicyName: logs_access |
|
|
91 |
PolicyDocument: |
|
|
92 |
Version: 2012-10-17 |
|
|
93 |
Statement: |
|
|
94 |
- Effect: Allow |
|
|
95 |
Action: |
|
|
96 |
- logs:CreateLogStream |
|
|
97 |
- logs:DescribeLogStreams |
|
|
98 |
- logs:CreateLogGroup |
|
|
99 |
- logs:PutLogEvents |
|
|
100 |
Resource: |
|
|
101 |
- !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/sagemaker/* |
|
|
102 |
- !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/sagemaker/*:log-stream:aws-glue-* |
|
|
103 |
- PolicyName: s3_access |
|
|
104 |
PolicyDocument: |
|
|
105 |
Version: 2012-10-17 |
|
|
106 |
Statement: |
|
|
107 |
- Effect: Allow |
|
|
108 |
Action: |
|
|
109 |
- s3:ListBucket |
|
|
110 |
- s3:GetBucketLocation |
|
|
111 |
Resource: |
|
|
112 |
- !Sub arn:aws:s3:::${DataLakeBucket} |
|
|
113 |
- !Sub arn:aws:s3:::${ResourcesBucket} |
|
|
114 |
- Effect: Allow |
|
|
115 |
Action: |
|
|
116 |
- s3:GetObject |
|
|
117 |
- s3:GetObjectAcl |
|
|
118 |
- s3:PutObject |
|
|
119 |
- s3:DeleteObject |
|
|
120 |
Resource: |
|
|
121 |
- !Sub arn:aws:s3:::${DataLakeBucket}/* |
|
|
122 |
- Effect: Allow |
|
|
123 |
Action: |
|
|
124 |
- s3:GetObject |
|
|
125 |
Resource: |
|
|
126 |
- !Sub arn:aws:s3:::${ResourcesBucket}/* |
|
|
127 |
- PolicyName: glue_access |
|
|
128 |
PolicyDocument: |
|
|
129 |
Version: 2012-10-17 |
|
|
130 |
Statement: |
|
|
131 |
- Effect: Allow |
|
|
132 |
Action: |
|
|
133 |
- glue:StartCrawler |
|
|
134 |
- glue:StartJobRun |
|
|
135 |
- glue:StartTrigger |
|
|
136 |
Resource: |
|
|
137 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:crawler/${ResourcePrefixLowercase}* |
|
|
138 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:job/${ResourcePrefixLowercase}* |
|
|
139 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:trigger/${ResourcePrefixLowercase}* |
|
|
140 |
- Effect: Allow |
|
|
141 |
Action: |
|
|
142 |
- kms:GenerateDataKey |
|
|
143 |
- kms:Decrypt |
|
|
144 |
- kms:Encrypt |
|
|
145 |
Resource: |
|
|
146 |
- !GetAtt DataCatalogEncryptionKey.Arn |
|
|
147 |
- PolicyName: glue_table_access |
|
|
148 |
PolicyDocument: |
|
|
149 |
Version: 2012-10-17 |
|
|
150 |
Statement: |
|
|
151 |
- Effect: Allow |
|
|
152 |
Action: |
|
|
153 |
- glue:GetDatabases |
|
|
154 |
- glue:GetDatabase |
|
|
155 |
- glue:GetTables |
|
|
156 |
- glue:GetTable |
|
|
157 |
- lakeformation:GetDataAccess |
|
|
158 |
Resource: '*' |
|
|
159 |
- Effect: Allow |
|
|
160 |
Action: |
|
|
161 |
- glue:CreateDatabase |
|
|
162 |
Resource: |
|
|
163 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/default |
|
|
164 |
- Effect: Allow |
|
|
165 |
Action: |
|
|
166 |
- glue:GetTable |
|
|
167 |
- glue:GetTables |
|
|
168 |
- glue:CreateTable |
|
|
169 |
- glue:UpdateTable |
|
|
170 |
- glue:DeleteTable |
|
|
171 |
- glue:GetDatabase |
|
|
172 |
- glue:GetPartition |
|
|
173 |
- glue:GetPartitions |
|
|
174 |
- glue:GetDevEndpoint |
|
|
175 |
- glue:GetDevEndpoints |
|
|
176 |
- glue:UpdateDevEndpoint |
|
|
177 |
Resource: |
|
|
178 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog |
|
|
179 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/${ResourcePrefixLowercase} |
|
|
180 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefixLowercase}/* |
|
|
181 |
- !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:devEndpoint/* |
|
|
182 |
- PolicyName: athena_access |
|
|
183 |
PolicyDocument: |
|
|
184 |
Version: 2012-10-17 |
|
|
185 |
Statement: |
|
|
186 |
- Effect: Allow |
|
|
187 |
Action: |
|
|
188 |
- athena:StartQueryExecution |
|
|
189 |
- athena:GetQueryExecution |
|
|
190 |
- athena:GetQueryResults |
|
|
191 |
- athena:GetWorkGroup |
|
|
192 |
Resource: |
|
|
193 |
- !Sub arn:aws:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${ResourcePrefixLowercase}-${AWS::Region} |
|
|
194 |
|
|
|
195 |
- PolicyName: cfn_access |
|
|
196 |
PolicyDocument: |
|
|
197 |
Version: 2012-10-17 |
|
|
198 |
Statement: |
|
|
199 |
- Effect: Allow |
|
|
200 |
Action: |
|
|
201 |
- cloudformation:DescribeStacks |
|
|
202 |
Resource: |
|
|
203 |
- !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${ResourcePrefix}* |
|
|
204 |
- PolicyName: kms_access |
|
|
205 |
PolicyDocument: |
|
|
206 |
Version: 2012-10-17 |
|
|
207 |
Statement: |
|
|
208 |
- Effect: Allow |
|
|
209 |
Action: |
|
|
210 |
- kms:GenerateDataKey |
|
|
211 |
- kms:Decrypt |
|
|
212 |
- kms:Encrypt |
|
|
213 |
Resource: |
|
|
214 |
- !GetAtt DataCatalogEncryptionKey.Arn |
|
|
215 |
|
|
|
216 |
WorkGroup: |
|
|
217 |
Type: AWS::Athena::WorkGroup |
|
|
218 |
Properties: |
|
|
219 |
Description: !Sub ${ResourcePrefixLowercase} |
|
|
220 |
Name: !Sub ${ResourcePrefixLowercase}-${AWS::Region} |
|
|
221 |
RecursiveDeleteOption: True |
|
|
222 |
WorkGroupConfiguration: |
|
|
223 |
EngineVersion: |
|
|
224 |
EffectiveEngineVersion: "Athena engine version 3" |
|
|
225 |
SelectedEngineVersion: "Athena engine version 3" |
|
|
226 |
ResultConfiguration: |
|
|
227 |
OutputLocation: !Sub s3://${DataLakeBucket}/results |
|
|
228 |
|
|
|
229 |
|
|
|
230 |
DataCatalogEncryptionKey: |
|
|
231 |
DeletionPolicy: Retain |
|
|
232 |
Type: AWS::KMS::Key |
|
|
233 |
Properties: |
|
|
234 |
Description: KMS key used to encrypt the Glue data catalog |
|
|
235 |
Enabled: True |
|
|
236 |
EnableKeyRotation: True |
|
|
237 |
KeyPolicy: !Sub | |
|
|
238 |
{ |
|
|
239 |
"Version": "2012-10-17", |
|
|
240 |
"Id": "TestGlueCatalogEncryptionKeyPolicy", |
|
|
241 |
"Statement": [ |
|
|
242 |
{ |
|
|
243 |
"Sid": "Enable IAM User Permissions", |
|
|
244 |
"Effect": "Allow", |
|
|
245 |
"Principal": { |
|
|
246 |
"AWS": [ |
|
|
247 |
"arn:aws:iam::${AWS::AccountId}:root", |
|
|
248 |
"${DatabaseAdministrator}" |
|
|
249 |
] |
|
|
250 |
}, |
|
|
251 |
"Action": "kms:*", |
|
|
252 |
"Resource": "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/*" |
|
|
253 |
}, |
|
|
254 |
{ |
|
|
255 |
"Sid": "Allow access for Key Administrators", |
|
|
256 |
"Effect": "Allow", |
|
|
257 |
"Principal": { |
|
|
258 |
"AWS": [ |
|
|
259 |
"arn:aws:iam::${AWS::AccountId}:root", |
|
|
260 |
"${DatabaseAdministrator}" |
|
|
261 |
] |
|
|
262 |
}, |
|
|
263 |
"Action": [ |
|
|
264 |
"kms:Create*", |
|
|
265 |
"kms:Describe*", |
|
|
266 |
"kms:Enable*", |
|
|
267 |
"kms:List*", |
|
|
268 |
"kms:Put*", |
|
|
269 |
"kms:Update*", |
|
|
270 |
"kms:Revoke*", |
|
|
271 |
"kms:Disable*", |
|
|
272 |
"kms:Get*", |
|
|
273 |
"kms:Delete*", |
|
|
274 |
"kms:TagResource", |
|
|
275 |
"kms:UntagResource", |
|
|
276 |
"kms:ScheduleKeyDeletion", |
|
|
277 |
"kms:CancelKeyDeletion" |
|
|
278 |
], |
|
|
279 |
"Resource": "*" |
|
|
280 |
}, |
|
|
281 |
{ |
|
|
282 |
"Sid": "Allow use of the key", |
|
|
283 |
"Effect": "Allow", |
|
|
284 |
"Principal": { |
|
|
285 |
"Service": "logs.${AWS::Region}.amazonaws.com" |
|
|
286 |
}, |
|
|
287 |
"Action": [ |
|
|
288 |
"kms:Encrypt", |
|
|
289 |
"kms:Decrypt", |
|
|
290 |
"kms:ReEncrypt*", |
|
|
291 |
"kms:GenerateDataKey*", |
|
|
292 |
"kms:DescribeKey" |
|
|
293 |
], |
|
|
294 |
"Resource": "*" |
|
|
295 |
}, |
|
|
296 |
{ |
|
|
297 |
"Sid": "Allow use of the key", |
|
|
298 |
"Effect": "Allow", |
|
|
299 |
"Principal": "*", |
|
|
300 |
"Action": [ |
|
|
301 |
"kms:Encrypt", |
|
|
302 |
"kms:Decrypt", |
|
|
303 |
"kms:ReEncrypt*", |
|
|
304 |
"kms:GenerateDataKey*", |
|
|
305 |
"kms:DescribeKey" |
|
|
306 |
], |
|
|
307 |
"Resource": "*", |
|
|
308 |
"Condition": { |
|
|
309 |
"ArnEquals": { |
|
|
310 |
"aws:PrincipalARN": "arn:aws:iam::${AWS::AccountId}:role/${ResourcePrefix}*" |
|
|
311 |
} |
|
|
312 |
} |
|
|
313 |
} |
|
|
314 |
] |
|
|
315 |
} |
|
|
316 |
|
|
|
317 |
DataCatalogEncryptionSettings: |
|
|
318 |
Type: AWS::Glue::DataCatalogEncryptionSettings |
|
|
319 |
DependsOn: DataCatalogEncryptionKey |
|
|
320 |
Properties: |
|
|
321 |
CatalogId: !Ref AWS::AccountId |
|
|
322 |
DataCatalogEncryptionSettings: |
|
|
323 |
EncryptionAtRest: |
|
|
324 |
CatalogEncryptionMode: SSE-KMS |
|
|
325 |
SseAwsKmsKeyId: !Ref DataCatalogEncryptionKey |
|
|
326 |
|
|
|
327 |
SecurityConfiguration: |
|
|
328 |
Type: AWS::Glue::SecurityConfiguration |
|
|
329 |
Properties: |
|
|
330 |
EncryptionConfiguration: |
|
|
331 |
CloudWatchEncryption: |
|
|
332 |
CloudWatchEncryptionMode: SSE-KMS |
|
|
333 |
KmsKeyArn: !GetAtt DataCatalogEncryptionKey.Arn |
|
|
334 |
JobBookmarksEncryption: |
|
|
335 |
JobBookmarksEncryptionMode: CSE-KMS |
|
|
336 |
KmsKeyArn: !GetAtt DataCatalogEncryptionKey.Arn |
|
|
337 |
S3Encryptions: |
|
|
338 |
- S3EncryptionMode: SSE-KMS |
|
|
339 |
KmsKeyArn: !GetAtt DataCatalogEncryptionKey.Arn |
|
|
340 |
Name: !Sub ${ResourcePrefix}SecurityConfiguration |
|
|
341 |
|
|
|
342 |
DataCatalog: |
|
|
343 |
Type: AWS::Glue::Database |
|
|
344 |
DependsOn: DataCatalogEncryptionKey |
|
|
345 |
Properties: |
|
|
346 |
CatalogId: !Ref AWS::AccountId |
|
|
347 |
DatabaseInput: |
|
|
348 |
Name: !Sub ${ResourcePrefixLowercase} |
|
|
349 |
Description: Data catalog for Human NGS Tertiary Analysis and Data Lakes solution |
|
|
350 |
|
|
|
351 |
RunbookLifecycle: |
|
|
352 |
Type: AWS::SageMaker::NotebookInstanceLifecycleConfig |
|
|
353 |
Properties: |
|
|
354 |
NotebookInstanceLifecycleConfigName: !Sub ${ResourcePrefixLowercase}Runbook |
|
|
355 |
OnStart: |
|
|
356 |
- Content: !Base64 |
|
|
357 |
Fn::Sub: | |
|
|
358 |
#!/bin/bash |
|
|
359 |
cd /home/ec2-user/SageMaker |
|
|
360 |
set -e |
|
|
361 |
aws s3 sync s3://${ResourcesBucket}/notebooks . |
|
|
362 |
chmod 666 *.ipynb |
|
|
363 |
echo "export RESOURCE_PREFIX='${ResourcePrefix}'" > /home/ec2-user/anaconda3/envs/python3/etc/conda/activate.d/env_vars.sh |
|
|
364 |
|
|
|
365 |
Runbook: |
|
|
366 |
Type: AWS::SageMaker::NotebookInstance |
|
|
367 |
Properties: |
|
|
368 |
NotebookInstanceName: !Sub ${ResourcePrefixLowercase}Runbook |
|
|
369 |
InstanceType: ml.t2.medium |
|
|
370 |
LifecycleConfigName: !GetAtt RunbookLifecycle.NotebookInstanceLifecycleConfigName |
|
|
371 |
RoleArn: !GetAtt RunbookRole.Arn |
|
|
372 |
PlatformIdentifier: notebook-al2-v2 |
|
|
373 |
|
|
|
374 |
|
|
|
375 |
Outputs: |
|
|
376 |
DataCatalogEncryptionKeyArn: |
|
|
377 |
Value: !GetAtt DataCatalogEncryptionKey.Arn |
|
|
378 |
Export: |
|
|
379 |
Name: !Sub "${ResourcePrefix}-DataCatalogEncryptionKeyArn" |