deletion_flow.yaml 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. AWSTemplateFormatVersion: "2010-09-09"
  2. Transform: AWS::Serverless-2016-10-31
  3. Description: Amazon S3 Find and Forget Deletion Flow
  4. Globals:
  5. Function:
  6. Runtime: python3.9
  7. Timeout: 900
  8. Layers: !Ref CommonLayers
  9. Environment:
  10. Variables:
  11. Cluster: !Ref ECSCluster
  12. LogLevel: !Ref LogLevel
  13. Parameters:
  14. CommonLayers:
  15. Type: CommaDelimitedList
  16. LogLevel:
  17. Type: String
  18. Default: INFO
  19. AllowedValues:
  20. - CRITICAL
  21. - FATAL
  22. - ERROR
  23. - WARNING
  24. - INFO
  25. - DEBUG
  26. - NOTSET
  27. DeletionTaskCPU:
  28. Type: String
  29. DeletionTaskMemory:
  30. Type: String
  31. EnableContainerInsights:
  32. Type: String
  33. JobTableName:
  34. Description: Table name for Jobs Table
  35. Type: String
  36. KMSKeyArns:
  37. Type: String
  38. LogRetentionInDays:
  39. Type: Number
  40. Default: 7
  41. ManifestsBucket:
  42. Type: String
  43. ResourcePrefix:
  44. Type: String
  45. VpcSecurityGroups:
  46. Type: CommaDelimitedList
  47. VpcSubnets:
  48. Type: CommaDelimitedList
  49. Conditions:
  50. WithContainerInsights: !Equals [!Ref EnableContainerInsights, "true"]
  51. WithKMS: !Not [!Equals [!Ref KMSKeyArns, ""]]
  52. Resources:
  53. ECSCluster:
  54. Type: AWS::ECS::Cluster
  55. Properties:
  56. ClusterSettings:
  57. - Name: containerInsights
  58. Value: !If
  59. - WithContainerInsights
  60. - enabled
  61. - disabled
  62. ECRRepository:
  63. Type: AWS::ECR::Repository
  64. ECSTaskExecutionRole:
  65. Type: AWS::IAM::Role
  66. Properties:
  67. AssumeRolePolicyDocument:
  68. Statement:
  69. - Effect: Allow
  70. Principal:
  71. Service:
  72. - ecs-tasks.amazonaws.com
  73. Action:
  74. - sts:AssumeRole
  75. Path: /
  76. ManagedPolicyArns:
  77. - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
  78. DelObjQ:
  79. Type: AWS::SQS::Queue
  80. Properties:
  81. ContentBasedDeduplication: true
  82. FifoQueue: true
  83. ReceiveMessageWaitTimeSeconds: 0
  84. KmsMasterKeyId: alias/aws/sqs
  85. VisibilityTimeout: 10800
  86. RedrivePolicy:
  87. deadLetterTargetArn: !GetAtt DLQ.Arn
  88. maxReceiveCount: 1
  89. DelObjQPolicy:
  90. Type: AWS::SQS::QueuePolicy
  91. Properties:
  92. Queues:
  93. - !Ref DelObjQ
  94. PolicyDocument:
  95. Id: FargateConsumerPolicy
  96. Statement:
  97. - Sid: AllowFargateAccess
  98. Effect: Allow
  99. Principal:
  100. AWS:
  101. - !GetAtt DeleteTaskRole.Arn
  102. Action:
  103. - sqs:ChangeMessageVisibility
  104. - sqs:DeleteMessage
  105. - sqs:GetQueueAttributes
  106. - sqs:ReceiveMessage
  107. Resource: !GetAtt DelObjQ.Arn
  108. DeleteTaskDefinition:
  109. Type: AWS::ECS::TaskDefinition
  110. Properties:
  111. TaskRoleArn: !Ref DeleteTaskRole
  112. ExecutionRoleArn: !GetAtt ECSTaskExecutionRole.Arn
  113. NetworkMode: awsvpc
  114. Memory: !Ref DeletionTaskMemory
  115. Cpu: !Ref DeletionTaskCPU
  116. RequiresCompatibilities:
  117. - FARGATE
  118. ContainerDefinitions:
  119. - Name: !Sub ${ResourcePrefix}_DeleteTask
  120. Essential: true
  121. Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/${ECRRepository}:latest
  122. LogConfiguration:
  123. LogDriver: awslogs
  124. Options:
  125. awslogs-group: !Ref DeleteTaskLogGroup
  126. awslogs-region: !Ref 'AWS::Region'
  127. awslogs-stream-prefix: !Ref 'AWS::StackName'
  128. Environment:
  129. - Name: AWS_STS_REGIONAL_ENDPOINTS
  130. Value: regional
  131. - Name: AWS_URL_SUFFIX
  132. Value: !Ref AWS::URLSuffix
  133. - Name: DELETE_OBJECTS_QUEUE
  134. Value: !Ref DelObjQ
  135. - Name: ECS_ENABLE_CONTAINER_METADATA
  136. Value: 'true'
  137. - Name: LOG_LEVEL
  138. Value: !Ref LogLevel
  139. - Name: JobTable
  140. Value: !Ref JobTableName
  141. DeleteService:
  142. Type: AWS::ECS::Service
  143. Properties:
  144. Cluster: !GetAtt ECSCluster.Arn
  145. DesiredCount: 0
  146. LaunchType: FARGATE
  147. PlatformVersion: 1.4.0
  148. NetworkConfiguration:
  149. AwsvpcConfiguration:
  150. SecurityGroups: !Ref VpcSecurityGroups
  151. Subnets: !Ref VpcSubnets
  152. TaskDefinition: !Ref DeleteTaskDefinition
  153. DeleteTaskRole:
  154. Type: AWS::IAM::Role
  155. Properties:
  156. AssumeRolePolicyDocument:
  157. Statement:
  158. - Effect: Allow
  159. Principal:
  160. Service: ecs-tasks.amazonaws.com
  161. Action: sts:AssumeRole
  162. Path: /
  163. Policies:
  164. - PolicyName: ReadWriteExecutionPolicy
  165. PolicyDocument:
  166. Version: "2012-10-17"
  167. Statement:
  168. - Action:
  169. - dynamodb:GetItem
  170. - dynamodb:PutItem
  171. Effect: Allow
  172. Resource: !Sub "arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobTableName}"
  173. - Action:
  174. - logs:DescribeLogStreams
  175. - logs:CreateLogStream
  176. - logs:PutLogEvents
  177. Effect: Allow
  178. Resource: !Sub "arn:${AWS::Partition}:logs:*:*:*"
  179. - Action: sts:AssumeRole
  180. Effect: Allow
  181. Resource: !Sub "arn:${AWS::Partition}:iam::*:role/S3F2DataAccessRole"
  182. - Action: s3:GetObject*
  183. Effect: Allow
  184. Resource: !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/manifests/*
  185. - !If
  186. - WithKMS
  187. - Action:
  188. - kms:Decrypt
  189. - kms:GenerateDataKey
  190. Effect: Allow
  191. Resource: !Split [",", !Ref KMSKeyArns]
  192. - !Ref AWS::NoValue
  193. DeleteTaskLogGroup:
  194. Type: AWS::Logs::LogGroup
  195. Properties:
  196. RetentionInDays: !Ref LogRetentionInDays
  197. DLQ:
  198. Type: AWS::SQS::Queue
  199. Properties:
  200. ContentBasedDeduplication: true
  201. FifoQueue: true
  202. KmsMasterKeyId: alias/aws/sqs
  203. ReceiveMessageWaitTimeSeconds: 0
  204. Outputs:
  205. DeleteObjectsQueueUrl:
  206. Value: !Ref DelObjQ
  207. DeleteServiceName:
  208. Value: !GetAtt DeleteService.Name
  209. DeleteTaskRole:
  210. Value: !Ref DeleteTaskRole
  211. DeleteTaskRoleArn:
  212. Value: !GetAtt DeleteTaskRole.Arn
  213. DLQUrl:
  214. Value: !Ref DLQ
  215. ECSCluster:
  216. Value: !Ref ECSCluster
  217. ECRRepository:
  218. Value: !Ref ECRRepository