manifests.yaml 2.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596
  1. AWSTemplateFormatVersion: "2010-09-09"
  2. Description: Amazon S3 Find and Forget Manifests stack
  3. Parameters:
  4. JobDetailsRetentionDays:
  5. Type: Number
  6. Conditions:
  7. WithoutRetentionPolicy: !Equals [!Ref JobDetailsRetentionDays, 0]
  8. Resources:
  9. ManifestsBucket:
  10. Type: AWS::S3::Bucket
  11. DeletionPolicy: Retain
  12. UpdateReplacePolicy: Retain
  13. Properties:
  14. VersioningConfiguration:
  15. Status: Enabled
  16. BucketEncryption:
  17. ServerSideEncryptionConfiguration:
  18. - ServerSideEncryptionByDefault:
  19. SSEAlgorithm: AES256
  20. LifecycleConfiguration:
  21. Rules:
  22. - Id: ExpireDeletedManifests
  23. Prefix: manifests/
  24. Status: Enabled
  25. ExpirationInDays: !If
  26. - WithoutRetentionPolicy
  27. - !Ref AWS::NoValue
  28. - !Ref JobDetailsRetentionDays
  29. NoncurrentVersionExpirationInDays: 1
  30. ManifestsBucketPolicy:
  31. Type: AWS::S3::BucketPolicy
  32. Properties:
  33. Bucket: !Ref ManifestsBucket
  34. PolicyDocument:
  35. Statement:
  36. - Sid: HttpsOnly
  37. Action: '*'
  38. Effect: Deny
  39. Resource:
  40. - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}
  41. - !Sub arn:${AWS::Partition}:s3:::${ManifestsBucket}/*
  42. Principal: '*'
  43. Condition:
  44. Bool:
  45. 'aws:SecureTransport': 'false'
  46. GlueDatabase:
  47. Type: AWS::Glue::Database
  48. Properties:
  49. CatalogId: !Ref AWS::AccountId
  50. DatabaseInput:
  51. Description: Database used by S3 Find and Forget Solution
  52. JobManifestsGlueTable:
  53. Type: AWS::Glue::Table
  54. Properties:
  55. CatalogId: !Ref AWS::AccountId
  56. DatabaseName: !Ref GlueDatabase
  57. TableInput:
  58. Description: Table used by S3 Find and Forget Solution
  59. StorageDescriptor:
  60. Columns:
  61. - Name: Columns
  62. Type: array<string>
  63. - Name: MatchId
  64. Type: array<string>
  65. - Name: DeletionQueueItemId
  66. Type: string
  67. - Name: CreatedAt
  68. Type: int
  69. - Name: QueryableColumns
  70. Type: string
  71. - Name: QueryableMatchId
  72. Type: string
  73. Location: !Sub s3://${ManifestsBucket}/manifests/
  74. InputFormat: org.apache.hadoop.mapred.TextInputFormat
  75. OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  76. SerdeInfo:
  77. SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
  78. PartitionKeys:
  79. - Name: JobId
  80. Type: string
  81. - Name: DataMapperId
  82. Type: string
  83. Outputs:
  84. ManifestsBucket:
  85. Value: !Ref ManifestsBucket
  86. GlueDatabase:
  87. Value: !Ref GlueDatabase
  88. JobManifestsGlueTable:
  89. Value: !Ref JobManifestsGlueTable