Merge branch 'master' of github.com:awslabs/amazon-textract-document-understanding-solution

Author: George Price

deployment/custom-deployment/bin/stack-variables-to-client.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source .env
+
+#STACK
+
+echo "==update %%CLIENT_APP_BUCKET%% in stack with $2=="
+replace="s/%%CLIENT_APP_BUCKET%%/$ClientAppBucketName/g"
+sed -i -e $replace ./lib/cdk-textract-client-stack.js
+
+echo "==update Elastic Search Cluster ($ElasticSearchCluster) with log streams to Log Groups: $ElasticSearchSearchLogGroup and $ElasticSearchIndexLogGroup"
+INDEX_LOG_ARN=$(aws logs describe-log-groups --log-group-name $ElasticSearchIndexLogGroup | jq -r '.logGroups[0].arn')
+SEARCH_LOG_ARN=$(aws logs describe-log-groups --log-group-name $ElasticSearchSearchLogGroup | jq -r '.logGroups[0].arn')
+
+echo "==adding permissions to ES service role first for creating log stream"
+aws logs put-resource-policy --policy-name es-to-log-stream --policy-document '{ "Version": "2012-10-17", "Statement": [ { "Sid": "ElasticSearchLogsToCloudWatchLogs", "Effect": "Allow", "Principal": { "Service": [ "es.amazonaws.com" ] }, "Action":["logs:PutLogEvents", "logs:CreateLogStream", "logs:DeleteLogStream"], "Resource": "*" } ] }'
+
+echo "==Log Groups are $INDEX_LOG_ARN and $SEARCH_LOG_ARN"
+aws es update-elasticsearch-domain-config --domain-name $ElasticSearchCluster --log-publishing-options '{"INDEX_SLOW_LOGS": { "CloudWatchLogsLogGroupArn": "'"$INDEX_LOG_ARN"'", "Enabled": true }, "SEARCH_SLOW_LOGS": { "CloudWatchLogsLogGroupArn": "'"$SEARCH_LOG_ARN"'", "Enabled": true } }'
+
+

deployment/custom-deployment/bin/update-es-logs-and-client-stack-vars.sh

@@ -170,6 +170,10 @@ Resources:
               commands:
                 - echo "This buildspec is based on image - aws/codebuild/amazonlinux2-x86_64-standard:2.0"
                 - node --version
+                - echo "Installing jq package"
+                - wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+                - chmod +x ./jq
+                - cp jq /usr/bin
                 - npm install -g cdk@1.18.0
                 - cdk --version
                 - npm install -g typescript
@@ -541,9 +545,11 @@ Resources:
                         "Action": [
                           "logs:CreateLogGroup",
                           "logs:CreateLogStream",
+                          "logs:DeleteLogStream",
                           "logs:PutLogEvents",
                           "logs:Describe*",
-                          "logs:PutRetentionPolicy"
+                          "logs:PutRetentionPolicy",
+                          "logs:PutResourcePolicy"
                         ]
                     },
                     {
@@ -672,7 +678,8 @@ Resources:
                           "es:CreateElasticsearchServiceRole",
                           "es:DeleteElasticsearchDomain",
                           "es:DeleteElasticsearchServiceRole",
-                          "es:List*"
+                          "es:List*",
+                          "es:UpdateElasticsearchDomainConfig"
                         ]
                     },
                     {
@@ -686,7 +693,10 @@ Resources:
                           "iam:Get*",
                           "iam:AttachRolePolicy",
                           "iam:PutRolePolicy",
-                          "iam:CreateServiceLinkedRole"
+                          "iam:DeleteRolePolicy",
+                          "iam:DetachRolePolicy",
+                          "iam:CreateServiceLinkedRole",
+                          "iam:DeleteServiceLinkedRole"
                         ]
                     },
                     {

deployment/document-understanding-solution.template

@@ -1,4 +1,3 @@
-
 /**********************************************************************************************************************
  *  Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.                                           *
  *                                                                                                                    *
@@ -118,6 +117,15 @@ const GetResources = new Promise((resolve, reject) => {
     resources.PdfGenLambda = stackDescriptionObj.find((x) =>
       /pdfgenerator/i.test(x.LogicalResourceId)
     ).PhysicalResourceId;
+    resources.ElasticSearchSearchLogGroup = stackDescriptionObj.find((x) =>
+      /ElasticSearchSearchLogGroup/i.test(x.LogicalResourceId)
+    ).PhysicalResourceId;
+    resources.ElasticSearchIndexLogGroup = stackDescriptionObj.find((x) =>
+      /ElasticSearchIndexLogGroup/i.test(x.LogicalResourceId)
+    ).PhysicalResourceId;
+    resources.ElasticSearchCluster = stackDescriptionObj.find((x) =>
+      /ElasticSearchCluster/i.test(x.LogicalResourceId)
+    ).PhysicalResourceId;
 
     resolve(resources);
   });
@@ -130,6 +138,6 @@ const setEnv = async () => {
     outputArray.push(`${key}=${data[key]}`);
   });
   fs.writeFileSync(".env", outputArray.join("\n"));
-  fs.appendFileSync(".env","\nisROMode="+isROMode);
+  fs.appendFileSync(".env", "\nisROMode=" + isROMode);
 };
 setEnv();

source/bin/pre-build.js

@@ -46,7 +46,7 @@ import { QueueEncryption } from "@aws-cdk/aws-sqs";
 import { LogGroup } from "@aws-cdk/aws-logs";
 import { LogGroupLogDestination } from "@aws-cdk/aws-apigateway";
 
-const API_CONCURRENT_REQUESTS = 20; //approximate number of 1-2 page documents to be processed parallelly
+const API_CONCURRENT_REQUESTS = 20; //approximate number of 1-2 page documents to be processed in parallell
 
 export interface TextractStackProps {
   email: string;
@@ -171,12 +171,14 @@ export class CdkTextractStack extends cdk.Stack {
             behaviors: [{ isDefaultBehavior: true }],
           },
         ],
-        errorConfigurations: [{
-          errorCode: 404,
-          responseCode: 200,
-          errorCachingMinTtl: 5,
-          responsePagePath: '/index.html'
-        }],
+        errorConfigurations: [
+          {
+            errorCode: 404,
+            responseCode: 200,
+            errorCachingMinTtl: 5,
+            responsePagePath: "/index.html",
+          },
+        ],
         priceClass: PriceClass.PRICE_CLASS_100,
         httpVersion: HttpVersion.HTTP2,
         enableIpV6: true,
@@ -231,11 +233,19 @@ export class CdkTextractStack extends cdk.Stack {
       cloudfrontDocumentsBucketPolicyStatement
     );
 
-    const esLogGroup = new LogGroup(
+    const esSearchLogGroup = new LogGroup(
+      this,
+      this.resourceName("ElasticSearchSearchLogGroup"),
+      {
+        logGroupName: this.resourceName("ElasticSearchSearchLogGroup"),
+      }
+    );
+
+    const esIndexLogGroup = new LogGroup(
       this,
-      this.resourceName("ElasticSearchLogGroup"),
+      this.resourceName("ElasticSearchIndexLogGroup"),
       {
-        logGroupName: this.resourceName("ElasticSearchLogGroup"),
+        logGroupName: this.resourceName("ElasticSearchIndexLogGroup"),
       }
     );
 
@@ -270,18 +280,6 @@ export class CdkTextractStack extends cdk.Stack {
         }
       );
     } else {
-      const serviceLinkedRole = new cdk.CfnResource(
-        this,
-        this.resourceName("es-service-linked-role"),
-        {
-          type: "AWS::IAM::ServiceLinkedRole",
-          properties: {
-            AWSServiceName: "es.amazonaws.com",
-            Description: "Role for ES to access resources in my VPC",
-          },
-        }
-      );
-
       elasticSearch = new es.CfnDomain(
         this,
         this.resourceName("ElasticSearchCluster"),
@@ -308,28 +306,43 @@ export class CdkTextractStack extends cdk.Stack {
           nodeToNodeEncryptionOptions: {
             enabled: true,
           },
-          logPublishingOptions: {
-            INDEX_SLOW_LOGS: {
-              cloudWatchLogsLogGroupArn: esLogGroup.logGroupArn,
-              enabled: true,
-            },
-            SEARCH_SLOW_LOGS: {
-              cloudWatchLogsLogGroupArn: esLogGroup.logGroupArn,
-              enabled: true,
-            },
-          },
         }
       );
-
-      elasticSearch.node.addDependency(serviceLinkedRole);
     }
 
+    const jobResultsKey = new kms.Key(
+      this,
+      this.resourceName("JobResultsKey"),
+      {
+        enableKeyRotation: true,
+        enabled: true,
+        trustAccountIdentities: true,
+        policy: new iam.PolicyDocument({
+          assignSids: true,
+          statements: [
+            new iam.PolicyStatement({
+              actions: ["kms:GenerateDataKey*", "kms:Decrypt"],
+              resources: ["*"], // Resource level permissions are not necessary in this policy statement, as it is automatically restricted to this key
+              effect: iam.Effect.ALLOW,
+              principals: [
+                new iam.ServicePrincipal("sns.amazonaws.com"),
+                new iam.ServicePrincipal("lambda.amazonaws.com"),
+                new iam.ServicePrincipal("textract.amazonaws.com"),
+                new iam.ServicePrincipal("sqs.amazonaws.com"),
+              ],
+            }),
+          ],
+        }),
+      }
+    );
+
     // SNS Topic
     const jobCompletionTopic = new sns.Topic(
       this,
-      this.resourceName("JobCompletion"),
+      this.resourceName("JobCompletionTopic"),
       {
         displayName: "Job completion topic",
+        masterKey: jobResultsKey,
       }
     );
 
@@ -349,6 +362,13 @@ export class CdkTextractStack extends cdk.Stack {
         resources: [jobCompletionTopic.topicArn],
       })
     );
+    textractServiceRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ["kms:Decrypt", "kms:GenerateDataKey*"],
+        resources: [jobResultsKey.keyArn],
+      })
+    );
 
     // DynamoDB tables
     const outputTable = new ddb.Table(this, this.resourceName("OutputTable"), {
@@ -440,6 +460,7 @@ export class CdkTextractStack extends cdk.Stack {
       {
         visibilityTimeout: cdk.Duration.seconds(900),
         retentionPeriod: cdk.Duration.seconds(1209600),
+        encryption: QueueEncryption.KMS_MANAGED,
       }
     );
 
@@ -449,12 +470,16 @@ export class CdkTextractStack extends cdk.Stack {
       {
         visibilityTimeout: cdk.Duration.seconds(900),
         retentionPeriod: cdk.Duration.seconds(1209600),
+        encryption: QueueEncryption.KMS,
+        encryptionMasterKey: jobResultsKey,
+        dataKeyReuse: cdk.Duration.seconds(86400),
         deadLetterQueue: {
           maxReceiveCount: 3,
           queue: jobResultsDLQueue,
         },
       }
     );
+
     // trigger
     jobCompletionTopic.addSubscription(
       new snsSubscriptions.SqsSubscription(jobResultsQueue)
@@ -874,6 +899,7 @@ export class CdkTextractStack extends cdk.Stack {
     jobResultProcessor.addLayers(textractorLayer);
     jobResultProcessor.addLayers(boto3Layer);
     jobResultProcessor.addLayers(elasticSearchLayer);
+    jobResultsKey.grantEncryptDecrypt(jobResultProcessor);
 
     // Triggers
     jobResultProcessor.addEventSource(