Skip to content
This repository was archived by the owner on Oct 25, 2023. It is now read-only.

Commit 43ff253

Browse files
GWPGeorge Price
GWP
and
George Price
authored
Es log group fix (#86)
* removed generated files from tracking, updated gitignore * Revert "removed generated files from tracking, updated gitignore" This reverts commit a4620a4. * added update of es logs post-stack creation Co-authored-by: George Price <gwprice@amazon.com>
1 parent de75be5 commit 43ff253

File tree

6 files changed

+54
-43
lines changed

6 files changed

+54
-43
lines changed

‎deployment/custom-deployment/bin/stack-variables-to-client.sh‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎deployment/custom-deployment/bin/update-es-logs-and-client-stack-vars.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
source .env
4+
5+
#STACK
6+
7+
echo "==update %%CLIENT_APP_BUCKET%% in stack with $2=="
8+
replace="s/%%CLIENT_APP_BUCKET%%/$ClientAppBucketName/g"
9+
sed -i -e $replace ./lib/cdk-textract-client-stack.js
10+
11+
echo "==update Elastic Search Cluster ($ElasticSearchCluster) with log streams to Log Groups: $ElasticSearchSearchLogGroup and $ElasticSearchIndexLogGroup"
12+
INDEX_LOG_ARN=$(aws logs describe-log-groups --log-group-name $ElasticSearchIndexLogGroup | jq -r '.logGroups[0].arn')
13+
SEARCH_LOG_ARN=$(aws logs describe-log-groups --log-group-name $ElasticSearchSearchLogGroup | jq -r '.logGroups[0].arn')
14+
15+
echo "==adding permissions to ES service role first for creating log stream"
16+
aws logs put-resource-policy --policy-name es-to-log-stream --policy-document '{ "Version": "2012-10-17", "Statement": [ { "Sid": "ElasticSearchLogsToCloudWatchLogs", "Effect": "Allow", "Principal": { "Service": [ "es.amazonaws.com" ] }, "Action":["logs:PutLogEvents", "logs:CreateLogStream", "logs:DeleteLogStream"], "Resource": "*" } ] }'
17+
18+
echo "==Log Groups are $INDEX_LOG_ARN and $SEARCH_LOG_ARN"
19+
aws es update-elasticsearch-domain-config --domain-name $ElasticSearchCluster --log-publishing-options '{"INDEX_SLOW_LOGS": { "CloudWatchLogsLogGroupArn": "'"$INDEX_LOG_ARN"'", "Enabled": true }, "SEARCH_SLOW_LOGS": { "CloudWatchLogsLogGroupArn": "'"$SEARCH_LOG_ARN"'", "Enabled": true } }'
20+
21+

‎deployment/document-understanding-solution.template‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,10 @@ Resources:
170170
commands:
171171
- echo "This buildspec is based on image - aws/codebuild/amazonlinux2-x86_64-standard:2.0"
172172
- node --version
173+
- echo "Installing jq package"
174+
- wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
175+
- chmod +x ./jq
176+
- cp jq /usr/bin
173177
- npm install -g cdk@1.18.0
174178
- cdk --version
175179
- npm install -g typescript
@@ -541,9 +545,11 @@ Resources:
541545
"Action": [
542546
"logs:CreateLogGroup",
543547
"logs:CreateLogStream",
548+
"logs:DeleteLogStream",
544549
"logs:PutLogEvents",
545550
"logs:Describe*",
546-
"logs:PutRetentionPolicy"
551+
"logs:PutRetentionPolicy",
552+
"logs:PutResourcePolicy"
547553
]
548554
},
549555
{
@@ -672,7 +678,8 @@ Resources:
672678
"es:CreateElasticsearchServiceRole",
673679
"es:DeleteElasticsearchDomain",
674680
"es:DeleteElasticsearchServiceRole",
675-
"es:List*"
681+
"es:List*",
682+
"es:UpdateElasticsearchDomainConfig"
676683
]
677684
},
678685
{

‎source/bin/pre-build.js‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
/**********************************************************************************************************************
32
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. *
43
* *
@@ -118,6 +117,15 @@ const GetResources = new Promise((resolve, reject) => {
118117
resources.PdfGenLambda = stackDescriptionObj.find((x) =>
119118
/pdfgenerator/i.test(x.LogicalResourceId)
120119
).PhysicalResourceId;
120+
resources.ElasticSearchSearchLogGroup = stackDescriptionObj.find((x) =>
121+
/ElasticSearchSearchLogGroup/i.test(x.LogicalResourceId)
122+
).PhysicalResourceId;
123+
resources.ElasticSearchIndexLogGroup = stackDescriptionObj.find((x) =>
124+
/ElasticSearchIndexLogGroup/i.test(x.LogicalResourceId)
125+
).PhysicalResourceId;
126+
resources.ElasticSearchCluster = stackDescriptionObj.find((x) =>
127+
/ElasticSearchCluster/i.test(x.LogicalResourceId)
128+
).PhysicalResourceId;
121129

122130
resolve(resources);
123131
});
@@ -130,6 +138,6 @@ const setEnv = async () => {
130138
outputArray.push(`${key}=${data[key]}`);
131139
});
132140
fs.writeFileSync(".env", outputArray.join("\n"));
133-
fs.appendFileSync(".env","\nisROMode="+isROMode);
141+
fs.appendFileSync(".env", "\nisROMode=" + isROMode);
134142
};
135143
setEnv();

‎source/lib/cdk-textract-stack.ts‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -233,11 +233,19 @@ export class CdkTextractStack extends cdk.Stack {
233233
cloudfrontDocumentsBucketPolicyStatement
234234
);
235235

236-
const esLogGroup = new LogGroup(
236+
const esSearchLogGroup = new LogGroup(
237237
this,
238-
this.resourceName("ElasticSearchLogGroup"),
238+
this.resourceName("ElasticSearchSearchLogGroup"),
239239
{
240-
logGroupName: this.resourceName("ElasticSearchLogGroup"),
240+
logGroupName: this.resourceName("ElasticSearchSearchLogGroup"),
241+
}
242+
);
243+
244+
const esIndexLogGroup = new LogGroup(
245+
this,
246+
this.resourceName("ElasticSearchIndexLogGroup"),
247+
{
248+
logGroupName: this.resourceName("ElasticSearchIndexLogGroup"),
241249
}
242250
);
243251

@@ -272,18 +280,6 @@ export class CdkTextractStack extends cdk.Stack {
272280
}
273281
);
274282
} else {
275-
const serviceLinkedRole = new cdk.CfnResource(
276-
this,
277-
this.resourceName("es-service-linked-role"),
278-
{
279-
type: "AWS::IAM::ServiceLinkedRole",
280-
properties: {
281-
AWSServiceName: "es.amazonaws.com",
282-
Description: "Role for ES to access resources in my VPC",
283-
},
284-
}
285-
);
286-
287283
elasticSearch = new es.CfnDomain(
288284
this,
289285
this.resourceName("ElasticSearchCluster"),
@@ -310,20 +306,8 @@ export class CdkTextractStack extends cdk.Stack {
310306
nodeToNodeEncryptionOptions: {
311307
enabled: true,
312308
},
313-
logPublishingOptions: {
314-
INDEX_SLOW_LOGS: {
315-
cloudWatchLogsLogGroupArn: esLogGroup.logGroupArn,
316-
enabled: true,
317-
},
318-
SEARCH_SLOW_LOGS: {
319-
cloudWatchLogsLogGroupArn: esLogGroup.logGroupArn,
320-
enabled: true,
321-
},
322-
},
323309
}
324310
);
325-
326-
elasticSearch.node.addDependency(serviceLinkedRole);
327311
}
328312

329313
const jobResultsKey = new kms.Key(

‎source/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@
2626
"compile-ts-backend-stack": "tsc lib/cdk-textract-stack.ts --target es2018 --module commonjs --allowjs true",
2727
"compile-ts-client-stack": "tsc lib/cdk-textract-client-stack.ts --target es2018 --module commonjs --allowjs true",
2828
"deploy:backend": "ISCICD=$npm_package_cicd STACKNAME=$npm_package_stack_stackname AWS_REGION=$npm_package_stack_region USER_EMAIL=$npm_package_email cdk deploy --toolkit-stack-name DocumentUnderstandingCDKToolkit --require-approval never -v true",
29-
"deploy:client": "yarn export && yarn update-client-stack-variables && AWS_REGION=$npm_package_stack_region STACKNAME=$npm_package_stack_stackname cdk deploy --toolkit-stack-name DocumentUnderstandingCDKToolkit --require-approval never -v true -a \"node bin/deploy-client-stack.js\"",
29+
"deploy:client": "yarn export && yarn update-es-logs-and-stack-variables && AWS_REGION=$npm_package_stack_region STACKNAME=$npm_package_stack_stackname cdk deploy --toolkit-stack-name DocumentUnderstandingCDKToolkit --require-approval never -v true -a \"node bin/deploy-client-stack.js\"",
3030
"deploy:setup-user": "AWS_REGION=$npm_package_stack_region node bin/user-setup.js",
3131
"deploy:setup-samples": "AWS_REGION=$npm_package_stack_region bash ./bin/upload-samples.sh",
3232
"deploy-all": "yarn compile-ts-stacks && yarn bootstrap && yarn deploy:backend && yarn deploy:client && yarn deploy:setup-samples",
3333
"deploy": "yarn deploy-all && yarn update-pdf-lambda-code",
3434
"deploy:frontend": "yarn deploy:client && yarn deploy:setup-samples",
3535
"destroy": "STACKNAME=$npm_package_stack_stackname AWS_REGION=$npm_package_stack_region cdk destroy -v true -a \"node bin/deploy-client-stack.js\" && STACKNAME=$npm_package_stack_stackname AWS_REGION=$npm_package_stack_region USER_EMAIL=$npm_package_email cdk destroy -v true",
36-
"update-client-stack-variables": "bash ../deployment/custom-deployment/bin/stack-variables-to-client.sh",
36+
"update-es-logs-and-stack-variables": "bash ../deployment/custom-deployment/bin/update-es-logs-and-client-stack-vars.sh",
3737
"update-pdf-lambda-code": "bash ../deployment/custom-deployment/bin/update-pdf-lambda-code.sh",
3838
"dev": "next app",
3939
"build": "STACKNAME=$npm_package_stack_stackname AWS_REGION=$npm_package_stack_region yarn pre-build && next build app",
@@ -116,4 +116,4 @@
116116
"@aws-amplify/analytics": "1.2.16",
117117
"@aws-amplify/ui": "1.0.19"
118118
}
119-
}
119+
}

0 commit comments

Comments
 (0)