Skip to content

Commit 665e4b8

Browse files
committed
refactor(cdk): improve lambda source code publishing
This changes the way the lambda source code is published by creating the asset check sum from the source code rather then the zipped archive. This creates a stable checksum which only changes on source code changes thus preventing CDK from updating the lambda source code on every deploy. Fixes #50
1 parent 83b0fa0 commit 665e4b8

15 files changed

+173
-23
lines changed
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import path from 'node:path'
2+
import { checkSumOfFiles } from './checksumOfFiles.js'
3+
4+
describe('checkSumOfFiles()', () => {
5+
it('should calculate a checksum of files', async () =>
6+
expect(
7+
await checkSumOfFiles([
8+
// sha1sum cdk/helpers/lambdas/test-data/1.txt
9+
// 6ae3f2029d36e029175cc225c2c4cda51a5ac602 cdk/helpers/lambdas/test-data/1.txt
10+
path.join(
11+
process.cwd(),
12+
'cdk',
13+
'helpers',
14+
'lambdas',
15+
'test-data',
16+
'1.txt',
17+
),
18+
// sha1sum cdk/helpers/lambdas/test-data/2.txt
19+
// 6a9c3333d7a3f9ee9fa1ef70224766fafb208fe4 cdk/helpers/lambdas/test-data/2.txt
20+
path.join(
21+
process.cwd(),
22+
'cdk',
23+
'helpers',
24+
'lambdas',
25+
'test-data',
26+
'2.txt',
27+
),
28+
]),
29+
).toEqual(
30+
// echo -n 6ae3f2029d36e029175cc225c2c4cda51a5ac6026a9c3333d7a3f9ee9fa1ef70224766fafb208fe4 | sha1sum
31+
'baa003a894945a0d2519b1f4340caa97c462058f',
32+
))
33+
})
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import * as crypto from 'node:crypto'
2+
import * as fs from 'node:fs'
3+
4+
/**
5+
* Computes the combined checksum of the given files
6+
*/
7+
export const checkSumOfFiles = async (files: string[]): Promise<string> => {
8+
const fileChecksums = await checkSum(files)
9+
const checksum = checkSumOfStrings(
10+
[...Object.entries(fileChecksums)].map(([, hash]) => hash),
11+
)
12+
return checksum
13+
}
14+
15+
const checkSumOfStrings = (strings: string[]): string => {
16+
const hash = crypto.createHash('sha1')
17+
hash.update(strings.join(''))
18+
return hash.digest('hex')
19+
}
20+
21+
const hashCache: { [key: string]: string } = {}
22+
const hashFile = async (file: string) => {
23+
if (hashCache[file] === undefined) {
24+
hashCache[file] = await new Promise((resolve) => {
25+
const hash = crypto.createHash('sha1')
26+
hash.setEncoding('hex')
27+
const fileStream = fs.createReadStream(file)
28+
fileStream.pipe(hash, { end: false })
29+
fileStream.on('end', () => {
30+
hash.end()
31+
const h = hash.read().toString()
32+
resolve(h)
33+
})
34+
})
35+
}
36+
return hashCache[file] as string
37+
}
38+
39+
/**
40+
* Computes the checksum for the given files
41+
*/
42+
const checkSum = async (
43+
files: string[],
44+
): Promise<{ [key: string]: string }> => {
45+
const hashes: { [key: string]: string } = {}
46+
await files.reduce(
47+
async (p, file) =>
48+
p.then(async () => {
49+
hashes[file] = await hashFile(file)
50+
}),
51+
Promise.resolve() as Promise<any>,
52+
)
53+
return hashes
54+
}

cdk/helpers/lambdas/packLambda.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,16 @@ import swc from '@swc/core'
22
import { createWriteStream } from 'node:fs'
33
import { parse } from 'path'
44
import yazl from 'yazl'
5+
import { checkSumOfFiles } from './checksumOfFiles.js'
56
import { commonParent } from './commonParent.js'
67
import { findDependencies } from './findDependencies.js'
78

8-
export type PackedLambda = { zipFile: string; handler: string }
9+
export type PackedLambda = {
10+
id: string
11+
zipFile: string
12+
handler: string
13+
hash: string
14+
}
915

1016
const removeCommonAncestor =
1117
(parentDir: string) =>
@@ -35,7 +41,7 @@ export const packLambda = async ({
3541
zipFile: string
3642
debug?: (label: string, info: string) => void
3743
progress?: (label: string, info: string) => void
38-
}): Promise<{ handler: string }> => {
44+
}): Promise<{ handler: string; hash: string }> => {
3945
const lambdaFiles = [sourceFile, ...findDependencies(sourceFile)]
4046

4147
const zipfile = new yazl.ZipFile()
@@ -56,6 +62,8 @@ export const packLambda = async ({
5662
progress?.(`added`, jsFileName)
5763
}
5864

65+
const hash = await checkSumOfFiles(lambdaFiles)
66+
5967
// Mark it as ES module
6068
zipfile.addBuffer(
6169
Buffer.from(
@@ -76,5 +84,5 @@ export const packLambda = async ({
7684
})
7785
progress?.(`written`, zipFile)
7886

79-
return { handler: stripCommon(sourceFile) }
87+
return { handler: stripCommon(sourceFile), hash }
8088
}

cdk/helpers/lambdas/packLambdaFromPath.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ export const packLambdaFromPath = async (
1616
// Directory exists
1717
}
1818
const zipFile = path.join(process.cwd(), 'dist', 'lambdas', `${id}.zip`)
19-
const { handler } = await packLambda({
19+
const { handler, hash } = await packLambda({
2020
sourceFile: path.join(baseDir, sourceFile),
2121
zipFile,
2222
})
2323
return {
24+
id,
2425
zipFile,
2526
handler: handler.replace('.js', `.${handlerFunction}`),
27+
hash,
2628
}
2729
}

cdk/helpers/lambdas/packLayer.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ import { copyFile, mkdir, readFile, rm, writeFile } from 'fs/promises'
44
import { glob } from 'glob'
55
import path from 'path'
66
import { ZipFile } from 'yazl'
7+
import { checkSumOfFiles } from './checksumOfFiles.js'
78

8-
export type PackedLayer = { layerZipFile: string }
9+
export type PackedLayer = { layerZipFile: string; hash: string }
910

1011
export const packLayer = async ({
1112
id,
@@ -43,14 +44,18 @@ export const packLayer = async ({
4344
}
4445
}, {} as Record<string, string>)
4546

47+
const packageJSON = path.join(nodejsDir, 'package.json')
4648
await writeFile(
47-
path.join(nodejsDir, 'package.json'),
49+
packageJSON,
4850
JSON.stringify({
4951
dependencies: depsToBeInstalled,
5052
}),
5153
'utf-8',
5254
)
53-
await copyFile(packageLockJsonFile, path.join(nodejsDir, 'package-lock.json'))
55+
const packageLock = path.join(nodejsDir, 'package-lock.json')
56+
await copyFile(packageLockJsonFile, packageLock)
57+
58+
const hash = await checkSumOfFiles([packageJSON, packageLock])
5459

5560
await new Promise<void>((resolve, reject) => {
5661
const [cmd, ...args] = [
@@ -95,5 +100,6 @@ export const packLayer = async ({
95100

96101
return {
97102
layerZipFile: zipFileName,
103+
hash,
98104
}
99105
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
76f7c069-8dde-49c6-9ac4-887e4dfe0aa0
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5ee3850c-03bc-48b3-876a-61ae6c4291c3

cdk/resources/ConvertDeviceMessages.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
import { Construct } from 'constructs'
1010
import type { PackedLambda } from '../helpers/lambdas/packLambda.js'
1111
import type { DeviceStorage } from './DeviceStorage.js'
12+
import { LambdaSource } from './LambdaSource.js'
1213
import type { WebsocketAPI } from './WebsocketAPI.js'
1314

1415
/**
@@ -39,7 +40,7 @@ export class ConvertDeviceMessages extends Construct {
3940
runtime: Lambda.Runtime.NODEJS_18_X,
4041
timeout: Duration.seconds(5),
4142
memorySize: 1792,
42-
code: Lambda.Code.fromAsset(lambdaSources.onDeviceMessage.zipFile),
43+
code: new LambdaSource(this, lambdaSources.onDeviceMessage).code,
4344
description: 'Convert device messages and publish them on the EventBus',
4445
environment: {
4546
VERSION: this.node.tryGetContext('version'),

cdk/resources/DeviceShadow.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
} from 'aws-cdk-lib'
1414
import { Construct } from 'constructs'
1515
import type { PackedLambda } from '../helpers/lambdas/packLambda'
16+
import { LambdaSource } from './LambdaSource.js'
1617
import type { WebsocketAPI } from './WebsocketAPI.js'
1718

1819
export class DeviceShadow extends Construct {
@@ -74,7 +75,7 @@ export class DeviceShadow extends Construct {
7475
runtime: Lambda.Runtime.NODEJS_18_X,
7576
timeout: Duration.seconds(5),
7677
memorySize: 1792,
77-
code: Lambda.Code.fromAsset(lambdaSources.prepareDeviceShadow.zipFile),
78+
code: new LambdaSource(this, lambdaSources.prepareDeviceShadow).code,
7879
description: 'Generate queue to fetch the shadow data',
7980
environment: {
8081
VERSION: this.node.tryGetContext('version'),
@@ -96,7 +97,7 @@ export class DeviceShadow extends Construct {
9697
runtime: Lambda.Runtime.NODEJS_18_X,
9798
timeout: processDeviceShadowTimeout,
9899
memorySize: 1792,
99-
code: Lambda.Code.fromAsset(lambdaSources.fetchDeviceShadow.zipFile),
100+
code: new LambdaSource(this, lambdaSources.fetchDeviceShadow).code,
100101
description: `Fetch devices' shadow from nRF Cloud`,
101102
environment: {
102103
VERSION: this.node.tryGetContext('version'),

cdk/resources/HistoricalData.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
} from 'aws-cdk-lib'
1111
import { Construct } from 'constructs'
1212
import type { PackedLambda } from '../helpers/lambdas/packLambda.js'
13+
import { LambdaSource } from './LambdaSource.js'
1314
import type { WebsocketAPI } from './WebsocketAPI.js'
1415

1516
/**
@@ -57,9 +58,8 @@ export class HistoricalData extends Construct {
5758
runtime: Lambda.Runtime.NODEJS_18_X,
5859
timeout: Duration.seconds(5),
5960
memorySize: 1792,
60-
code: Lambda.Code.fromAsset(
61-
lambdaSources.storeMessagesInTimestream.zipFile,
62-
),
61+
code: new LambdaSource(this, lambdaSources.storeMessagesInTimestream)
62+
.code,
6363
description: 'Save converted messages into Timestream database',
6464
environment: {
6565
VERSION: this.node.tryGetContext('version'),

0 commit comments

Comments
 (0)