diff --git a/node-typescript/storage-cleaner/.gitignore b/node-typescript/storage-cleaner/.gitignore new file mode 100644 index 00000000..46afb6b3 --- /dev/null +++ b/node-typescript/storage-cleaner/.gitignore @@ -0,0 +1,133 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# Directory used by Appwrite CLI for local development +.appwrite \ No newline at end of file diff --git a/node-typescript/storage-cleaner/.prettierrc.json b/node-typescript/storage-cleaner/.prettierrc.json new file mode 100644 index 00000000..0a725205 --- /dev/null +++ b/node-typescript/storage-cleaner/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": true +} diff --git a/node-typescript/storage-cleaner/README.md b/node-typescript/storage-cleaner/README.md new file mode 100644 index 00000000..1031e004 --- /dev/null +++ b/node-typescript/storage-cleaner/README.md @@ -0,0 +1,44 @@ +# ๐Ÿงน Node.js Storage Cleaner Function + +Storage cleaner function to remove all files older than X number of days from the specified bucket. + +## ๐Ÿงฐ Usage + +### GET / + +Remove files older than X days from the specified bucket + +**Response** + +Sample `200` Response: Buckets cleaned + +## โš™๏ธ Configuration + +| Setting | Value | +| ----------------- | ------------- | +| Runtime | Node (18.0) | +| Entrypoint | `src/main.js` | +| Build Commands | `npm install` | +| Permissions | `any` | +| CRON | `0 1 * * *` | +| Timeout (Seconds) | 900 | + +## ๐Ÿ”’ Environment Variables + +### RETENTION_PERIOD_DAYS + +The number of days you want to retain a file. + +| Question | Answer | +| ------------ | ------ | +| Required | Yes | +| Sample Value | `1` | + +### APPWRITE_BUCKET_ID + +The ID of the bucket from which the files are to be deleted. + +| Question | Answer | +| ------------ | -------------- | +| Required | Yes | +| Sample Value | `652d...b4daf` | diff --git a/node-typescript/storage-cleaner/package-lock.json b/node-typescript/storage-cleaner/package-lock.json new file mode 100644 index 00000000..d85606cc --- /dev/null +++ b/node-typescript/storage-cleaner/package-lock.json @@ -0,0 +1,77 @@ +{ + "name": "starter-template", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "starter-template", + "version": "1.0.0", + "dependencies": { + "node-appwrite": "^14.1.0", + "typescript": "^5.4.5" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "prettier": "^3.2.5" + } + }, + "node_modules/@types/node": { + "version": "20.12.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz", + "integrity": "sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/node-appwrite": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/node-appwrite/-/node-appwrite-14.1.0.tgz", + "integrity": "sha512-kuKAZrdaAcGYOMUXtxNb1j+uIy+FIMiiU1dFkgwTXLsMLeLvC6HJ8/FH/kN9JyrWR2a2zcGN7gWfyQgWYoLMTA==", + "license": "BSD-3-Clause", + "dependencies": { + "node-fetch-native-with-agent": "1.7.2" + } + }, + "node_modules/node-fetch-native-with-agent": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/node-fetch-native-with-agent/-/node-fetch-native-with-agent-1.7.2.tgz", + "integrity": "sha512-5MaOOCuJEvcckoz7/tjdx1M6OusOY6Xc5f459IaruGStWnKzlI1qpNgaAwmn4LmFYcsSlj+jBMk84wmmRxfk5g==", + "license": "MIT" + }, + "node_modules/prettier": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", + "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/typescript": { + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/node-typescript/storage-cleaner/package.json b/node-typescript/storage-cleaner/package.json new file mode 100644 index 00000000..852c2c13 --- /dev/null +++ b/node-typescript/storage-cleaner/package.json @@ -0,0 +1,19 @@ +{ + "name": "storage-cleaner", + "version": "1.0.0", + "description": "", + "main": "dist/main.js", + "type": "module", + "scripts": { + "format": "prettier --write .", + "build": "tsc" + }, + "dependencies": { + "node-appwrite": "^14.1.0", + "typescript": "^5.4.5" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "prettier": "^3.2.5" + } +} diff --git a/node-typescript/storage-cleaner/src/appwrite.ts b/node-typescript/storage-cleaner/src/appwrite.ts new file mode 100644 index 00000000..d781e7a2 --- /dev/null +++ b/node-typescript/storage-cleaner/src/appwrite.ts @@ -0,0 +1,44 @@ +import { Client, Storage, Query } from 'node-appwrite'; +import { getExpiryDate } from './util.js'; +import { throwIfMissing } from './util.js'; + +class AppwriteService { + storage: Storage; + + constructor(apiKey: string) { + throwIfMissing(process.env, [ + 'APPWRITE_FUNCTION_API_ENDPOINT', + 'APPWRITE_FUNCTION_PROJECT_ID', + ]); + + const client = new Client() + .setEndpoint(process.env.APPWRITE_FUNCTION_API_ENDPOINT!) + .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID!) + .setKey(apiKey); + this.storage = new Storage(client); + } + + /** + * Clean up files from the storage bucket by removing files older than a specified retention period. + * + * @param {string} bucketId - The ID of the storage bucket to clean. + * @returns {Promise} A Promise that resolves when the bucket is cleaned. + */ + async cleanBucket(bucketId: string): Promise { + let response; + const queries = [ + Query.lessThan('$createdAt', getExpiryDate()), + Query.limit(25), + ]; + do { + response = await this.storage.listFiles(bucketId, queries); + await Promise.all( + response.files.map((file) => + this.storage.deleteFile(bucketId, file.$id) + ) + ); + } while (response.files.length > 0); + } +} + +export default AppwriteService; diff --git a/node-typescript/storage-cleaner/src/main.ts b/node-typescript/storage-cleaner/src/main.ts new file mode 100644 index 00000000..a79e070a --- /dev/null +++ b/node-typescript/storage-cleaner/src/main.ts @@ -0,0 +1,18 @@ +import AppwriteService from './appwrite.js'; +import { throwIfMissing } from './util.js'; + +type Context = { + req: any; + res: any; + log: (msg: any) => void; + error: (msg: any) => void; +}; +export default async ({ req, res, log, error }: Context) => { + throwIfMissing(process.env, ['RETENTION_PERIOD_DAYS', 'APPWRITE_BUCKET_ID']); + + const appwrite = new AppwriteService(req.headers['x-appwrite-key']); + + await appwrite.cleanBucket(process.env.APPWRITE_BUCKET_ID!); + + return res.text('Buckets cleaned', 200); +}; diff --git a/node-typescript/storage-cleaner/src/util.ts b/node-typescript/storage-cleaner/src/util.ts new file mode 100644 index 00000000..29b39679 --- /dev/null +++ b/node-typescript/storage-cleaner/src/util.ts @@ -0,0 +1,30 @@ +/** + * Returns a date subtracted by the retention period from the current date. + * The retention period is fetched from the RETENTION_PERIOD_DAYS environment variable. + * Defaults to 30 days if the environment variable is not set or invalid. + * @returns {string} The calculated expiry date in ISO 8601 format. + */ +export function getExpiryDate(): string { + const retentionPeriod = Number(process.env.RETENTION_PERIOD_DAYS ?? 30); + return new Date( + Date.now() - retentionPeriod * 24 * 60 * 60 * 1000 + ).toISOString(); +} + +/** + * Throws an error if any of the keys are missing from the object + * @param {*} obj + * @param {string[]} keys + * @throws {Error} + */ +export function throwIfMissing(obj: any, keys: string[]) { + const missing: string[] = []; + for (let key of keys) { + if (!(key in obj && obj[key] !== 0)) { + missing.push(key); + } + } + if (missing.length > 0) { + throw new Error(`Missing required fields: ${missing.join(', ')}`); + } +} diff --git a/node-typescript/storage-cleaner/tsconfig.json b/node-typescript/storage-cleaner/tsconfig.json new file mode 100644 index 00000000..ad436948 --- /dev/null +++ b/node-typescript/storage-cleaner/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "rootDir": "src", + "resolveJsonModule": true, + "allowJs": true, + "outDir": "dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitAny": true, + "skipLibCheck": true + } +}