├── .gitignore
├── package-lock.json
├── package.json
├── readme.md
├── src
└── index.js
└── test
├── index.js
├── package-lock.json
└── package.json
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | firebase-debug.log*
8 |
9 | .DS_Store
10 |
11 | # secret and credential
12 | serviceAccount.json
13 | firebaseConfig.js
14 | *ServiceAccount.json
15 | *FirebaseConfig.js
16 |
17 | # Firebase cache
18 | .firebase/
19 |
20 | # Firebase config
21 |
22 | # Uncomment this if you'd like others to create their own Firebase project.
23 | # For a team working on the same Firebase project(s), it is recommended to leave
24 | # it commented so all members can deploy to the same project(s) in .firebaserc.
25 | # .firebaserc
26 |
27 | # Runtime data
28 | pids
29 | *.pid
30 | *.seed
31 | *.pid.lock
32 |
33 | # Directory for instrumented libs generated by jscoverage/JSCover
34 | lib-cov
35 |
36 | # Coverage directory used by tools like istanbul
37 | coverage
38 |
39 | # nyc test coverage
40 | .nyc_output
41 |
42 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
43 | .grunt
44 |
45 | # Bower dependency directory (https://bower.io/)
46 | bower_components
47 |
48 | # node-waf configuration
49 | .lock-wscript
50 |
51 | # Compiled binary addons (http://nodejs.org/api/addons.html)
52 | build/Release
53 |
54 | # Dependency directories
55 | node_modules/
56 |
57 | # Optional npm cache directory
58 | .npm
59 |
60 | # Optional eslint cache
61 | .eslintcache
62 |
63 | # Optional REPL history
64 | .node_repl_history
65 |
66 | # Output of 'npm pack'
67 | *.tgz
68 |
69 | # Yarn Integrity file
70 | .yarn-integrity
71 |
72 | # dotenv environment variables file
73 | .env
74 |
--------------------------------------------------------------------------------
/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@samwen/lambda-emfiles",
3 | "version": "1.0.1",
4 | "lockfileVersion": 1
5 | }
6 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@samwen/lambda-emfiles",
3 | "version": "1.1.1",
4 | "description": "solve node AWS lambda EMFILE issue",
5 | "main": "src/index.js",
6 | "directories": {
7 | "test": "test"
8 | },
9 | "repository": {
10 | "type": "git",
11 | "url": "git+https://github.com/samswen/lambda-emfiles.git"
12 | },
13 | "keywords": [
14 | "node",
15 | "AWS",
16 | "lambda",
17 | "EMFILE",
18 | "file descriptor",
19 | "limit",
20 | "leak",
21 | "Timeout",
22 | "ioredis",
23 | "mongodb",
24 | "AWS Lambda function errors",
25 | "MongoServerSelectionError",
26 | "Failed to refresh slots cache",
27 | "getaddrinfo EMFILE"
28 | ],
29 | "author": "Sam Wen",
30 | "license": "ISC",
31 | "bugs": {
32 | "url": "https://github.com/samswen/lambda-emfiles/issues"
33 | },
34 | "homepage": "https://github.com/samswen/lambda-emfiles#readme",
35 | "dependencies": {}
36 | }
37 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # lambda-emfiles
2 |
3 | A solution to node AWS lambda EMFILE issue. If you spot following errors in your lambda function logs:
4 |
5 | 1) getaddrinfo EMFILE
6 | 2) EMFILE, too many open files
7 | 3) with ioredis, Failed to refresh slots cache
8 | 4) with mongodb, MongoError: no connection available, MongoServerSelectionError
9 |
10 | ...
11 |
12 | etc
13 |
14 | Most likely, it is caused by exceeding the file descriptors limit of AWS lambda.
15 |
16 | lambda-emfiles provides solution to the problem.
17 |
18 | # how to use
19 |
20 | ## install
21 |
22 | npm install @samwen/lambda-emfiles
23 |
24 | ## in your lambda code
25 |
26 |
27 | const lambda_emfiles = require('@samwen/lambda-emfiles');
28 |
29 | exports.handler = async (event) => {
30 | try {
31 | await lambda_emfiles.start_verify();
32 | ...
33 | your code
34 | ...
35 | } catch (err) {
36 | ...
37 | error handle code
38 | ...
39 | } finally {
40 | ...
41 | code to close and release resources
42 | ...
43 | await lambda_emfiles.final_check();
44 | }
45 | return 'OK';
46 | };
47 |
48 | output by lambda_emfiles:
49 |
50 | example 1:
51 | *** new process, emfiles count: 23
52 | ...
53 | ...
54 | *** emfiles count: 24, leaks: 1
55 |
56 | example 2:
57 | *** old process, emfiles count: 24
58 | ...
59 | ...
60 | *** emfiles count: 33, leaks: 9
61 |
62 | example 3:
63 | *** old process, emfiles count: 890
64 | ...
65 | ...
66 | *** emfiles count: 910, leaks: 20
67 | Runtime exited with error: exit status 1 Runtime.ExitError
68 |
69 |
70 | # what does it do:
71 |
72 | 1) report file descriptor leaks to help debug.
73 | 2) prevent it. if the file descriptors will reach the max limit in next run, it exits the process.
74 |
75 | # detail of the issue and solution
76 |
77 | AWS lambda process runs within a docker container in Amazon Linux environment. The maximum limit on file descriptors is 1000. Normally, it is very hard for a lambda function to exceed the limit.
78 |
79 | However, the lambda process within the container may be reused for performance optimization.
80 |
81 | This is the reason for most cases of exceeding file descriptors limit.
82 |
83 | ## Here is how it happens:
84 |
85 | A lambda function leaks 100 file descriptors each time. It will hit the limit in abut 10 runs.
86 |
87 | The chance that the lambda process is reused 10 times is really low.
88 |
89 | This is why the lambda runs OK for most of times.
90 |
91 | But you can spot few errors caused by exceeding file descriptors limit after a while, it depends on how frequently the lambda is running and the concurrency level of the lambda.
92 |
93 | The best solution to the problem is to fix file descriptor leakage. lambda-emfiles provides report for this.
94 |
95 | It takes time to fix file descriptor leakage, specially it works most of times.
96 |
97 | Alternatively, lambda-emfiles calls process.exit(1) when it predicts a deficit of file descriptors in next run. Once the process is gone, it will not reused.
98 |
99 | ## Tunning of parameters: max_emfiles_needed and exit_process
100 |
101 | The 2 public methods come with default values for max_emfiles_needed and exit_process. The default values should work for most scenarios.
102 |
103 |
104 | async start_verify(max_emfiles_needed = 100, exit_process = false)
105 |
106 | async lambda_emfiles.final_check(max_emfiles_needed = 100, exit_process = true)
107 |
108 |
109 | max_emfiles_needed: is the estimated max file descriptors will open in the same time.
110 |
111 | exit_process: if it is true, it instructs lambda-emfiles to call process.exit(1), when it sees a deficit of file descriptors.
--------------------------------------------------------------------------------
/src/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const { exec } = require('child_process');
4 |
5 | class LambdaEmfiles {
6 |
7 | constructor() {
8 | this.is_new = true;
9 | this.emfiles_count = 0;
10 | this.max_leaks = 0;
11 | }
12 |
13 | /**
14 | * public
15 | *
16 | * @param {*} max_emfiles_needed estimated max file descriptors are needed
17 | * @param {*} exit_process to exit the running process if not ok
18 | * @returns
19 | */
20 | async start_verify(max_emfiles_needed = 100, exit_process = false) {
21 | if (!await this.__update_lambda_emfiles_count()) {
22 | console.log(`*** ${this.is_new ? 'new' : 'old'} process, NOT OK`);
23 | } else {
24 | console.log(`*** ${this.is_new ? 'new' : 'old'} process, emfiles count: ${this.emfiles_count}`);
25 | }
26 | this.is_new = false;
27 | if (exit_process && (!this.is_ok || 1000 - this.emfiles_count < max_emfiles_needed)) {
28 | process.exit(1);
29 | }
30 | return this.is_ok;
31 | }
32 |
33 | /**
34 | * public
35 | * *
36 | * @param {*} max_emfiles_needed estimated max file descriptors are needed
37 | * @param {*} exit_process to exit the running process if not ok
38 | *
39 | */
40 | async final_check(max_emfiles_needed = 100, exit_process = true) {
41 | const emfiles_count = this.emfiles_count;
42 | if (await this.__update_lambda_emfiles_count()) {
43 | if (this.emfiles_count > emfiles_count) {
44 | const leaks = this.emfiles_count - emfiles_count;
45 | if (leaks > this.max_leaks) {
46 | this.max_leaks = leaks;
47 | }
48 | console.log(`*** emfiles count: ${this.emfiles_count}, leaks: ${leaks}`);
49 | } else {
50 | console.log('*** no leak emfiles found');
51 | }
52 | } else {
53 | console.log('*** process, NOT OK');
54 | }
55 | if (exit_process) {
56 | if (max_emfiles_needed < this.max_leaks) {
57 | max_emfiles_needed = this.max_leaks;
58 | }
59 | if (!this.is_ok || 1000 - this.emfiles_count < max_emfiles_needed) {
60 | process.exit(1);
61 | }
62 | }
63 | }
64 |
65 | /**
66 | * private implementation
67 | */
68 | __update_lambda_emfiles_count() {
69 | this.is_ok = true;
70 | return new Promise(resolve => {
71 | exec(`ls /proc/${process.pid}/fd`, (err, stdout, stderr) => {
72 | if (err || stderr) {
73 | this.is_ok = false;
74 | resolve(this.is_ok);
75 | } else {
76 | const parts = stdout.split('\n');
77 | this.emfiles_count = parts.length - 1;
78 | resolve(this.is_ok);
79 | }
80 | });
81 | });
82 | }
83 | }
84 |
85 | module.exports = new LambdaEmfiles();
--------------------------------------------------------------------------------
/test/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const fs = require('fs');
4 | const lambda_emfiles = require('@samwen/lambda-emfiles');
5 |
6 | exports.handler = async (event) => {
7 | console.log(event);
8 | let max_fds_needed = 100;
9 | if (event.max_fds_needed) {
10 | max_fds_needed = event.max_fds_needed;
11 | }
12 | let leaks = 100;
13 | if (event.leaks) {
14 | leaks = event.leaks;
15 | }
16 | try {
17 | await lambda_emfiles.start_verify(max_fds_needed);
18 | //
19 | // simulate file descriptor leaks
20 | for (let i = 0; i < leaks; i++) {
21 | const filename = '/tmp/test' + i + '.txt';
22 | fs.open(filename, 'w', (err, fd) => {
23 | if (err) {
24 | console.error(err);
25 | }
26 | });
27 | }
28 | } catch (err) {
29 | console.error(err);
30 | } finally {
31 | await lambda_emfiles.final_check(max_fds_needed);
32 | }
33 | return 'OK';
34 | };
35 |
36 |
--------------------------------------------------------------------------------
/test/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "test",
3 | "version": "1.0.0",
4 | "lockfileVersion": 1,
5 | "requires": true,
6 | "dependencies": {
7 | "@samwen/lambda-emfiles": {
8 | "version": "1.0.1",
9 | "resolved": "https://registry.npmjs.org/@samwen/lambda-emfiles/-/lambda-emfiles-1.0.1.tgz",
10 | "integrity": "sha512-QiCyIJTp5/K6PrOEfWntoixyVSjXJ4qH6/99+NQuGurEQ4xe+jTAtkkbmo4xN7J9frhnvfCYy114O3+M9O7k3A=="
11 | }
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/test/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "test",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "keywords": [],
10 | "author": "",
11 | "license": "ISC",
12 | "dependencies": {
13 | "@samwen/lambda-emfiles": "^1.0.2"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------