├── .ctags ├── .gitignore ├── .lldbinit ├── .vimrc ├── Makefile ├── README.md ├── gyp ├── .gitignore ├── README.md ├── example.gyp └── src │ └── example.cc ├── lib ├── async.js ├── rsa_private_4096.pem ├── rsa_public_4096.pem ├── sign.js └── tasks.js ├── node-gyp-openssl-issue ├── .gitignore ├── README.md ├── binding.gyp ├── example.c ├── node-v17.9.1-headers.tar.gz ├── node_opensslconf.h └── opensslconf.h ├── notes ├── async_hooks.md ├── debugging.md ├── eventloop.md ├── heap-snapshot.md ├── lto-ppc64le-issue.md ├── quic.md └── snapshot.md ├── src ├── add.js ├── add.wasm ├── add.wat ├── import.js ├── import.wasm ├── import.wat ├── mem.js ├── mem.wasm ├── mem.wat ├── start.js ├── start.wasm └── start.wat └── test ├── base-object_test.cc ├── environment_test.cc └── main.cc /.ctags: -------------------------------------------------------------------------------- 1 | --recurse=yes 2 | --exclude=.git 3 | --exclude=deps 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | deps 2 | tags 3 | *.dSYM 4 | *.swp 5 | *.swo 6 | test/base-object_test 7 | test/environment_test 8 | -------------------------------------------------------------------------------- /.lldbinit: -------------------------------------------------------------------------------- 1 | # Print HeapObjects. 2 | command regex -h 'Print a v8 JavaScript object' job 's/(.+)/expr -- '_v8_internal_Print_Object((void*)(%1))/' 3 | 4 | # Print v8::Local handle value. 5 | command regex -h 'Print content of a v8::Local handle' jlh 's/(.+)/expr -- '_v8_internal_Print_Object(*(v8::internal::Object**)(*%1))/' 6 | 7 | # Print Code objects containing given PC. 8 | command regex -h 'Print a v8 Code object from an internal code address' jco 's/(.+)/expr -- '_v8_internal_Print_Code((void*)(*%1))/' 9 | 10 | # Print TypeFeedbackVector 11 | command regex -h 'Print a v8 TypeFeedbackVector object' jfv 's/(.+)/expr -- '_v8_internal_Print_TypeFeedbackVector((void*)(%1))/' 12 | 13 | # Print TypeFeedbackMetadata 14 | command regex -h 'Print a v8 TypeFeedbackMetadata object' jfm 's/(.+)/expr -- '_v8_internal_Print_TypeFeedbackMetadata((void*)(%1))/' 15 | 16 | # Print DescriptorArray. 17 | command regex -h 'Print a v8 DescriptorArray object' jda 's/(.+)/expr -- '_v8_internal_Print_DescriptorArray((void*)(%1))/' 18 | 19 | # Print LayoutDescriptor. 20 | command regex -h 'Print a v8 LayoutDescriptor object' jld 's/(.+)/expr -- '_v8_internal_Print_LayoutDescriptor((void*)(%1))/' 21 | 22 | # Print TransitionArray. 23 | command regex -h 'Print a v8 TransitionArray object' jta 's/(.+)/expr -- '_v8_internal_Print_TransitionArray((void*)(%1))/' 24 | 25 | # Print JavaScript stack trace. 26 | command alias -h 'Print the current JavaScript stack trace' -- jst expr _v8_internal_Print_StackTrace() 27 | -------------------------------------------------------------------------------- /.vimrc: -------------------------------------------------------------------------------- 1 | set tags=./tags 2 | set tags+=~/work/nodejs/node/tags 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NODE_HOME ?= /Users/danielbevenius/work/nodejs/node 2 | node_build_dir = $(NODE_HOME)/out/Release 3 | node_include_dir = $(NODE_HOME)/src 4 | node_obj_dir = $(node_build_dir)/obj.target/node/src 5 | v8_build_dir = $(node_build_dir) 6 | v8_include_dir = $(NODE_HOME)/deps/v8/include 7 | cares_include_dir = $(NODE_HOME)/deps/cares/include 8 | 9 | gtest_dont_defines = -D GTEST_DONT_DEFINE_ASSERT_EQ \ 10 | -D GTEST_DONT_DEFINE_ASSERT_NE \ 11 | -D GTEST_DONT_DEFINE_ASSERT_LE \ 12 | -D GTEST_DONT_DEFINE_ASSERT_LT \ 13 | -D GTEST_DONT_DEFINE_ASSERT_GE \ 14 | -D GTEST_DONT_DEFINE_ASSERT_GT 15 | 16 | v8_libs = $(v8_build_dir)/libv8_base.a \ 17 | $(v8_build_dir)/libv8_libbase.a \ 18 | $(v8_build_dir)/libv8_snapshot.a \ 19 | $(v8_build_dir)/libv8_libplatform.a \ 20 | $(v8_build_dir)/libicuucx.a \ 21 | $(v8_build_dir)/libicui18n.a \ 22 | $(v8_build_dir)/libicudata.a \ 23 | $(v8_build_dir)/libicustubdata.a \ 24 | $(v8_build_dir)/libv8_inspector_stl.a \ 25 | $(v8_build_dir)/libuv.a \ 26 | $(v8_build_dir)/libv8_snapshot.a \ 27 | $(v8_build_dir)/libhttp_parser.a \ 28 | $(v8_build_dir)/libopenssl.a \ 29 | $(v8_build_dir)/libcares.a \ 30 | $(v8_build_dir)/libzlib.a \ 31 | $(v8_build_dir)/libv8_libsampler.a 32 | 33 | node_obj = $(node_obj_dir)/async-wrap.o \ 34 | $(node_obj_dir)/backtrace_posix.o \ 35 | $(node_obj_dir)/cares_wrap.o \ 36 | $(node_obj_dir)/connection_wrap.o \ 37 | $(node_obj_dir)/connect_wrap.o \ 38 | $(node_obj_dir)/debug-agent.o \ 39 | $(node_obj_dir)/env.o \ 40 | $(node_obj_dir)/fs_event_wrap.o \ 41 | $(node_obj_dir)/handle_wrap.o \ 42 | $(node_obj_dir)/inspector_agent.o \ 43 | $(node_obj_dir)/inspector_socket.o \ 44 | $(node_obj_dir)/js_stream.o \ 45 | $(node_obj_dir)/node.o \ 46 | $(node_obj_dir)/node_buffer.o \ 47 | $(node_obj_dir)/node_config.o \ 48 | $(node_obj_dir)/node_constants.o \ 49 | $(node_obj_dir)/node_contextify.o \ 50 | $(node_obj_dir)/node_crypto.o \ 51 | $(node_obj_dir)/node_crypto_bio.o \ 52 | $(node_obj_dir)/node_crypto_clienthello.o \ 53 | $(node_obj_dir)/node_dtrace.o \ 54 | $(node_obj_dir)/node_file.o \ 55 | $(node_obj_dir)/node_http_parser.o \ 56 | $(node_obj_dir)/node_i18n.o \ 57 | $(node_obj_dir)/node_javascript.o \ 58 | $(node_obj_dir)/node_os.o \ 59 | $(node_obj_dir)/node_revert.o \ 60 | $(node_obj_dir)/node_stat_watcher.o \ 61 | $(node_obj_dir)/node_util.o \ 62 | $(node_obj_dir)/node_v8.o \ 63 | $(node_obj_dir)/node_watchdog.o \ 64 | $(node_obj_dir)/node_zlib.o \ 65 | $(node_obj_dir)/pipe_wrap.o \ 66 | $(node_obj_dir)/process_wrap.o \ 67 | $(node_obj_dir)/signal_wrap.o \ 68 | $(node_obj_dir)/spawn_sync.o \ 69 | $(node_obj_dir)/stream_base.o \ 70 | $(node_obj_dir)/stream_wrap.o \ 71 | $(node_obj_dir)/string_bytes.o \ 72 | $(node_obj_dir)/string_search.o \ 73 | $(node_obj_dir)/tcp_wrap.o \ 74 | $(node_obj_dir)/timer_wrap.o \ 75 | $(node_obj_dir)/tls_wrap.o \ 76 | $(node_obj_dir)/tty_wrap.o \ 77 | $(node_obj_dir)/udp_wrap.o \ 78 | $(node_obj_dir)/util.o \ 79 | $(node_obj_dir)/uv.o 80 | 81 | node_defs = -D_DARWIN_USE_64_BIT_INODE=1 \ 82 | -DNODE_WANT_INTERNALS=1 \ 83 | -DV8_DEPRECATION_WARNINGS=1 \ 84 | -DNODE_USE_V8_PLATFORM=1 \ 85 | -DNODE_HAVE_I18N_SUPPORT=1 \ 86 | -DNODE_HAVE_SMALL_ICU=1 \ 87 | -DHAVE_INSPECTOR=1 \ 88 | -DV8_INSPECTOR_USE_STL=1 \ 89 | -DHAVE_OPENSSL=1 \ 90 | -DHAVE_DTRACE=1 \ 91 | -D__POSIX__ \ 92 | -DNODE_PLATFORM="darwin" \ 93 | -DUCONFIG_NO_TRANSLITERATION=1 \ 94 | -DUCONFIG_NO_SERVICE=1 \ 95 | -DUCONFIG_NO_REGULAR_EXPRESSIONS=1 \ 96 | -DU_ENABLE_DYLOAD=0 \ 97 | -DU_STATIC_IMPLEMENTATION=1 \ 98 | -DU_HAVE_STD_STRING=0 \ 99 | -DUCONFIG_NO_BREAK_ITERATION=0 \ 100 | -DUCONFIG_NO_LEGACY_CONVERSION=1 \ 101 | -DUCONFIG_NO_CONVERSION=1 \ 102 | -DHTTP_PARSER_STRICT=0 \ 103 | -D_LARGEFILE_SOURCE \ 104 | -D_FILE_OFFSET_BITS=64 \ 105 | -DDEBUG \ 106 | -D_DEBUG 107 | 108 | node_cc = c++ $(gtest_dont_defines) \ 109 | $(node_defs) \ 110 | -g \ 111 | -std=gnu++0x \ 112 | -stdlib=libc++ \ 113 | -O0 \ 114 | -gdwarf-2 \ 115 | -mmacosx-version-min=10.7 \ 116 | -arch x86_64 \ 117 | -Wall \ 118 | -Wendif-labels \ 119 | -W \ 120 | -Wno-unused-parameter \ 121 | -fno-rtti \ 122 | -fno-exceptions \ 123 | -fno-threadsafe-statics \ 124 | -fno-strict-aliasing \ 125 | -I`pwd`/deps/googletest/googletest/include \ 126 | -I$(node_include_dir) \ 127 | -I$(cares_include_dir) \ 128 | -I$(v8_include_dir) \ 129 | $(v8_libs) \ 130 | $(node_obj) \ 131 | -pthread \ 132 | lib/libgtest.a test/main.cc 133 | 134 | check: test/base-object_test 135 | ./test/base-object_test 136 | 137 | test/base-object_test: test/base-object_test.cc 138 | $ $(node_cc) test/base-object_test.cc -o test/base-object_test 139 | 140 | test/environment_test: test/environment_test.cc 141 | $ $(node_cc) -o test/environment_test 142 | 143 | src/import.wasm: src/import.wat 144 | wat2wasm $< -o $@ 145 | 146 | .PHONY: clean 147 | 148 | clean: 149 | rm -rf test/base-object_test 150 | rm -rf test/environment_test 151 | -------------------------------------------------------------------------------- /gyp/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | build 4 | -------------------------------------------------------------------------------- /gyp/README.md: -------------------------------------------------------------------------------- 1 | ## Generate your project (gyp) 2 | Just an example to understand how gyp works. 3 | 4 | 5 | ### Install gyp 6 | 7 | git clone https://chromium.googlesource.com/external/gyp.git 8 | cd gyp 9 | sudo python setup.py install 10 | 11 | ### Building 12 | Generate the make file 13 | 14 | $ gyp example.gyp --depth=. -f make --generator-output=./build/makefiles 15 | 16 | Make: 17 | 18 | $ make -C ./build/makefiles 19 | 20 | ### Running 21 | 22 | $ $ ./build/makefiles/out/Default/example 23 | 24 | ### gyp configuration notes 25 | 26 | Within a variables section, keys named with percent sign (%) suffixes mean that the variable should be set 27 | only if it is undefined at the time it is processed. 28 | 29 | -------------------------------------------------------------------------------- /gyp/example.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [{ 3 | "target_name": "example", 4 | "type": "executable", 5 | "sources": [ 6 | "src/example.cc" 7 | ] 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /gyp/src/example.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | std::cout << "In main..." << std::endl; 5 | } 6 | -------------------------------------------------------------------------------- /lib/async.js: -------------------------------------------------------------------------------- 1 | const async_hooks = require('async_hooks'); 2 | const fs = require('fs'); 3 | const util = require('util'); 4 | const asyncHook = async_hooks.createHook({ init, before, after, destroy }); 5 | asyncHook.enable(); 6 | 7 | function debug(...args) { 8 | fs.writeFileSync(1, `${util.format(...args)}\n`, { flag: 'a' }); 9 | } 10 | 11 | function init(asyncId, type, triggerAsyncId, resource) { 12 | if (type === "Timeout") { 13 | debug("asyncId for Timout:", asyncId); 14 | } 15 | } 16 | 17 | function before(asyncId) { 18 | debug("before:", asyncId); 19 | } 20 | 21 | function after(asyncId) { 22 | debug("after:", asyncId); 23 | } 24 | 25 | function destroy(asyncId) { 26 | debug("destroy:", asyncId); 27 | } 28 | 29 | setTimeout(() => debug('in timeout...'), 0); 30 | -------------------------------------------------------------------------------- /lib/rsa_private_4096.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKAIBAAKCAgEAxeStwofbjtZuol4lwKn1w08AzcSNLHfCqNFHa+W7er8is7LQ 3 | sPljtPT4yn4lsao83ngHFvSC3tbMiRNDpUHYqH2wBuUkuOmCtYkZLi0307H0CwcV 4 | V6W5P3tNEt80IJ+PqlRxtTknezUtbOasbIi/aornVWG+psgqDGrFZ4oTsWtiE0Sv 5 | i7sDqN5E2dijmH/YYnlnwqszgzHdtAnARp1bG34E64sqWCmLoGCfPdHtym/CSdxO 6 | LOsDV15jrwODZQ/TJZ5thkwKZRxu7g9fwlhA1PiI5WDP4reXNaqa2bSgrzpAljQE 7 | xYs4N0L7okSVOJQX9BEaoWtq8NLU8MpMdGoHNDU0Xr60Lfr58Z5qn8RGEvlTxoCb 8 | PJzPV2zgzD/lmEqft6NnfTclveA3sd8xSrOBUn4o3S8hS0b9Su7PBukHjM96/e0R 9 | eoIshSwXlQTLr2Ft8KwupyPm1ltNcTDtjqHcIWU6Bg+kPy9mxSVtGGZYAPtqGzNB 10 | A/m+oOja/OSPxAblPdln691DaDuZs5nuZCGwGcLaJWgiyoqvXAcyXDZFyH4OZZh8 11 | rsBLKbnFXHZ/ziG0cAozEygZEPJappw8Lx/ady7WL/SJjxooiKapc7Bnfy8eSLV3 12 | +XAKxhLW/MQ6ChJ+e/8ExAY02ca4MpCvqwIk9TfV6FM8pWGqHzQFj0v3NL0CAwEA 13 | AQKCAgBTb8eTbZS09NRQwUFJql9kqbq9B1I+nYAFjbd/Vq1lY5FOEubKt1vCwEbl 14 | mapq7kwbwJ+8nftP2WEDqouq8chXwialwZdqH4ps4BEt1wLizvUGcUYeXlFs4p/s 15 | hQ+FccExH8mRjzeGSzWL5PZuDHoogchnx36K83pHIf15Wk5TT+NaHGunjoJMgOqm 16 | ryDK+5xQaL/G5Egj2LKRZksbet0fClMovNRtt5aXWCXL+uc3o0dXvPt5FN2jyLhe 17 | 4ixUQAfWpKWpKgZ3+zUKSpElb/Bl2yRdEiSUgrPOfNAtWmsldnok2mnooHpjUmqm 18 | UCRaZpZy4YNI6/F6+Gmv3Ju/ubSvHzoxQLlvgUqWAnVshivF1TJImHSIiLIvBKPp 19 | 29SD6maWIT1DC9sKC4E1gq7VO4762l1//zEOAY7XK0Z7LrbZO4WXHnsgFOpGthQ3 20 | g9Qi/SeM6mb5xEJTBUBTmkhGs1x8jolzca30mqv8T63W4PXkXHmZdK7vyH5useiI 21 | s0eGUeaYK892WgfxCBo24JCNQiAcH/wTwV4l4yROqeH2V4ShbIYmCzla++7vsPYW 22 | hAwQR9eH0+4ogTkaMQrm16plZk0ezVX9BKK8KTnd4G9/T18VstQbiowF2/cKnGKC 23 | OqrmoR2vHOksQdUJVmnwCRqU1symBxhY0GSIps98v+lUYExKQQKCAQEA/uVYE2/H 24 | eNcV/uWAI9LspANXHJE33TFMZ8SuyOYtp3MYJizmQ1uT7Om2LEanDnNiz+fAQhrE 25 | vo1sDIF9xOAde2qjIH+iDzcLvFPgC3gkQspFjU31M9OO5xAjzBxfL3KDiG2MtmTR 26 | hNuKJX56eCOqkEp6WKaWOA35ccaKYHxNzMS49weCv95ZPpR9q0J1sgzD7HtVh4yu 27 | XI01/BC8F0RmYjtsuUo+PmB6sO2K94uqqo0GPUos7Mhgrbff3L36EkOPgmRiA1AV 28 | Zy1sKKxUKspGQ3m1fg+CA/+GZGckvYkVot1lFrwmrS2dok8EhT1HcVJde+++jx7z 29 | JsRLgFRvKHXklwKCAQEAxsAfxIQjjjKmuyJCzIvxG7lnuzovdy4OEdSuJL4yK5m3 30 | 4BHJHn+yHeRIcrDnJKUTUYffcH/OjOnJS94BA6wH1tEuvGQz6LV6UpwApZ1M/2md 31 | nP0eC2L2JtSRL8mdxfyqXDloWMpD7rncBZ6ChLEZ6sWYa6WBQTARmPVePyUpNNG2 32 | qymxN3/vRBGGBunD3j6zX0M0szWK5iU+qsYDy3KzCKG8FU7XxwzRbP7iARRD5Hpt 33 | Zmy2W52EJg1uhmlVXJMm32SEBfrD2oDmlnjAqaZdqi5Mq2e4uB3dhM9RwJppSALG 34 | BY6k9DeanAFbOlawMJri2pk7B0phCn+DN2pg0+W3ywKCAQBeTwzfZCQxmaMRxGg8 35 | 2PWlWXcJotFAjdTvL95bho6tve/ZcBNiKKf6qB43E40L07VjpyODUdQpjLnFhsO5 36 | 7BH8b+AbTh3v8zXsYDwtAi6oZ56EQavPmR7ubxJPms+9BmmUOLQvZ+39ch0S8lDt 37 | 0oRxDp1l330FEGaSqhrYyCUg9khZXfYKd4IdnWNB0j0pu39iJ9/lXy/EHpsywB5X 38 | nX8kKUh45fdRrPC4NauNG6fxomwEkUU99oWOwNGbIs87orOeUvXQs/i3TB8QjXI2 39 | wtBsdsOn+KTqRci7rU3ysp3GvJOCbesBeDcyrnnFsn6Udx0Plgyzd4gPd+FXgeX+ 40 | 2l/RAoIBAH81FKAY2xD2RlTb1tlIcGeIQWZKFXs4VPUApP0LZt0VI+UcPRdyL7SG 41 | GgCeTTLdHQI/7rj4dGEoeRg/3XJWNyY8+KbHk5nMHaCmDJvzlAaduK10LDipfFba 42 | Epr9dif0Ua15aNn7i4NOHg7Sp0L6f1YOZkHvykzI0VqPIWVVCYyu9TWUF8Mn9SIh 43 | /SCLmjuy8ed1AlP5Xw9yoyt2VZNvtDtAGTuiHOVfxOL4N/rs149y9HZr+kOlC6G3 44 | Uxhgbqwz2tt8YCvblmNRwURpwRZUTvrPa28Bke713oRUlUSrD9txOwDvjZBpzmEv 45 | VQ5/0YEqgSvcizVdW8L2XiunwJWfIAUCggEBALr4RF9TYa37CImZOs+vJ8FGRKMz 46 | h1EUwO2PvuITvkTtu/7E4IjyxAo5dkAokkWQCGABciiDJJEYUWqcUX45qQChOgtm 47 | NU2od6f9tgyDFxN5KS8cE32NXV3rJXs3bBZmIKLSPETf3uIPuEpFPjpdR5v5jlV+ 48 | TDjH4RrItE3hDCvypTXhXXMmWp3VfYbgEfIP03uR2iIhL+/g3BUqbrywPEsTViSN 49 | NM/uBDQyamXLXB1bQ2I/Ob41I82PD1iNCqGi7ZvZ3eVYGgUTQyw6Q4O8glTPP9cC 50 | SFVXwE9gHbLe8TqfTZCWrM6crGX6Bb6hV2tqNsA+7J69U9NGuw5GNqXjafU= 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /lib/rsa_public_4096.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeStwofbjtZuol4lwKn1 3 | w08AzcSNLHfCqNFHa+W7er8is7LQsPljtPT4yn4lsao83ngHFvSC3tbMiRNDpUHY 4 | qH2wBuUkuOmCtYkZLi0307H0CwcVV6W5P3tNEt80IJ+PqlRxtTknezUtbOasbIi/ 5 | aornVWG+psgqDGrFZ4oTsWtiE0Svi7sDqN5E2dijmH/YYnlnwqszgzHdtAnARp1b 6 | G34E64sqWCmLoGCfPdHtym/CSdxOLOsDV15jrwODZQ/TJZ5thkwKZRxu7g9fwlhA 7 | 1PiI5WDP4reXNaqa2bSgrzpAljQExYs4N0L7okSVOJQX9BEaoWtq8NLU8MpMdGoH 8 | NDU0Xr60Lfr58Z5qn8RGEvlTxoCbPJzPV2zgzD/lmEqft6NnfTclveA3sd8xSrOB 9 | Un4o3S8hS0b9Su7PBukHjM96/e0ReoIshSwXlQTLr2Ft8KwupyPm1ltNcTDtjqHc 10 | IWU6Bg+kPy9mxSVtGGZYAPtqGzNBA/m+oOja/OSPxAblPdln691DaDuZs5nuZCGw 11 | GcLaJWgiyoqvXAcyXDZFyH4OZZh8rsBLKbnFXHZ/ziG0cAozEygZEPJappw8Lx/a 12 | dy7WL/SJjxooiKapc7Bnfy8eSLV3+XAKxhLW/MQ6ChJ+e/8ExAY02ca4MpCvqwIk 13 | 9TfV6FM8pWGqHzQFj0v3NL0CAwEAAQ== 14 | -----END PUBLIC KEY----- 15 | -------------------------------------------------------------------------------- /lib/sign.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto') 2 | const tls = require('tls') 3 | const fs = require('fs') 4 | 5 | const private_key = fs.readFileSync('./lib/rsa_private_4096.pem', 'utf-8') 6 | const public_key = fs.readFileSync('./lib/rsa_public_4096.pem', 'utf-8') 7 | const message = 'bajja'; 8 | 9 | 10 | const signer = crypto.createSign('sha256'); 11 | signer.update(message); 12 | signer.end(); 13 | 14 | // Just here to trigger NewRootCertDir 15 | const root_certs = tls.rootCertificates; 16 | 17 | const signature = signer.sign(private_key, 'base64') 18 | const signature_hex = signature.toString('hex') 19 | 20 | const verifier = crypto.createVerify('sha256'); 21 | verifier.update(message); 22 | verifier.end(); 23 | 24 | const verified = verifier.verify(public_key, signature); 25 | 26 | console.log(JSON.stringify({ 27 | message: message, 28 | signature: signature_hex, 29 | verified: verified, 30 | }, null, 2)); 31 | -------------------------------------------------------------------------------- /lib/tasks.js: -------------------------------------------------------------------------------- 1 | function run() { 2 | const timeout = setTimeout(function timeout_cb(something) { 3 | console.log('timeout, arg1:', something); 4 | }, 0, "argument1"); 5 | console.log(timeout); 6 | console.log(timeout._idlePrev); 7 | setImmediate(() => console.log('immediate')); 8 | process.nextTick(() => console.log('nextTick')); 9 | console.log('run completed.'); 10 | } 11 | 12 | class S { 13 | constructor() { 14 | } 15 | }; 16 | console.log({s: new S()}); 17 | run(); 18 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/README.md: -------------------------------------------------------------------------------- 1 | ## node-gyp OpenSSL include issue 2 | 3 | Issue description: https://github.com/nodejs/node/issues/40575 4 | 5 | ### Building 6 | ```console 7 | $ node-gyp configure 8 | $ node-gyp build 9 | ``` 10 | 11 | ### Error 12 | This is the error when running `node-gyp build` (with the addition of some 13 | debugging of the macros involved to help troubleshoot this): 14 | ```console 15 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:83:9: note: ‘#pragma message: OPENSSL_API_COMPAT: 30000’ 16 | 83 | #pragma message "OPENSSL_API_COMPAT: " OPENSSL_MSTR(OPENSSL_API_COMPAT) 17 | | ^~~~~~~ 18 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:141:9: note: ‘#pragma message: OPENSSL_VERSION_MAJOR: 3’ 19 | 141 | #pragma message "OPENSSL_VERSION_MAJOR: " OPENSSL_MSTR(OPENSSL_VERSION_MAJOR) 20 | | ^~~~~~~ 21 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:142:9: note: ‘#pragma message: OPENSSL_VERSION_MINOR: 0’ 22 | 142 | #pragma message "OPENSSL_VERSION_MINOR: " OPENSSL_MSTR(OPENSSL_VERSION_MINOR) 23 | | ^~~~~~~ 24 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:143:9: note: ‘#pragma message: OPENSSL_VERSION: (3 * 1000 + OPENSSL_MINOR * 100)’ 25 | 143 | #pragma message "OPENSSL_VERSION: " OPENSSL_MSTR((OPENSSL_VERSION_MAJOR * 1000 + OPENSSL_MINOR * 100)) 26 | | ^~~~~~~ 27 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:144:9: note: ‘#pragma message: OPENSSL_API_LEVEL: (30000)’ 28 | 144 | #pragma message "OPENSSL_API_LEVEL: " OPENSSL_MSTR(OPENSSL_API_LEVEL) 29 | | ^~~~~~~ 30 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:145:9: note: ‘#pragma message: OPENSSL_API_COMPAT: 30000’ 31 | 145 | #pragma message "OPENSSL_API_COMPAT: " OPENSSL_MSTR(OPENSSL_API_COMPAT) 32 | | ^~~~~~~ 33 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:146:9: note: ‘#pragma message: OPENSSL_CONFIGURED_API: OPENSSL_CONFIGURED_API’ 34 | 146 | #pragma message "OPENSSL_CONFIGURED_API: " OPENSSL_MSTR(OPENSSL_CONFIGURED_API) 35 | | ^~~~~~~ 36 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:148:4: error: #error "The requested API level higher than the configured API compatibility level" 37 | 148 | # error "The requested API level higher than the configured API compatibility level" 38 | | ^~~~~ 39 | make: *** [example.target.mk:113: Release/obj.target/example/example.o] Error 1 40 | ``` 41 | Notice that `OPENSSL_CONFIGURED_API` has not been set which is the causs of 42 | the error message above. 43 | 44 | Now macros.c has the following include: 45 | ```c 46 | #include 47 | #include 48 | ``` 49 | In Node.js this is a "redirect" openssl header which choose different headers8 50 | depending of whether ASM of NO_ASM was configured: 51 | ```c 52 | #if defined(OPENSSL_NO_ASM) 53 | # include "./opensslconf_no-asm.h" 54 | #else 55 | # include "./opensslconf_asm.h" 56 | #endif 57 | ``` 58 | And if we assume this using ASM this would end up in: 59 | ```c 60 | #elif defined(OPENSSL_LINUX) && defined(__x86_64__) 61 | # include "./archs/linux-x86_64/asm/include/openssl/opensslconf.h 62 | ``` 63 | And this file does not include configuration.h. 64 | 65 | In OpenSSL the following headers are generated: 66 | ```console 67 | $ find . -name '*.in' 68 | ./include/openssl/conf.h.in 69 | ./include/openssl/lhash.h.in 70 | ./include/openssl/crmf.h.in 71 | ./include/openssl/err.h.in 72 | ./include/openssl/safestack.h.in 73 | ./include/openssl/ess.h.in 74 | ./include/openssl/opensslv.h.in 75 | ./include/openssl/ocsp.h.in 76 | ./include/openssl/pkcs12.h.in 77 | ./include/openssl/asn1.h.in 78 | ./include/openssl/bio.h.in 79 | ./include/openssl/x509.h.in 80 | ./include/openssl/x509v3.h.in 81 | ./include/openssl/cmp.h.in 82 | ./include/openssl/pkcs7.h.in 83 | ./include/openssl/srp.h.in 84 | ./include/openssl/x509_vfy.h.in 85 | ./include/openssl/asn1t.h.in 86 | ./include/openssl/ct.h.in 87 | ./include/openssl/fipskey.h.in 88 | ./include/openssl/ui.h.in 89 | ./include/openssl/configuration.h.in 90 | ./include/openssl/crypto.h.in 91 | ./include/openssl/cms.h.in 92 | ./include/openssl/ssl.h.in 93 | ./include/crypto/bn_conf.h.in 94 | ./include/crypto/dso_conf.h.in 95 | ``` 96 | And notice that opensslconf.h is not among these. 97 | What I think is happening is that we have an old opensslconf.h which may 98 | have been generated with past versions but is not generated any more, instead 99 | `configuration.h` is the file being generated. 100 | 101 | If we replace this header, 102 | `/.cache/node-gyp/17.8.0/include/node/openssl/opensslconf.h`, with the header 103 | from OpenSSL 3.0: 104 | ```console 105 | $ cp opensslconf.h /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/opensslconf.h 106 | ``` 107 | And the run the build again: 108 | ```console 109 | $ node-gyp build 110 | gyp info it worked if it ends with ok 111 | gyp info using node-gyp@9.0.0 112 | gyp info using node@17.8.0 | linux | x64 113 | gyp info spawn make 114 | gyp info spawn args [ 'BUILDTYPE=Release', '-C', 'build' ] 115 | make: Entering directory '/home/danielbevenius/work/nodejs/node-gyp-openssl-issue/build' 116 | CC(target) Release/obj.target/example/example.o 117 | In file included from /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/././archs/linux-x86_64/asm/include/openssl/ssl.h:21, 118 | from /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/./ssl_asm.h:11, 119 | from /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/ssl.h:4, 120 | from ../example.c:1: 121 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:83:9: note: ‘#pragma message: OPENSSL_API_COMPAT: 30000’ 122 | 83 | #pragma message "OPENSSL_API_COMPAT: " OPENSSL_MSTR(OPENSSL_API_COMPAT) 123 | | ^~~~~~~ 124 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:141:9: note: ‘#pragma message: OPENSSL_VERSION_MAJOR: 3’ 125 | 141 | #pragma message "OPENSSL_VERSION_MAJOR: " OPENSSL_MSTR(OPENSSL_VERSION_MAJOR) 126 | | ^~~~~~~ 127 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:142:9: note: ‘#pragma message: OPENSSL_VERSION_MINOR: 0’ 128 | 142 | #pragma message "OPENSSL_VERSION_MINOR: " OPENSSL_MSTR(OPENSSL_VERSION_MINOR) 129 | | ^~~~~~~ 130 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:143:9: note: ‘#pragma message: OPENSSL_API_LEVEL: (30000)’ 131 | 143 | #pragma message "OPENSSL_API_LEVEL: " OPENSSL_MSTR(OPENSSL_API_LEVEL) 132 | | ^~~~~~~ 133 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:144:9: note: ‘#pragma message: OPENSSL_API_COMPAT: 30000’ 134 | 144 | #pragma message "OPENSSL_API_COMPAT: " OPENSSL_MSTR(OPENSSL_API_COMPAT) 135 | | ^~~~~~~ 136 | /home/danielbevenius/.cache/node-gyp/17.8.0/include/node/openssl/macros.h:145:9: note: ‘#pragma message: OPENSSL_CONFIGURED_API: 30000’ 137 | 145 | #pragma message "OPENSSL_CONFIGURED_API: " OPENSSL_MSTR(OPENSSL_CONFIGURED_API) 138 | | ^~~~~~~ 139 | SOLINK_MODULE(target) Release/obj.target/example.node 140 | COPY Release/example.node 141 | make: Leaving directory '/home/danielbevenius/work/nodejs/node-gyp-openssl-issue/build' 142 | gyp info ok 143 | ``` 144 | 145 | ### Solution 146 | So we should remove this template from Node.js: 147 | ```console 148 | $ git st . 149 | On branch opensslconf 150 | Changes to be committed: 151 | (use "git restore --staged ..." to unstage) 152 | deleted: deps/openssl/config/opensslconf.h 153 | deleted: deps/openssl/config/opensslconf.h.tmpl 154 | deleted: deps/openssl/config/opensslconf_no-asm.h 155 | ``` 156 | 157 | After this change we need to regenerate the headers: 158 | ```console 159 | $ cd deps/openssl/config 160 | $ make 161 | $ cd - 162 | ``` 163 | Next we can create the tar-headers file (by makeing some updates to the Makefile 164 | to allow this to be built for a non-release): 165 | ``` 166 | $ make node-v19.0.0.tar-headers 167 | ``` 168 | First we can revert the headers: 169 | ```console 170 | $ node-gyp remove v17.8.0 171 | ``` 172 | And running node-gyp rebuild should fail with the above error message. 173 | 174 | And then unzip the new headers to replace the headers used by node-gyp: 175 | ```console 176 | $ tar xvzf node-v17.9.1-headers.tar.gz -C ~/.cache/node-gyp/17.8.0 --strip 1 177 | ``` 178 | And this running node-gyp should succeed: 179 | ```console 180 | $ node-gyp build 181 | gyp info it worked if it ends with ok 182 | gyp info using node-gyp@9.0.0 183 | gyp info using node@17.8.0 | linux | x64 184 | gyp info spawn make 185 | gyp info spawn args [ 'BUILDTYPE=Release', '-C', 'build' ] 186 | make: Entering directory '/home/danielbevenius/work/nodejs/learning-nodejs/node-gyp-openssl-issue/build' 187 | make: Nothing to be done for 'all'. 188 | make: Leaving directory '/home/danielbevenius/work/nodejs/learning-nodejs/node-gyp-openssl-issue/build' 189 | ``` 190 | 191 | There are addons in Node.js that include openssl/ssl.h but this error does not 192 | happen for those and I've not had time to find out why this is the case. 193 | 194 | ### OPENSSL_API_COMPAT 195 | This macro specifies what is exposed to be exposed by openssl headers. 196 | So if we want to only include stuff (functions, constants, macros) from 197 | 3.0 then we would use: 198 | ```python 199 | "defines": [ "OPENSSL_API_COMPAT=30000"] 200 | ``` 201 | 202 | But if we only want older stuff to be exposed, for example also 1.1.1 we could 203 | use: (1 * 1000 + 1 * 100 + 1 = 11001 204 | ```python 205 | "defines": [ "OPENSSL_API_COMPAT=11001"] 206 | ``` 207 | 208 | ```console 209 | $ node-gyp build 210 | ``` 211 | 212 | ### Debug macros 213 | To see the actual values of macros we can add the following pre-processor 214 | statements, for example to 215 | `/.cache/node-gyp/17.8.0/include/node/openssl/macros.h`: 216 | ```c 217 | #pragma message "OPENSSL_API_COMPAT: " OPENSSL_MSTR(OPENSSL_API_COMPAT) 218 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "example", 5 | "sources": [ "example.c" ], 6 | #"defines": [ "OPENSSL_API_COMPAT=30000"] 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/example.c: -------------------------------------------------------------------------------- 1 | #include "openssl/ssl.h" 2 | 3 | int main(int argc, char** argv) { 4 | return 0; 5 | } 6 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/node-v17.9.1-headers.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danbev/learning-nodejs/69fdbc8e9b8e42ee14389faf6d13aa6a788b5360/node-gyp-openssl-issue/node-v17.9.1-headers.tar.gz -------------------------------------------------------------------------------- /node-gyp-openssl-issue/node_opensslconf.h: -------------------------------------------------------------------------------- 1 | #undef OPENSSL_LINUX 2 | #if defined(__linux) && !defined(__ANDROID__) 3 | # define OPENSSL_LINUX 1 4 | #endif 5 | 6 | #if defined(OPENSSL_NO_ASM) 7 | # include "./opensslconf_no-asm.h" 8 | #else 9 | # include "./opensslconf_asm.h" 10 | #endif 11 | 12 | /* GOST is not included in all platform */ 13 | #ifndef OPENSSL_NO_GOST 14 | # define OPENSSL_NO_GOST 15 | #endif 16 | /* HW_PADLOCK is not included in all platform */ 17 | #ifndef OPENSSL_NO_HW_PADLOCK 18 | # define OPENSSL_NO_HW_PADLOCK 19 | #endif 20 | /* musl in Alpine Linux does not support getcontext etc.*/ 21 | #if defined(OPENSSL_LINUX) && !defined(__GLIBC__) && !defined(__clang__) 22 | # define OPENSSL_NO_ASYNC 23 | #endif 24 | -------------------------------------------------------------------------------- /node-gyp-openssl-issue/opensslconf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use 5 | * this file except in compliance with the License. You can obtain a copy 6 | * in the file LICENSE in the source distribution or at 7 | * https://www.openssl.org/source/license.html 8 | */ 9 | 10 | #ifndef OPENSSL_OPENSSLCONF_H 11 | # define OPENSSL_OPENSSLCONF_H 12 | # pragma once 13 | 14 | # include 15 | # include 16 | 17 | #endif /* OPENSSL_OPENSSLCONF_H */ 18 | -------------------------------------------------------------------------------- /notes/async_hooks.md: -------------------------------------------------------------------------------- 1 | ### AsyncHooks 2 | AsyncHooks provide an API for tracking async resources and this page contains 3 | notes about this works. 4 | 5 | ### AsyncHook class 6 | The class AsyncHook is declared in `src/env.h` and has the following private 7 | fields: 8 | ```c++ 9 | // Stores the ids of the current execution context stack. 10 | AliasedFloat64Array async_ids_stack_; 11 | // Attached to a Uint32Array that tracks the number of active hooks for each type. 12 | AliasedUint32Array fields_; 13 | // Attached to a Float64Array that tracks the state of async resources. 14 | AliasedFloat64Array async_id_fields_; 15 | v8::Global execution_async_resources_; 16 | ``` 17 | What is AliasedFloat64Array? 18 | ```c++ 19 | typedef AliasedBufferBase AliasedFloat64Array; 20 | ``` 21 | And if we look at AliasedBufferBase: 22 | ```c++ 23 | template ::value>> 27 | class AliasedBufferBase { 28 | public: 29 | AliasedBufferBase(v8::Isolate* isolate, const size_t count) 30 | : isolate_(isolate), count_(count), byte_offset_(0) { 31 | const v8::HandleScope handle_scope(isolate_); 32 | const size_t size_in_bytes = 33 | MultiplyWithOverflowCheck(sizeof(NativeT), count); 34 | v8::Local ab = v8::ArrayBuffer::New(isolate_, size_in_bytes); 35 | buffer_ = static_cast(ab->GetBackingStore()->Data()); 36 | 37 | // allocate v8 TypedArray 38 | v8::Local js_array = V8T::New(ab, byte_offset_, count); 39 | js_array_ = v8::Global(isolate, js_array); 40 | } 41 | }; 42 | ``` 43 | In this case `NativeT`(native type) will be set to `double`, and the V8 Type will 44 | be Float64Array. 45 | 46 | `async_ids_stack_` is create in the constructor of AsyncHooks: 47 | ```c++ 48 | inline AsyncHooks::AsyncHooks() 49 | : async_ids_stack_(env()->isolate(), 16 * 2), 50 | fields_(env()->isolate(), kFieldsCount), 51 | async_id_fields_(env()->isolate(), kUidFieldsCount) { 52 | ``` 53 | So we can see that we are creating a AliasedBuffer with a count of 32 and this 54 | is a TypedArray. 55 | ```c++ 56 | AliasedBufferBase(v8::Isolate* isolate, const size_t count) 57 | : isolate_(isolate), count_(count), byte_offset_(0) { 58 | ``` 59 | The entries in this "stack" will be two double values, the async_id and the 60 | trigger_async_id: 61 | `async_context` is a struct defined in `src/node.h`: 62 | ```c++ 63 | typedef double async_id; 64 | struct async_context { 65 | ::node::async_id async_id; 66 | ::node::async_id trigger_async_id; 67 | }; 68 | ``` 69 | These values are pushed into the async_ids_stack_ (which is an TypedArray which 70 | it just a area of memory (ArrayBuffer) that we treat as we wish. In this case 71 | it is used as a stack: 72 | ```c++ 73 | inline void AsyncHooks::push_async_context(double async_id, 74 | double trigger_async_id, 75 | v8::Local resource) { 76 | 77 | uint32_t offset = fields_[kStackLength]; 78 | if (offset * 2 >= async_ids_stack_.Length()) 79 | grow_async_ids_stack(); 80 | async_ids_stack_[2 * offset] = async_id_fields_[kExecutionAsyncId]; 81 | async_ids_stack_[2 * offset + 1] = async_id_fields_[kTriggerAsyncId]; 82 | fields_[kStackLength] += 1; 83 | async_id_fields_[kExecutionAsyncId] = async_id; 84 | async_id_fields_[kTriggerAsyncId] = trigger_async_id; 85 | ``` 86 | The AsyncHooks class has an enum named Fields and one named UidFields 87 | ```c++ 88 | enum Fields { 89 | kInit, 90 | kBefore, 91 | kAfter, 92 | kDestroy, 93 | kPromiseResolve, 94 | kTotals, 95 | kCheck, 96 | kStackLength, 97 | kFieldsCount, 98 | }; 99 | 100 | enum UidFields { 101 | kExecutionAsyncId, 102 | kTriggerAsyncId, 103 | kAsyncIdCounter, 104 | kDefaultTriggerAsyncId, 105 | kUidFieldsCount, 106 | }; 107 | ``` 108 | So we first get the current size or the stack which is stored in a 109 | `AliasedUint32Array`. And we check if the stack need to grow. 110 | Next we have: 111 | ```c++ 112 | async_ids_stack_[2 * offset] = async_id_fields_[kExecutionAsyncId]; 113 | async_ids_stack_[2 * offset + 1] = async_id_fields_[kTriggerAsyncId]; 114 | ``` 115 | Notice the multipying by 2 which is to take into account that the two values 116 | that make up an entry in this "stack". 117 | The first line will set the new stack entry to be the current executions resources 118 | id. Then the current async trigger id will be set (in the same stack entry). 119 | 120 | `fields_` is used a count of how many active hooks there are for the different 121 | types of hooks. The types are in the Fields enum and I believe they are only 122 | kInit, kBefore, kAfter, kDestroy, kPromiseResolve, and the others are for house 123 | keeping track of the total number of hooks (kTotals), kCheck is used to toggle 124 | if checks should be performed or not on the async_ids for example, kStackLength 125 | is the holds size of the stack, and kFieldsCount just contains the number of 126 | elements in the Fields enum. 127 | 128 | `async_id_fields_` is create in the constructor of AsyncHooks and is given a 129 | kUidFieldsCount and these contain 130 | 131 | `async_ids_stack_` is the stack of async ids. 132 | 133 | Lets take a look at executionAsyncId(): 134 | ```js 135 | async_hooks.executionAsyncId() 136 | ``` 137 | This is exported by lib/async_hooks.js but implemented in lib/internal/async_hooks.js. 138 | ```js 139 | function executionAsyncId() { 140 | return async_id_fields[kExecutionAsyncId]; 141 | } 142 | ``` 143 | When we require('async_hooks') this will in turn require('lib/internal/async_hooks'). 144 | Where we have: 145 | ```js 146 | const async_wrap = internalBinding('async_wrap'); 147 | ``` 148 | And is we take a look at `src/async_wrap.cc` and its Intialize method: 149 | ```c++ 150 | FORCE_SET_TARGET_FIELD(target, 151 | "async_hook_fields", 152 | env->async_hooks()->fields().GetJSArray()); 153 | ``` 154 | We can see that we are setting up the export object which `async_wrap` will 155 | be a reference to: 156 | ```console 157 | (lldb) jlh target 158 | 0x222d1f46acf1: [JS_API_OBJECT_TYPE] 159 | - map: 0x1e72ee4a45e9 [FastProperties] 160 | - prototype: 0x184584387929 161 | - elements: 0x062e98140b29 [HOLEY_ELEMENTS] 162 | - embedder fields: 1 163 | - properties: 0x222d1f46b451 { 164 | #setupHooks: 0x13eae07aaf01 (const data field 0) properties[0] 165 | #pushAsyncContext: 0x13eae07ab009 (const data field 1) properties[1] 166 | #popAsyncContext: 0x13eae07ab111 (const data field 2) properties[2] 167 | #queueDestroyAsyncId: 0x13eae07ab219 (const data field 3) properties[3] 168 | #enablePromiseHook: 0x13eae07ab349 (const data field 4) properties[4] 169 | #disablePromiseHook: 0x13eae07ab451 (const data field 5) properties[5] 170 | #registerDestroyHook: 0x13eae07ab559 (const data field 6) properties[6] 171 | #async_hook_fields: 0x184584382d99 (const data field 7) properties[7] 172 | #async_id_fields: 0x184584382e31 (const data field 8) properties[8] 173 | #execution_async_resources: 0x184584382e89 (const data field 9) properties[9] 174 | #async_ids_stack: 0x184584382d01 (const data field 10) properties[10] 175 | } 176 | - embedder fields = { 177 | 0x062e98140471 178 | } 179 | ``` 180 | There are more properties like `constants` but I'm not showing the all to save 181 | some space. Looking closer as async_id_fields can see that it has 8 fields: 182 | ``` 183 | #async_id_fields: 0x184584382e31 (const data field 8) properties[8] 184 | ``` 185 | ```console 186 | (lldb) jlh env->async_hooks()->async_id_fields().GetJSArray() 187 | 0x184584382e31: [JSTypedArray] 188 | - map: 0x1e72ee481b01 [FastProperties] 189 | - prototype: 0x3a5af5dcb8d9 190 | - elements: 0x062e98141dd9 [FLOAT64ELEMENTS] 191 | - embedder fields: 2 192 | - buffer: 0x184584382df1 193 | - byte_offset: 0 194 | - byte_length: 32 195 | - length: 4 196 | - data_ptr: 0x56d6b20 197 | - base_pointer: 0 198 | - external_pointer: 0x56d6b20 199 | - properties: 0x062e98140b29 {} 200 | - elements: 0x062e98141dd9 { 201 | 0-1: 0 202 | 2: 1 203 | 3: -1 204 | } 205 | - embedder fields = { 206 | 0, aligned pointer: 0 207 | 0, aligned pointer: 0 208 | } 209 | ``` 210 | Where is async_id_fields[kExecutionAsyncId] set to 1? 211 | This is done in `src/node.cc`: 212 | ```c++ 213 | MaybeLocal StartExecution(Environment* env, StartExecutionCallback cb) { 214 | InternalCallbackScope callback_scope( 215 | env, 216 | Object::New(env->isolate()), 217 | { 1, 0 }, // async_id, async_trigger_id 218 | InternalCallbackScope::kSkipAsyncHooks); 219 | ``` 220 | And if we set a break point in InternalCallbackScope's constructor we can 221 | inspect the values: 222 | ```console 223 | lldb) expr asyncContext 224 | (const node::async_context) $8 = (async_id = 1, trigger_async_id = 0) 225 | ``` 226 | Next we have the following line: 227 | ``` 228 | async_ids_stack_[2 * offset] = async_id_fields_[kExecutionAsyncId]; 229 | async_ids_stack_[2 * offset + 1] = async_id_fields_[kTriggerAsyncId]; 230 | ``` 231 | So, we are setting two values which is the execution async id, and the trigger 232 | async id. 233 | ``console 234 | (lldb) expr async_id_fields_[node::AsyncHooks::UidFields::kExecutionAsyncId] 235 | (node::AliasedBufferBase::Reference) $21 = {} 236 | (double) $23 = 0 237 | ``` 238 | The first line will set the new stack entry to be the current executions resources 239 | id. Then the current async trigger id will be set (in the same stack entry). 240 | Next we increment the stack size: 241 | ```c++ 242 | fields_[kStackLength] += 1; 243 | ``` 244 | And after that we set the current async execution id to the passed-in async_id 245 | and the async trigger id in async_id_fields: 246 | ```c++ 247 | async_id_fields_[kExecutionAsyncId] = async_id; 248 | async_id_fields_[kTriggerAsyncId] = trigger_async_id; 249 | ``` 250 | And finally we add the current async resource to an Array of async resources: 251 | ```c++ 252 | auto resources = execution_async_resources(); 253 | USE(resources->Set(env()->context(), offset, resource)); 254 | ``` 255 | We can find the declaration of it in `src/env.h`: 256 | ```c++ 257 | v8::Global execution_async_resources_; 258 | ``` 259 | When the InternalCallbackScope's destructor is run Close will be called which 260 | in turn will calls pop_async_context passing in the async_id. It will do the 261 | revers of push which makes sense. 262 | So to recap, we have `async_ids_stack` which contains the execution async ids and 263 | the async trigger ids. And `async_id_fields` is a storage area for various things 264 | and two of them store the current execution async id and current async trigger id. 265 | 266 | 267 | ### InternalCallbackScope 268 | The declaration for this class can be found in `src/node_internals.h` and the 269 | implementation in `node/api/callback.cc`. 270 | ```c++ 271 | InternalCallbackScope::InternalCallbackScope(Environment* env, 272 | Local object, 273 | const async_context& asyncContext, 274 | int flags) 275 | ``` 276 | `async_context` is a struct defined in `src/node.h`: 277 | ```c++ 278 | typedef double async_id; 279 | struct async_context { 280 | ::node::async_id async_id; 281 | ::node::async_id trigger_async_id; 282 | }; 283 | ``` 284 | Now, lets take a closer look at the InternalCallbackScope constructor, and the 285 | the destructor as it is an implemention of RAII. 286 | 287 | ```c++ 288 | InternalCallbackScope::~InternalCallbackScope() { 289 | Close(); 290 | env_->PopAsyncCallbackScope(); 291 | } 292 | ``` 293 | Now, `Close` is interesting. 294 | ```c++ 295 | TickInfo* tick_info = env_->tick_info(); 296 | 297 | if (!tick_info->has_tick_scheduled()) { 298 | MicrotasksScope::PerformCheckpoint(env_->isolate()); 299 | } 300 | 301 | HandleScope handle_scope(env_->isolate()); 302 | Local process = env_->process_object(); 303 | Local tick_callback = env_->tick_callback_function(); 304 | 305 | if (tick_callback->Call(env_->context(), process, 0, nullptr).IsEmpty()) { 306 | failed_ = true; 307 | } 308 | ```c++ 309 | And we can inspect `tick_callback` to see what will be run: 310 | ```console 311 | (lldb) jlh tick_callback 312 | 0x32e4e57c2dc1: [Function] 313 | - map: 0x19edc0cc0679 [FastProperties] 314 | - prototype: 0x3b0e0cd40a39 315 | - elements: 0x3eab4f380b29 [HOLEY_ELEMENTS] 316 | - function prototype: 317 | - initial_map: 318 | - shared_info: 0x325f36fad639 319 | - name: 0x325f36fad2b9 320 | - formal_parameter_count: 0 321 | - safe_to_skip_arguments_adaptor 322 | - kind: NormalFunction 323 | - context: 0x32e4e57e3149 324 | - code: 0x23b2d5c43341 325 | - interpreted 326 | - bytecode: 0x28b4edb53d91 327 | - source code: () { 328 | let tock; 329 | do { 330 | while (tock = queue.shift()) { 331 | const asyncId = tock[async_id_symbol]; 332 | emitBefore(asyncId, tock[trigger_async_id_symbol], tock); 333 | 334 | try { 335 | const callback = tock.callback; 336 | if (tock.args === undefined) { 337 | callback(); 338 | } else { 339 | const args = tock.args; 340 | switch (args.length) { 341 | case 1: callback(args[0]); break; 342 | case 2: callback(args[0], args[1]); break; 343 | case 3: callback(args[0], args[1], args[2]); break; 344 | case 4: callback(args[0], args[1], args[2], args[3]); break; 345 | default: callback(...args); 346 | } 347 | } 348 | } finally { 349 | if (destroyHooksExist()) 350 | emitDestroy(asyncId); 351 | } 352 | 353 | emitAfter(asyncId); 354 | } 355 | runMicrotasks(); 356 | } while (!queue.isEmpty() || processPromiseRejections()); 357 | setHasTickScheduled(false); 358 | setHasRejectionToWarn(false); 359 | } 360 | - properties: 0x3eab4f380b29 { 361 | #length: 0x1c1042c004a1 (const accessor descriptor) 362 | #name: 0x1c1042c00431 (const accessor descriptor) 363 | #prototype: 0x1c1042c00511 (const accessor descriptor) 364 | } 365 | - feedback vector: not available 366 | ``` 367 | TODO: trace the above js function 368 | 369 | 370 | -------------------------------------------------------------------------------- /notes/debugging.md: -------------------------------------------------------------------------------- 1 | ### Debugging Node.js applications/tests 2 | 3 | #### Breakpoints in child process tests 4 | This section shows one way of setting breakpoints in tests that use child 5 | processes. The problem that one can run into when debugging these are that a 6 | test is started using lldb like this: 7 | ``` 8 | $ lldb -- out/Debug/node /home/danielbevenius/work/nodejs/openssl/test/parallel/test-crypto-secure-heap.js 9 | ``` 10 | And then a break point is set in a C++ function of interest, for example: 11 | ```console 12 | (lldb) br s -r DiffieHellman::* 13 | (lldb) r 14 | ``` 15 | The breakpoint is never hit which can be somewhat suprising because you know 16 | that one of the functions will be called and you can see output to the console. 17 | 18 | The problem is that when the test is run it will spawn a new process and it is 19 | that process that we really are interested in setting the breakpoint in. 20 | 21 | What can be done is instead start debugging the javascript application and then 22 | when the child process has been created attach to it and then set our breakpoint: 23 | 24 | In this case a break point will be added in the child process code: 25 | ```js 26 | if (process.argv[2] === 'child') { 27 | 28 | const a = secureHeapUsed(); 29 | 30 | assert(a); 31 | assert.strictEqual(typeof a, 'object'); 32 | assert.strictEqual(a.total, 65536); 33 | assert.strictEqual(a.min, 4); 34 | assert.strictEqual(a.used, 0); 35 | 36 | { 37 | const dh1 = createDiffieHellman(common.hasFipsCrypto ? 1024 : 256); 38 | ``` 39 | The last line above will have a breakpoint set on it and we can add 40 | `--inspect-brk` to the fork function call argument: 41 | ```javascript 42 | 43 | const child = fork( 44 | process.argv[1], 45 | ['child'], 46 | { execArgv: ['--secure-heap=65536', '--secure-heap-min=4', '--inspect-brk'] }); 47 | ``` 48 | We start the test just like we normally would: 49 | ```console 50 | $ out/Debug/node /home/danielbevenius/work/nodejs/openssl/test/parallel/test-crypto-secure-heap.js 51 | Debugger listening on ws://127.0.0.1:9229/2bc45af4-47cf-4b9e-8095-a5e2bfaec1c0 52 | For help, see: https://nodejs.org/en/docs/inspector 53 | Debugger attached. 54 | ``` 55 | Notice that the debugger is waiting to be connected to. We do this by opening 56 | chrome and entering `chrome://inspect`. Now we can contine until we hit the 57 | breakpoint we set. 58 | 59 | Next we attach to this process by first finding the process id: 60 | ```console 61 | $ ps ef | grep inspect-brk 62 | ``` 63 | Then we start lldb and attach to the process: 64 | ```console 65 | $ lldb -- out/Debug/node 66 | (lldb) process attach --pid 1275062 67 | (lldb) br s -n DiffieHellman::New 68 | Breakpoint 1: where = node`node::crypto::DiffieHellman::New(v8::FunctionCallbackInfo const&) + 23 at crypto_dh.cc:201:45, address = 0x00000000012189bd 69 | ``` 70 | And now we can step through in the chrome devtools debugger and our breakpoint 71 | will be triggered: 72 | ```console 73 | Process 1275062 stopped 74 | * thread #1, name = 'node', stop reason = breakpoint 1.1 75 | frame #0: 0x00000000012189bd node`node::crypto::DiffieHellman::New(args=0x00007fff25e4e210) at crypto_dh.cc:201:45 76 | 189 if (group == nullptr) 77 | 190 return THROW_ERR_CRYPTO_UNKNOWN_DH_GROUP(env); 78 | 191 79 | 192 initialized = diffieHellman->Init(group->prime, 80 | 193 group->prime_size, 81 | 194 group->gen); 82 | 195 if (!initialized) 83 | 196 THROW_ERR_CRYPTO_INITIALIZATION_FAILED(env); 84 | 197 } 85 | 198 86 | 199 87 | 200 void DiffieHellman::New(const FunctionCallbackInfo& args) { 88 | -> 201 Environment* env = Environment::GetCurrent(args); 89 | 202 DiffieHellman* diffieHellman = 90 | ``` 91 | 92 | -------------------------------------------------------------------------------- /notes/eventloop.md: -------------------------------------------------------------------------------- 1 | ### Event Loop 2 | It all starts with the javascript file to be executed, which is you main program. 3 | ``` 4 | (while there are things to process) 5 | ------------> javascript.js --+------------------↓ 6 | ↑ | setTimeout/SetInterval 7 | | | JavaScript callbacks 8 | | ↓ 9 | | | network/disk/child_processes 10 | | | JavaScript callbacks 11 | | ↓ 12 | | | setImmedate 13 | ↑ | JavaScript callbacks 14 | \ ↓ 15 | \ | close events 16 | \ | JavaScript callbacks 17 | \ ↓ 18 | \ | 19 | <----------- process.exit (event) <--------------< 20 | ``` 21 | 22 | -JavaScript callbacks 23 | ``` 24 | --------> callback ----+------------------------------------↓ 25 | ↑ | 26 | | nextTick callback 27 | | ↓ 28 | | Resolve Promies 29 | | ↓ 30 | | | 31 | ↑------------------------------------< 32 | ``` 33 | 34 | ### setTimeout/setInternval/setImmediate 35 | When node starts `internal/bootstrap/node` will be run which has the following 36 | lines of code: 37 | ```js 38 | const timers = require('timers'); 39 | defineOperation(global, 'clearInterval', timers.clearInterval); 40 | defineOperation(global, 'clearTimeout', timers.clearTimeout); 41 | defineOperation(global, 'setInterval', timers.setInterval); 42 | defineOperation(global, 'setTimeout', timers.setTimeout); 43 | 44 | defineOperation(global, 'queueMicrotask', queueMicrotask); 45 | defineOperation(global, 'setImmediate', timers.setImmediate); 46 | ``` 47 | 48 | For example, using `setTimeout` could look like this: 49 | ```js 50 | const timeout = setTimeout(function timeout_cb(something) { 51 | console.log('timeout. arg1', something); 52 | }, 0, "argument1"); 53 | ``` 54 | If we look in `lib/timers.js` we can find the definitions for setTimeout: 55 | ```js 56 | const { 57 | Timeout, 58 | ... 59 | insert 60 | } = require('internal/timers'); 61 | 62 | function setTimeout(callback, after, arg1, arg2, arg3) { 63 | ... 64 | const timeout = new Timeout(callback, after, args, false, true); 65 | insert(timeout, timeout._idleTimeout); 66 | 67 | return timeout; 68 | } 69 | ``` 70 | Timeout can be found in `lib/internal/timers.js`: 71 | ```js 72 | function Timeout(callback, after, args, isRepeat, isRefed) { 73 | ... 74 | this._idleTimeout = after; 75 | this._idlePrev = this; 76 | this._idleNext = this; 77 | this._idleStart = null; 78 | this._onTimeout = callback; 79 | this._timerArgs = args; 80 | this._repeat = isRepeat ? after : null; 81 | this._destroyed = false; 82 | } 83 | ``` 84 | We can log the instance to the console to see some real values: 85 | ```console 86 | Timeout { 87 | _idleTimeout: 1, 88 | _idlePrev: [TimersList], 89 | _idleNext: [TimersList], 90 | _idleStart: 41, 91 | _onTimeout: [Function: timeout_cb], 92 | _timerArgs: undefined, 93 | _repeat: null, 94 | _destroyed: false, 95 | [Symbol(refed)]: true, 96 | [Symbol(asyncId)]: 2, 97 | [Symbol(triggerId)]: 1 98 | } 99 | ``` 100 | If we look back at where the Timeout constructor is called we see that the the 101 | instance created is passed into the `insert` function: 102 | ```js 103 | insert(timeout, timeout._idleTimeout); 104 | ``` 105 | And insert can be found in `lib/internal/timers.js`. 106 | ```js 107 | const { 108 | scheduleTimer, 109 | toggleTimerRef, 110 | getLibuvNow, 111 | immediateInfo 112 | } = internalBinding('timers'); 113 | 114 | const timerListQueue = new PriorityQueue(compareTimersLists, setPosition); 115 | 116 | function insert(item, msecs, start = getLibuvNow()) { 117 | item._idleStart = start; 118 | let list = timerListMap[msecs]; 119 | if (list === undefined) { 120 | debug('no %d list was found in insert, creating a new one', msecs); 121 | const expiry = start + msecs; 122 | timerListMap[msecs] = list = new TimersList(expiry, msecs); 123 | timerListQueue.insert(list); 124 | if (nextExpiry > expiry) { 125 | scheduleTimer(msecs); 126 | nextExpiry = expiry; 127 | } 128 | } 129 | L.append(list, item); 130 | } 131 | ``` 132 | So we have a map which is keyed with the expiry, and the value is a TimerList. 133 | TimerList: 134 | ```js 135 | function TimersList(expiry, msecs) { 136 | this._idleNext = this; 137 | this._idlePrev = this; 138 | this.expiry = expiry; 139 | this.id = timerListId++; 140 | this.msecs = msecs; 141 | this.priorityQueuePosition = null; 142 | } 143 | ``` 144 | So we create a new TimersList instance which is then inserted into the 145 | `timerListQueue` which is of type PriorityQueue and can be found in 146 | `internal/priority_queue`. AFter this `scheduleTimer` is called which can be 147 | found in `src/timers.cc` 148 | ```c++ 149 | void ScheduleTimer(const FunctionCallbackInfo& args) { 150 | auto env = Environment::GetCurrent(args); 151 | env->ScheduleTimer(args[0]->IntegerValue(env->context()).FromJust()); 152 | } 153 | ``` 154 | And in src/env.cc we can find: 155 | ```c++ 156 | void Environment::ScheduleTimer(int64_t duration_ms) { 157 | if (started_cleanup_) return; 158 | uv_timer_start(timer_handle(), RunTimers, duration_ms, 0); 159 | } 160 | ``` 161 | So we are starting a timer and the callback is `RunTimers` which will be called 162 | when the timer fires/expires. 163 | ```c++ 164 | Local process = env->process_object(); 165 | InternalCallbackScope scope(env, process, {0, 0}); 166 | 167 | Local cb = env->timers_callback_function(); 168 | MaybeLocal ret; 169 | Local arg = env->GetNow(); 170 | // This code will loop until all currently due timers will process. It is 171 | // impossible for us to end up in an infinite loop due to how the JS-side 172 | // is structured. 173 | do { 174 | TryCatchScope try_catch(env); 175 | try_catch.SetVerbose(true); 176 | ret = cb->Call(env->context(), process, 1, &arg); 177 | } while (ret.IsEmpty() && env->can_call_into_js()); 178 | ``` 179 | Notice the [InternalCallbackScope](#internalcallbackscope) that is here. 180 | 181 | 182 | We can inspect `cb` which is the function that will be called. 183 | ```console 184 | (lldb) jlh cb 185 | 0x367b934c2e69: [Function] 186 | - map: 0x07dddd580679 [FastProperties] 187 | - prototype: 0x0700ffe40a39 188 | - elements: 0x0f5694ac0b29 [HOLEY_ELEMENTS] 189 | - function prototype: 190 | - initial_map: 191 | - shared_info: 0x3d84c4812259 192 | - name: 0x0700ffe6c269 193 | - builtin: CompileLazy 194 | - formal_parameter_count: 1 195 | - kind: NormalFunction 196 | - context: 0x1d88087b5e69 197 | - code: 0x1fad42dc33a1 198 | - source code: (now) { 199 | debug('process timer lists %d', now); 200 | nextExpiry = Infinity; 201 | 202 | let list; 203 | let ranAtLeastOneList = false; 204 | while (list = timerListQueue.peek()) { 205 | if (list.expiry > now) { 206 | nextExpiry = list.expiry; 207 | return refCount > 0 ? nextExpiry : -nextExpiry; 208 | } 209 | if (ranAtLeastOneList) 210 | runNextTicks(); 211 | else 212 | ranAtLeastOneList = true; 213 | listOnTimeout(list, now); 214 | } 215 | return 0; 216 | } 217 | - properties: 0x0f5694ac0b29 { 218 | #length: 0x1e915df404a1 (const accessor descriptor) 219 | #name: 0x1e915df40431 (const accessor descriptor) 220 | #prototype: 0x1e915df40511 (const accessor descriptor) 221 | } 222 | - feedback vector: feedback metadata is not available in SFI 223 | ``` 224 | We can see above that the `name` is `processTimers` and this is a function that 225 | is defined in `lib/internal/timers.js`. 226 | 227 | Lets try debugging this: 228 | ```console 229 | $ env NODE_DEBUG=timer lldb -- ./node_g --inspect-brk-node lib/task.js 230 | (lldb) br s -n Environment::RunTimers 231 | (lldb) r 232 | ``` 233 | We can let the continue to the entry of our script and then set a breakpoint 234 | in `processTimers`. This will stop in the breakpoint in RunTimers. 235 | When I'm trying to understand is why there is a while loop there. I was thinking 236 | that it would be enough to just run the processTimers function once. 237 | 238 | 239 | ### InternalCallbackScope 240 | The declaration for this class can be found in `src/node_internals.h` and the 241 | implementation in `node/api/callback.cc`. 242 | ```c++ 243 | InternalCallbackScope::InternalCallbackScope(Environment* env, 244 | Local object, 245 | const async_context& asyncContext, 246 | int flags) 247 | ``` 248 | `async_context` is a struct defined in `src/node.h`: 249 | ```c++ 250 | typedef double async_id; 251 | struct async_context { 252 | ::node::async_id async_id; 253 | ::node::async_id trigger_async_id; 254 | }; 255 | ``` 256 | Now, lets take a closer look at the InternalCallbackScope constructor, and the 257 | the destructor as it is an implemention of RAII. 258 | 259 | ```c++ 260 | InternalCallbackScope::~InternalCallbackScope() { 261 | Close(); 262 | env_->PopAsyncCallbackScope(); 263 | } 264 | ``` 265 | Now, `Close` is interesting. 266 | ```c++ 267 | TickInfo* tick_info = env_->tick_info(); 268 | 269 | if (!tick_info->has_tick_scheduled()) { 270 | MicrotasksScope::PerformCheckpoint(env_->isolate()); 271 | } 272 | 273 | HandleScope handle_scope(env_->isolate()); 274 | Local process = env_->process_object(); 275 | Local tick_callback = env_->tick_callback_function(); 276 | 277 | if (tick_callback->Call(env_->context(), process, 0, nullptr).IsEmpty()) { 278 | failed_ = true; 279 | } 280 | ```c++ 281 | And we can inspect `tick_callback` to see what will be run: 282 | ```console 283 | (lldb) jlh tick_callback 284 | 0x32e4e57c2dc1: [Function] 285 | - map: 0x19edc0cc0679 [FastProperties] 286 | - prototype: 0x3b0e0cd40a39 287 | - elements: 0x3eab4f380b29 [HOLEY_ELEMENTS] 288 | - function prototype: 289 | - initial_map: 290 | - shared_info: 0x325f36fad639 291 | - name: 0x325f36fad2b9 292 | - formal_parameter_count: 0 293 | - safe_to_skip_arguments_adaptor 294 | - kind: NormalFunction 295 | - context: 0x32e4e57e3149 296 | - code: 0x23b2d5c43341 297 | - interpreted 298 | - bytecode: 0x28b4edb53d91 299 | - source code: () { 300 | let tock; 301 | do { 302 | while (tock = queue.shift()) { 303 | const asyncId = tock[async_id_symbol]; 304 | emitBefore(asyncId, tock[trigger_async_id_symbol], tock); 305 | 306 | try { 307 | const callback = tock.callback; 308 | if (tock.args === undefined) { 309 | callback(); 310 | } else { 311 | const args = tock.args; 312 | switch (args.length) { 313 | case 1: callback(args[0]); break; 314 | case 2: callback(args[0], args[1]); break; 315 | case 3: callback(args[0], args[1], args[2]); break; 316 | case 4: callback(args[0], args[1], args[2], args[3]); break; 317 | default: callback(...args); 318 | } 319 | } 320 | } finally { 321 | if (destroyHooksExist()) 322 | emitDestroy(asyncId); 323 | } 324 | 325 | emitAfter(asyncId); 326 | } 327 | runMicrotasks(); 328 | } while (!queue.isEmpty() || processPromiseRejections()); 329 | setHasTickScheduled(false); 330 | setHasRejectionToWarn(false); 331 | } 332 | - properties: 0x3eab4f380b29 { 333 | #length: 0x1c1042c004a1 (const accessor descriptor) 334 | #name: 0x1c1042c00431 (const accessor descriptor) 335 | #prototype: 0x1c1042c00511 (const accessor descriptor) 336 | } 337 | - feedback vector: not available 338 | ``` 339 | TODO: trace the above js function 340 | 341 | 342 | ### nextTick 343 | In `lib/internal/bootstrap/node.js` we have the function `setupTaskQueue` which 344 | can be found in `lib/internal/process/task_queues.js`: 345 | ```js 346 | module.exports = { 347 | setupTaskQueue() { 348 | listenForRejections(); 349 | setTickCallback(processTicksAndRejections); 350 | return { 351 | nextTick, 352 | runNextTicks 353 | }; 354 | }, 355 | queueMicrotask 356 | }; 357 | ``` 358 | And `setTickCallback` is an internal builtin name task_queue which can be 359 | found in `src/node_task_queue.cc` 360 | 361 | ```js 362 | const { 363 | // For easy access to the nextTick state in the C++ land, 364 | // and to avoid unnecessary calls into JS land. 365 | tickInfo, 366 | // Used to run V8's micro task queue. 367 | runMicrotasks, 368 | setTickCallback, 369 | enqueueMicrotask 370 | } = internalBinding('task_queue'); 371 | ``` 372 | And the implementation: 373 | ```c++ 374 | static void SetTickCallback(const FunctionCallbackInfo& args) { 375 | Environment* env = Environment::GetCurrent(args); 376 | CHECK(args[0]->IsFunction()); 377 | env->set_tick_callback_function(args[0].As()); 378 | } 379 | ``` 380 | So we can see that we are just setting the function on the environment which 381 | is what is then used in `src/api/callback.cc`: 382 | ```c++ 383 | Local tick_callback = env_->tick_callback_function(); 384 | ``` 385 | 386 | ### Task queues in node 387 | Upon startup of node `Environment::BootstrapNode` will be called which will 388 | call: 389 | ```c++ 390 | MaybeLocal result = ExecuteBootstrapper( 391 | this, "internal/bootstrap/node", &node_params, &node_args); 392 | ``` 393 | This will invoke `internal/bootstrap/node.js`. 394 | 395 | ```js 396 | const { 397 | setupTaskQueue, 398 | queueMicrotask 399 | } = require('internal/process/task_queues'); 400 | ... 401 | 402 | const { nextTick, runNextTicks } = setupTaskQueue(); 403 | ``` 404 | 405 | And in `task_queues.js` we have: 406 | ```js 407 | module.exports = { 408 | setupTaskQueue() { 409 | // Sets the per-isolate promise rejection callback 410 | listenForRejections(); 411 | // Sets the callback to be run in every tick. 412 | setTickCallback(processTicksAndRejections); 413 | return { 414 | nextTick, 415 | runNextTicks 416 | }; 417 | }, 418 | queueMicrotask 419 | }; 420 | ``` 421 | Notice that `processTicksAndRejections` passed to the `setTickCallback` 422 | function, which is defined in `src/node_task_queue.cc`. 423 | ```js 424 | const { 425 | // For easy access to the nextTick state in the C++ land, 426 | // and to avoid unnecessary calls into JS land. 427 | tickInfo, 428 | // Used to run V8's micro task queue. 429 | runMicrotasks, 430 | setTickCallback, 431 | enqueueMicrotask 432 | } = internalBinding('task_queue'); 433 | ``` 434 | And the implementation looks like this: 435 | ```c++ 436 | static void SetTickCallback(const FunctionCallbackInfo& args) { 437 | Environment* env = Environment::GetCurrent(args); 438 | CHECK(args[0]->IsFunction()); 439 | env->set_tick_callback_function(args[0].As()); 440 | ``` 441 | 442 | 443 | ### process.nextTick 444 | 445 | -------------------------------------------------------------------------------- /notes/heap-snapshot.md: -------------------------------------------------------------------------------- 1 | ### V8 Heap Snapshot builder 2 | This feature in v8 that enables a snapshot of the heap to be taken (not to be 3 | confused with the snapshot generated by mksnapshot or node_mksnapshot. 4 | 5 | Upon startup of Node `LoadEnvironment` will call the following function: 6 | ```c++ 7 | void Environment::InitializeDiagnostics() { 8 | isolate_->GetHeapProfiler()->AddBuildEmbedderGraphCallback( 9 | Environment::BuildEmbedderGraph, this); 10 | ``` 11 | ```console 12 | $ lldb -- ./out/Debug/node -p 'process.versions.v8' 13 | (lldb) br s -n Environment::InitializeDiagnostics 14 | (lldb) r 15 | ``` 16 | This is adding a callback that will be called during heap snapshot generation 17 | is in progress, allowing an embedder to add nodes and edges to the EmbedderGraph 18 | that is passed to this callback. The implementation can be found in env.cc: 19 | ```c++ 20 | void Environment::BuildEmbedderGraph(Isolate* isolate, 21 | EmbedderGraph* graph, 22 | void* data) { 23 | MemoryTracker tracker(isolate, graph); 24 | Environment* env = static_cast(data); 25 | tracker.Track(env); 26 | env->ForEachBaseObject([&](BaseObject* obj) { 27 | if (obj->IsDoneInitializing()) 28 | tracker.Track(obj); 29 | }); 30 | } 31 | ``` 32 | First notice that an instance of `MemoryTracker` is being created. Next, the 33 | environment pointer is casted from the `void* data` pointer. 34 | After that we have the call to `tracker.Track(env)`: 35 | ```c++ 36 | inline void Track(const MemoryRetainer* retainer, 37 | const char* edge_name = nullptr); 38 | ``` 39 | And note that Environment extends MemoryRetainer which why `Track` can be 40 | called: 41 | ```c++ 42 | class Environment : public MemoryRetainer { 43 | ``` 44 | So what does `Track` do (src/memory_tracker-inl.h): 45 | ```c++ 46 | void MemoryTracker::Track(const MemoryRetainer* retainer, 47 | const char* edge_name) { 48 | v8::HandleScope handle_scope(isolate_); 49 | auto it = seen_.find(retainer); 50 | if (it != seen_.end()) { 51 | if (CurrentNode() != nullptr) { 52 | graph_->AddEdge(CurrentNode(), it->second, edge_name); 53 | } 54 | return; // It has already been tracked, no need to call MemoryInfo again 55 | } 56 | MemoryRetainerNode* n = PushNode(retainer, edge_name); 57 | retainer->MemoryInfo(this); 58 | CHECK_EQ(CurrentNode(), n); 59 | CHECK_NE(n->size_, 0); 60 | PopNode(); 61 | ``` 62 | If the MemoryRetainer passed in is in the `seen_` map then an edge/link from 63 | the current node to it will be added to the graph. 64 | If the MemoryRetainer was not in the map a new MemoryRetainerNode will be 65 | created by calling `PushNode`. 66 | ```c++ 67 | MemoryRetainerNode* MemoryTracker::PushNode(const MemoryRetainer* retainer, 68 | const char* edge_name) { 69 | MemoryRetainerNode* n = AddNode(retainer, edge_name); 70 | node_stack_.push(n); 71 | return n; 72 | } 73 | ``` 74 | After the MemoryRetainerNode has been created and added, `retainer->MemoryInfo` 75 | will be called allowing the current MemoryRetainer to add fields to the 76 | MemoryTracker. For example, for Environment this will call 77 | `Environment::MemoryInfo(MemoryTracker* tracker)`: 78 | ```c++ 79 | tracker->TrackField("isolate_data", isolate_data_); 80 | ``` 81 | So we are passing `isolate_data` as the `edge_name`, and in our case this 82 | retainers (isolate_data) has not been added to the map so `Track` will 83 | create a new MemoryRetainerNode by using PushNode like we saw earlier and 84 | then call isolate_data->MemoryInfo(this) which has a few macros which can 85 | be expanded using: 86 | ```console 87 | $ g++ -DNODE_WANT_INTERNALS=true -E -Ideps/uv/include -Ideps/v8/include -Isrc src/env.cc 88 | ``` 89 | ```c++ 90 | void IsolateData::MemoryInfo(MemoryTracker* tracker) const { 91 | tracker->TrackField("async_id_symbol", async_id_symbol()); 92 | ... 93 | ``` 94 | This will call `TrackField` in memory_tracker-inl.h: 95 | ```c++ 96 | template 97 | void MemoryTracker::TrackField(const char* edge_name, 98 | const v8::Local& value, 99 | const char* node_name) { 100 | if (!value.IsEmpty()) 101 | graph_->AddEdge(CurrentNode(), graph_->V8Node(value), edge_name); 102 | } 103 | ``` 104 | TODO: continue this walkthrough. 105 | 106 | 107 | When debugging you might not see correct values for std::string's. For example 108 | you might see: 109 | ```console 110 | (lldb) expr name_ 111 | (std::string) $0 = "error: summary string parsing error" 112 | 113 | ``` 114 | This can be fixed by specifying `-fno-limit-debug-info` in `node.gypi`: 115 | ``` 116 | [ 'debug_node=="true"', { 117 | 'cflags!': [ '-O3' ], 118 | 'cflags': [ '-g', '-O0', '-fno-limit-debug-info' ], 119 | ``` 120 | Note that this makes debug build time longer. 121 | ```console 122 | (lldb) expr name_ 123 | (std::string) $0 = "Environment" 124 | ``` 125 | 126 | #### MemoryTracker 127 | This class is declared in src/memory-tracker.h 128 | ```c++ 129 | class MemoryTracker { 130 | public: 131 | 132 | inline explicit MemoryTracker(v8::Isolate* isolate, 133 | v8::EmbedderGraph* graph) 134 | : isolate_(isolate), graph_(graph) {} 135 | private: 136 | typedef std::unordered_map NodeMap; 137 | 138 | NodeMap seen_; 139 | ``` 140 | `seen_` is a map and the keys are the MemoryRetainer pointers, and values 141 | are MemoryRetainerNode pointers. 142 | 143 | #### MemoryRetainerNode 144 | This class is defined in src/memory_tracker-inl.h. 145 | ```c++ 146 | class MemoryRetainerNode : public v8::EmbedderGraph::Node { 147 | public: 148 | inline MemoryRetainerNode(MemoryTracker* tracker, 149 | const MemoryRetainer* retainer) : retainer_(retainer) { 150 | v8::HandleScope handle_scope(tracker->isolate()); 151 | v8::Local obj = retainer_->WrappedObject(); 152 | if (!obj.IsEmpty()) wrapper_node_ = tracker->graph()->V8Node(obj); 153 | 154 | name_ = retainer_->MemoryInfoName(); 155 | size_ = retainer_->SelfSize(); 156 | } 157 | 158 | private: 159 | const MemoryRetainer* retainer_; 160 | Node* wrapper_node_ = nullptr; 161 | 162 | // Otherwise (retainer == nullptr), we set these fields in an ad-hoc way 163 | bool is_root_node_ = false; 164 | std::string name_; 165 | size_t size_ = 0; 166 | }; 167 | ``` 168 | 169 | #### MemoryRetainer 170 | -------------------------------------------------------------------------------- /notes/lto-ppc64le-issue.md: -------------------------------------------------------------------------------- 1 | ## Node.js PPC64LE LTO issue 2 | This issue happens on RHEL 8.5 ppc64le using gcc: 3 | ```console 4 | $ . /opt/rh/gcc-toolset-11/enable 5 | $ export CC=ccache gcc 6 | $ export CXX=ccache g++ 7 | ``` 8 | And only when Node's build is configured for Link Time Optimizations: 9 | ```console 10 | $ configure --enable-lto 11 | $ make -j $JOBS test 12 | ``` 13 | 14 | Logs from a CI [run](https://ci.nodejs.org/job/node-test-commit-linux-lto/23/nodes=rhel8-ppc64le/console): 15 | ```console 16 | 21:52:27 [ RUN ] DebugSymbolsTest.ReqWrapList 17 | 21:52:27 ../test/cctest/test_node_postmortem_metadata.cc:203: Failure 18 | 21:52:27 Expected equality of these values: 19 | 21:52:27 expected 20 | 21:52:27 Which is: 140736537072320 21 | 21:52:27 calculated 22 | 21:52:27 Which is: 1099680328560 23 | 21:52:27 [ FAILED ] DebugSymbolsTest.ReqWrapList (43 ms) 24 | ``` 25 | 26 | ### Running the test 27 | ```console 28 | $ ./out/Debug/cctest --gtest_filter=DebugSymbolsTest.ReqWrapList 29 | ``` 30 | 31 | ### Debugging the test 32 | lldb: 33 | ```console 34 | $ lldb -- ./out/Debug/cctest --gtest_filter=DebugSymbolsTest.ReqWrapList 35 | (lldb) br s -f test_node_postmortem_metadata.cc -l 171 36 | (lldb) r 37 | ``` 38 | 39 | gdb: 40 | ```console 41 | $ gdb --args ./out/Release/cctest --gtest_filter=DebugSymbolsTest.ReqWrapList 42 | (gdb) br test_node_postmortem_metadata.cc:203 43 | (gdb) r 44 | ``` 45 | 46 | ### Troubleshooting 47 | 48 | ```console 49 | (lldb) target variable nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 50 | (uintptr_t) nodedbg_offset_ListNode_ReqWrap__next___uintptr_t = 8 51 | 52 | (lldb) image lookup -s nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 53 | 1 symbols match 'nodedbg_offset_ListNode_ReqWrap__next___uintptr_t' in /home/danielbevenius/work/nodejs/node-debug/out/Debug/cctest: 54 | Address: cctest[0x000000000606ea58] (cctest.PT_LOAD[4]..bss + 29912) 55 | Summary: cctest`nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 56 | ``` 57 | 58 | ```c++ 59 | v8::Local object = obj_template->GetFunction(env.context()) 60 | .ToLocalChecked() 61 | ->NewInstance(env.context()) 62 | .ToLocalChecked(); 63 | TestReqWrap obj(*env, object); 64 | 65 | // NOTE (mmarchini): Workaround to fix failing tests on ARM64 machines with 66 | // older GCC. Should be removed once we upgrade the GCC version used on our 67 | // ARM64 CI machinies. 68 | for (auto it : *(*env)->req_wrap_queue()) (void) ⁢ 69 | 70 | auto last = tail + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t; 71 | last = *reinterpret_cast(last); 72 | 73 | auto expected = reinterpret_cast(&obj); 74 | auto calculated = 75 | last - nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue; 76 | EXPECT_EQ(expected, calculated); 77 | ``` 78 | 79 | In `src/node_postmortem_metadata.cc` we have: 80 | ```c++ 81 | extern "C" { 82 | ... 83 | uintptr_t nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue; 84 | ... 85 | } 86 | ``` 87 | So the type of this is bacially a void pointer which is initialized by the 88 | function GenDebugSymbols: 89 | ```c++ 90 | int GenDebugSymbols() { 91 | ... 92 | nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue = 93 | OffsetOf, ReqWrap>( 94 | &ReqWrap::req_wrap_queue_); 95 | ... 96 | return 1; 97 | } 98 | 99 | const int debug_symbols_generated = GenDebugSymbols(); 100 | ``` 101 | This is getting the offset of the pointer-to-member 102 | `&ReqWrap::req_wrap_queue_` which is a private field in ReqWrapBase: 103 | ```c++ 104 | class ReqWrapBase { 105 | public: 106 | explicit inline ReqWrapBase(Environment* env); 107 | 108 | virtual ~ReqWrapBase() = default; 109 | 110 | virtual void Cancel() = 0; 111 | virtual AsyncWrap* GetAsyncWrap() = 0; 112 | 113 | private: 114 | friend int GenDebugSymbols(); 115 | friend class Environment; 116 | 117 | ListNode req_wrap_queue_; 118 | }; 119 | 120 | template 121 | class ReqWrap : public AsyncWrap, public ReqWrapBase { 122 | public: 123 | ... 124 | private: 125 | friend int GenDebugSymbols(); 126 | 127 | public: 128 | typedef void (*callback_t)(); 129 | callback_t original_callback_ = nullptr; 130 | 131 | protected: 132 | // req_wrap_queue_ needs to be at a fixed offset from the start of the class 133 | // because it is used by ContainerOf to calculate the address of the embedding 134 | // ReqWrap. ContainerOf compiles down to simple, fixed pointer arithmetic. It 135 | // is also used by src/node_postmortem_metadata.cc to calculate offsets and 136 | // generate debug symbols for ReqWrap, which assumes that the position of 137 | // members in memory are predictable. sizeof(req_) depends on the type of T, 138 | // so req_wrap_queue_ would no longer be at a fixed offset if it came after 139 | // req_. For more information please refer to 140 | // `doc/contributing/node-postmortem-support.md` 141 | T req_; 142 | }; 143 | ``` 144 | Lets take a look at the value of 145 | `nodedbg_offset_ListNode_ReqWrap__next___uintptr_t` on my local machine: 146 | ```console 147 | (lldb) expr nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 148 | (uintptr_t) $11 = 8 149 | ``` 150 | So this is saying that the req_wrap_queue_ is a offset 8 from the start of the 151 | ReqWrap, which I think makes sense as the only other field is a function 152 | pointer, `callback_t` which would by 8 bytes just like any pointer: 153 | ```console 154 | (lldb) expr sizeof(void*) 155 | (unsigned long) $12 = 8 156 | ``` 157 | So we have this pointer to a member which can be used to point an actual 158 | implementation. 159 | 160 | Stepping back up a little in the testcase we have: 161 | ```c++ 162 | auto queue = reinterpret_cast((*env)->req_wrap_queue()); 163 | ``` 164 | ```console 165 | (lldb) expr env 166 | (EnvironmentTestFixture::Env) $14 = { 167 | context_ = (val_ = 0x0000000006131e00) 168 | isolate_data_ = 0x00000000060f9d30 169 | environment_ = 0x00007ffff50598f0 170 | } 171 | 172 | (lldb) expr reinterpret_cast((*env)->req_wrap_queue()) 173 | (uintptr_t) $17 = 140737304175080 174 | ``` 175 | So that is the address of the ReqWrapQueue which is a typedef: 176 | ```c++ 177 | typedef ListHead ReqWrapQueue; 178 | inline ReqWrapQueue* req_wrap_queue() { return &req_wrap_queue_; } 179 | ``` 180 | So `queue` will be a pointer to the env req_wrap_queue, so we have a pointer 181 | to this List. 182 | 183 | Next, a variable named `head` is created: 184 | ```c++ 185 | auto head = 186 | queue + 187 | nodedbg_offset_Environment_ReqWrapQueue__head___ListNode_ReqWrapQueue; 188 | ``` 189 | So here we take the pointer queue which is 140737304175080 and add to it: 190 | ```console 191 | (lldb) expr nodedbg_offset_Environment_ReqWrapQueue__head___ListNode_ReqWrapQueue 192 | (uintptr_t) $20 = 0 193 | ``` 194 | So head in this case will be 140737304175080 (unchanged and still pointing to 195 | the List, so pointing to the start/head of the list): 196 | ```console 197 | (lldb) expr head 198 | (unsigned long) $25 = 140737304175080 199 | ``` 200 | 201 | Next, we have the variable `tail`: 202 | ```c++ 203 | auto tail = head + nodedbg_offset_ListNode_ReqWrap__prev___uintptr_t; 204 | ``` 205 | ```console 206 | (gdb) p nodedbg_offset_ListNode_ReqWrap__prev___uintptr_t 207 | $12 = 0 208 | 209 | (lldb) expr tail 210 | (unsigned long) $32 = 140737304175080 211 | ``` 212 | So at this stage both head and tail are pointing to the same value. 213 | 214 | ```c++ 215 | tail = *reinterpret_cast(tail); 216 | ``` 217 | And after the reinterpret_cast: 218 | ```console 219 | lldb) expr tail 220 | (unsigned long) $33 = 140737304175080 221 | ``` 222 | 223 | Next we are taking the value of tail 224 | ```c++ 225 | auto last = tail + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t; 226 | ``` 227 | ```console 228 | lldb) expr tail 229 | (unsigned long) $33 = 140737304175080 230 | (lldb) expr nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 231 | (uintptr_t) $38 = 8 232 | 233 | (lldb) expr tail + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t 234 | (unsigned long) $39 = 140737304175088 235 | 236 | lldb) expr tail 237 | (unsigned long) $40 = 140737304175088 238 | ``` 239 | 240 | This `last` is then reassigned to point to the dereferenced 241 | ```c++ 242 | last = *reinterpret_cast(last); 243 | ``` 244 | ```console 245 | (lldb) memory read -fx -s 8 -c 1 last 246 | 0x7ffff505a1f0: 0x00007fffffffd0a0 247 | ``` 248 | And after the reinterpret_cast last will point to: 249 | ```console 250 | (lldb) memory read -fx -s 8 -c 1 last 251 | 0x7fffffffd0a0: 0x00007ffff505a1e8 252 | ``` 253 | 254 | ```c++ 255 | auto expected = reinterpret_cast(&obj); 256 | auto calculated = 257 | last - nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue; 258 | ``` 259 | `expected` is a pointer to the TestReqWrap instance created earlier, and 260 | `calculated` is taking the pointer to `last` and then recasting that, then 261 | dereferencing it which is now a pointer to the member `req_` and which we know 262 | from above is at offset 8. This value is then subtracted to get the pointer to 263 | the instance holding `req_` which is expected to be the same, that is the 264 | TestReqWrap instance we created ealier. 265 | 266 | But this is what happends ppc64le: 267 | ```console 268 | ../test/cctest/test_node_postmortem_metadata.cc:203: Failure 269 | Expected equality of these values: 270 | expected 271 | Which is: 140737488346192 272 | calculated 273 | Which is: 353276184 274 | ``` 275 | The calculated value look way off in this case. Locally this would be pointing 276 | to the same value since we are trying to peek at the last entry in the queue. 277 | The values for `queue`, `head`, `tail`, and `last` are optimized out for a 278 | Release build which also happens locally, but the test does not fail. It only 279 | seems to fail on PPC64LE. 280 | 281 | Hmm, I've added print statements: 282 | ```console 283 | queue: 0x1002fd1d8a8 284 | head: 0x1002fd1d8a8 285 | tail: 0x1002fd1d8a8 286 | tast before cast:: 0x1002fd1d8a8 287 | tast: 0x1002fd1d8a8 288 | 289 | expected:: 0x7fffce7d9d50 290 | calculated:: 0x7fffce7d9d50 291 | ``` 292 | And noticed that this allows the test to pass. So there is something that is 293 | causing these values to be incorrect when compiling with LTO. How about if we 294 | try to prevent the compiler from tampering with these values. For example by 295 | making the variables volatile: 296 | ```console 297 | diff --git a/test/cctest/test_node_postmortem_metadata.cc b/test/cctest/test_node_postmortem_metadata.cc 298 | index 4cee7db4c8..d35e71f7f8 100644 299 | --- a/test/cctest/test_node_postmortem_metadata.cc 300 | +++ b/test/cctest/test_node_postmortem_metadata.cc 301 | @@ -172,11 +172,11 @@ TEST_F(DebugSymbolsTest, ReqWrapList) { 302 | const Argv argv; 303 | Env env{handle_scope, argv}; 304 | 305 | - auto queue = reinterpret_cast((*env)->req_wrap_queue()); 306 | - auto head = 307 | + volatile uintptr_t queue = reinterpret_cast((*env)->req_wrap_queue()); 308 | + volatile uintptr_t head = 309 | queue + 310 | nodedbg_offset_Environment_ReqWrapQueue__head___ListNode_ReqWrapQueue; 311 | - auto tail = head + nodedbg_offset_ListNode_ReqWrap__prev___uintptr_t; 312 | + volatile uintptr_t tail = head + nodedbg_offset_ListNode_ReqWrap__prev___uintptr_t; 313 | tail = *reinterpret_cast(tail); 314 | 315 | auto obj_template = v8::FunctionTemplate::New(isolate_); 316 | @@ -194,11 +194,11 @@ TEST_F(DebugSymbolsTest, ReqWrapList) { 317 | // ARM64 CI machinies. 318 | for (auto it : *(*env)->req_wrap_queue()) (void) ⁢ 319 | 320 | - auto last = tail + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t; 321 | + volatile uintptr_t last = tail + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t; 322 | last = *reinterpret_cast(last); 323 | 324 | - auto expected = reinterpret_cast(&obj); 325 | - auto calculated = 326 | + volatile uintptr_t expected = reinterpret_cast(&obj); 327 | + volatile uintptr_t calculated = 328 | last - nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue; 329 | EXPECT_EQ(expected, calculated); 330 | 331 | ``` 332 | Using this patch I was able to get the test to pass. 333 | -------------------------------------------------------------------------------- /notes/snapshot.md: -------------------------------------------------------------------------------- 1 | ### Node snapshot 2 | This document contains notes about Node's usage of V8 snapshots features. 3 | 4 | ### node_snapshot executable 5 | This is an executable that is run as part of Node's build. The result of running 6 | this is a generated C++ source file that by default can be found in 7 | `out/Release/obj/gen/node_snapshot.cc`. This will then be compiled with the 8 | rest of Node at a later stage in the build process. 9 | 10 | ### node_mkshapshot walkthrough 11 | ```console 12 | $ lldb -- ./out/Debug/node_mksnapshot out/Debug/obj/gen/node_snapshot.cc 13 | (lldb) settings set target.env-vars NODE_DEBUG_NATIVE=mksnapshot 14 | (lldb) br s -n main 15 | ``` 16 | For a snapshot to be created `SnapshotBuilder::Generate` will create a new 17 | V8 runtime (Isolate, Platform etc). After the Isolate has been registered with 18 | the v8 platform a vector of external references will be collected by the 19 | following call: 20 | ```c++ 21 | const std::vector& external_references = 22 | NodeMainInstance::CollectExternalReferences(); 23 | ``` 24 | That call will land in node_main_instance.cc: 25 | ```c++ 26 | const std::vector& NodeMainInstance::CollectExternalReferences() { 27 | registry_.reset(new ExternalReferenceRegistry()); 28 | return registry_->external_references(); 29 | ``` 30 | When we have functions that are not defined in V8 itself these functions will 31 | have addresses that V8 does not know about. The function will be a symbol that 32 | needs to be resolved when V8 deserialzes a blob. 33 | 34 | `ExternalReferenceRegistry` constructor will call all the registered external 35 | references (see ExternalReferenceRegistry for details about this). These are 36 | functions that exist in Node that need to be registered as external so that 37 | the object being serialized does not store the address to the function but 38 | instead an index into this array of external references. After the snapshot has 39 | been created it will be stored on disk and the snapshot blob cannot contain the 40 | function addresses as they will most probably change when loaded into another 41 | processs. 42 | 43 | ```console 44 | (lldb) expr external_references 45 | (const std::vector >) $1 = size=209 { 46 | [0] = 25494212 47 | ``` 48 | And we can verify that this matches node::SetupHooks: 49 | ```console 50 | (lldb) expr SetupHooks 51 | (void (*)(const v8::FunctionCallbackInfo &)) $0 = 0x00000000018502c4 (node_mksnapshot`node::SetupHooks(const v8::FunctionCallbackInfo &) at async_wrap.cc:413:65) 52 | (lldb) expr reinterpret_cast(SetupHooks) 53 | (intptr_t) $1 = 25494212 54 | ``` 55 | Next, we create a new SnapshotCreator: 56 | ```c++ 57 | SnapshotCreator creator(isolate, external_references.data()); 58 | ``` 59 | `SnapshotCreator` is a class in V8 which takes a pointer to external references 60 | and is declared in v8.h. 61 | After this a `NodeMainInstance` will be created: 62 | ```c++ 63 | const std::vector& external_references = 64 | NodeMainInstance::CollectExternalReferences(); 65 | SnapshotCreator creator(isolate, external_references.data()); 66 | Environment* env; 67 | { 68 | main_instance = 69 | NodeMainInstance::Create(isolate, 70 | uv_default_loop(), 71 | per_process::v8_platform.Platform(), 72 | args, 73 | exec_args); 74 | 75 | HandleScope scope(isolate); 76 | creator.SetDefaultContext(Context::New(isolate)); 77 | isolate_data_indexes = main_instance->isolate_data()->Serialize(&creator); 78 | ``` 79 | Notice the call to `IsolateData::Serialize` (src/env.cc). This function uses 80 | macros which can be expanded using: 81 | ```console 82 | $ g++ -DNODE_WANT_INTERNALS=true -E -Ideps/uv/include -Ideps/v8/include -Isrc src/env.cc 83 | ``` 84 | 85 | ```c++ 86 | v8::Eternal alpn_buffer_private_symbol_; 87 | v8::Eternal arrow_message_private_symbol_; 88 | ... 89 | 90 | std::vector IsolateData::Serialize(SnapshotCreator* creator) { 91 | Isolate* isolate = creator->GetIsolate(); 92 | std::vector indexes; 93 | HandleScope handle_scope(isolate); 94 | indexes.push_back(creator->AddData(alpn_buffer_private_symbol_.Get(isolate))); 95 | indexes.push_back(creator->AddData(arrow_message_private_symbol_.Get(isolate))); 96 | indexes.push_back(creator->AddData(contextify_context_private_symbol_.Get(isolate))); 97 | indexes.push_back(creator->AddData(contextify_global_private_symbol_.Get(isolate))); 98 | indexes.push_back(creator->AddData(decorated_private_symbol_.Get(isolate))); 99 | ... 100 | 101 | for (size_t i = 0; i < AsyncWrap::PROVIDERS_LENGTH; i++) 102 | indexes.push_back(creator->AddData(async_wrap_provider(i))); 103 | 104 | return indexes; 105 | } 106 | ``` 107 | Notice that we are calling `AddData` on the SnapshotCreator which allows for 108 | attaching arbitary V8:Data to the `isolate` snapshot. This data can later be 109 | retrieved using `Isolate::GetDataFromSnapshotOnce` and passing in the index 110 | (size_t) returned from `AddData`. 111 | 112 | After this we are back `SnapshotBuilder::Generate` and will create a new Context 113 | and enter a ContextScope. A new Environent instance will be created: 114 | ```c++ 115 | env = new Environment(main_instance->isolate_data(), 116 | context, 117 | args, 118 | exec_args, 119 | nullptr, 120 | node::EnvironmentFlags::kDefaultFlags, 121 | {}); 122 | env->RunBootstrapping().ToLocalChecked(); 123 | if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { 124 | env->PrintAllBaseObjects(); 125 | printf("Environment = %p\n", env); 126 | } 127 | env_info = env->Serialize(&creator); 128 | size_t index = creator.AddContext(context, {SerializeNodeContextInternalFields, env}); 129 | ``` 130 | Environment's constructor calls Environment::InitializeMainContext and at this 131 | stage we are passing in `nullptr` for the forth argument which is the 132 | EnvSerializeInfo pointer: 133 | ```c++ 134 | void Environment::InitializeMainContext(Local context, 135 | const EnvSerializeInfo* env_info) { 136 | ... 137 | if (env_info != nullptr) { 138 | DeserializeProperties(env_info); 139 | } else { 140 | CreateProperties(); 141 | } 142 | ... 143 | ``` 144 | The properties here are the properties that are available to all scripts, like 145 | the `primordials`, and the `process` object. We will take a look at 146 | `DeserializeProperties` when we startup node with the snapshot blob created by 147 | the current process (remember that we are currently executing node_mksnapshot 148 | to produce that blob). 149 | 150 | After the bootstrapping has run, notice the call to `env->Serialize` which can 151 | be found in env.cc. 152 | ```c++ 153 | EnvSerializeInfo Environment::Serialize(SnapshotCreator* creator) { 154 | EnvSerializeInfo info; 155 | Local ctx = context(); 156 | 157 | info.native_modules = std::vector( 158 | native_modules_without_cache.begin(), native_modules_without_cache.end()); 159 | 160 | info.async_hooks = async_hooks_.Serialize(ctx, creator); 161 | info.immediate_info = immediate_info_.Serialize(ctx, creator); 162 | info.tick_info = tick_info_.Serialize(ctx, creator); 163 | info.performance_state = performance_state_->Serialize(ctx, creator); 164 | info.stream_base_state = stream_base_state_.Serialize(ctx, creator); 165 | info.should_abort_on_uncaught_toggle = 166 | should_abort_on_uncaught_toggle_.Serialize(ctx, creator); 167 | } 168 | ``` 169 | First the non-cachable native modules are gathered and then Serialize is called 170 | on different object. Notice that they all take a context and the SnapshotCreator. 171 | The functions will be adding data to the context, for example: 172 | ```c++ 173 | return creator->AddData(context, GetJSArray()); 174 | ``` 175 | After that there is a macro that will 176 | ```c++ 177 | size_t id = 0; 178 | #define V(PropertyName, TypeName) 179 | do { 180 | Local field = PropertyName(); 181 | if (!field.IsEmpty()) { 182 | size_t index = creator->AddData(field); 183 | info.persistent_templates.push_back({#PropertyName, id, index}); 184 | } 185 | id++; 186 | } while (0); 187 | ENVIRONMENT_STRONG_PERSISTENT_TEMPLATES(V) 188 | #undef V 189 | ``` 190 | This expands to (just showing one example below and not all): 191 | ```c++ 192 | do { 193 | Local field = async_wrap_ctor_template(); 194 | if (!field.IsEmpty()) { 195 | size_t index = creator->AddData(field); 196 | info.persistent_templates.push_back({"async_wrap_ctor_template", id, index}); 197 | } 198 | id++; 199 | } while (0); 200 | ``` 201 | Notice that `info.persistent_templates. is declared as: 202 | ```c++ 203 | std::vector persistent_templates; 204 | ``` 205 | And the PropInfo struct look like this: 206 | ```c++ 207 | struct PropInfo { 208 | std::string name; // name for debugging 209 | size_t id; // In the list - in case there are any empty entires 210 | SnapshotIndex index; // In the snapshot 211 | }; 212 | typedef size_t SnapshotIndex; 213 | ``` 214 | So we are adding new PropInfo instances with the name given by the PropertyName, 215 | the `id` is just a counter that starts from 0, `index` is the value returned from 216 | `AddData` which is the index used to retrieve the value from the snapshot data 217 | later when calling isolate->GetDataFromSnapshotOnce(index). Note that 218 | there is no context passed to `AddData` which means we are adding this to the 219 | isolate. This explains the values that can be found in 220 | out/Debug/obj/gen/node_snapshot.cc: 221 | ```c++ 222 | // -- persistent_templates begins -- 223 | { 224 | // name id index 225 | { "async_wrap_ctor_template", 0, 337 }, 226 | { "base_object_ctor_template", 2, 338 }, 227 | { "binding_data_ctor_template", 3, 339 }, 228 | { "handle_wrap_ctor_template", 11, 340 }, 229 | { "i18n_converter_template", 16, 341 }, 230 | { "message_port_constructor_template", 18, 342 }, 231 | { "promise_wrap_template", 21, 343 }, 232 | { "worker_heap_snapshot_taker_template", 31, 344 }, 233 | }, 234 | // persistent_templates ends -- 235 | ``` 236 | 237 | After all of the templates have been added, there is another macro that adds 238 | properties: 239 | ```c++ 240 | id = 0; 241 | #define V(PropertyName, TypeName) 242 | do { 243 | Local field = PropertyName(); 244 | if (!field.IsEmpty()) { 245 | size_t index = creator->AddData(ctx, field); 246 | info.persistent_values.push_back({#PropertyName, id, index}); 247 | } 248 | id++; 249 | } while (0); 250 | ENVIRONMENT_STRONG_PERSISTENT_VALUES(V) 251 | #undef V 252 | ``` 253 | And expanded this will become (again only showing one): 254 | ```c++ 255 | id = 0; 256 | do { 257 | Local field = async_hooks_after_function(); 258 | if (!field.IsEmpty()) { 259 | size_t index = creator->AddData(ctx, field); 260 | info.persistent_values.push_back({"async_hooks_after_function", id, index}); 261 | } 262 | id++; 263 | } while (0); 264 | ``` 265 | Notice that this time the Context is passed to `AddData` so we are adding these 266 | to the context. 267 | 268 | Finally the context is added before the `EnvSerializeInfo` is returned: 269 | ```c++ 270 | info.context = creator->AddData(ctx, context()); 271 | ``` 272 | After this we will be back in `SnapshotBuilder::Generate`: 273 | ```c++ 274 | size_t index = creator.AddContext( 275 | context, {SerializeNodeContextInternalFields, env}); 276 | ``` 277 | This is adding the context created and passing in a new 278 | `SerializeInternalFieldsCallback` with the callback being 279 | SerializeNodeContextInternalFields and the arguments to be passed to that 280 | callback the pointer to the Environment: 281 | ```c++ 282 | struct SerializeInternalFieldsCallback { 283 | typedef StartupData (*CallbackFunction)(Local holder, int index, 284 | void* data); 285 | SerializeInternalFieldsCallback(CallbackFunction function = nullptr, 286 | void* data_arg = nullptr) 287 | : callback(function), data(data_arg) {} 288 | CallbackFunction callback; 289 | void* data; 290 | }; 291 | ``` 292 | In Node there are non-V8 objects attached to V8 objects by using embedder/internal 293 | fields. V8 does not know how to handle this, so this callback is a way to enable 294 | Node to extract the object from the `holder`, serialize the object into the an 295 | instance of the returned `StartupData` which will then be added as part of the 296 | blob. Later when deserializeing there will be a callback to recreate and 297 | instance of this type and then populate it with the data from the StartupData 298 | that was stored.. 299 | 300 | A standalone example can be found 301 | [here](https://github.com/danbev/learning-v8/blob/cb07dfd3aac4d76bbd3a14bdb1b268fdc4fd6587/test/snapshot_test.cc#L289-L306) which may help to clarify this. 302 | 303 | So far we have collected addresses to functions that are external to V8 and 304 | added them all to a vector. These will then be passed to the SnapshotCreator 305 | constructor making them available when it serialized the Isolate/Context. 306 | 307 | ### ExternalReferenceRegistry 308 | To see what the preprocessor generates for node_external_reference.cc the 309 | following command can be used: 310 | ```console 311 | $ g++ -DNODE_WANT_INTERNALS=true -I./src -I./deps/v8/include -I./deps/uv/include -E src/node_external_reference.cc 312 | ``` 313 | ```c++ 314 | ExternalReferenceRegistry::ExternalReferenceRegistry() { 315 | _register_external_reference_async_wrap(this); 316 | _register_external_reference_binding(this); 317 | _register_external_reference_buffer(this); 318 | _register_external_reference_credentials(this); 319 | _register_external_reference_env_var(this); 320 | _register_external_reference_errors(this); 321 | _register_external_reference_handle_wrap(this); 322 | _register_external_reference_messaging(this);x 323 | _register_external_reference_native_module(this); 324 | _register_external_reference_process_methods(this); 325 | _register_external_reference_process_object(this); 326 | _register_external_reference_task_queue(this); 327 | _register_external_reference_url(this); 328 | _register_external_reference_util(this); 329 | _register_external_reference_string_decoder(this); 330 | _register_external_reference_trace_events(this); 331 | _register_external_reference_timers(this); 332 | _register_external_reference_types(this); 333 | _register_external_reference_worker(this); 334 | } 335 | ``` 336 | And to see these functions use the following command: 337 | ```console 338 | $ g++ -DNODE_WANT_INTERNALS=true -I./src -I./deps/v8/include -I./deps/uv/include -E src/node_external_reference.h 339 | ``` 340 | Lets take a look at one of these, `_register_external_reference_async_wrap`: 341 | ```c++ 342 | void _register_external_reference_async_wrap(node::ExternalReferenceRegistry* registry); 343 | ``` 344 | So that is the declaration, and to see the implementation we have to look in 345 | src/async_wrap.cc: 346 | ```c++ 347 | NODE_MODULE_EXTERNAL_REFERENCE(async_wrap, 348 | node::AsyncWrap::RegisterExternalReferences) 349 | ``` 350 | And to expand that macro: 351 | ```console 352 | $ g++ -DNODE_WANT_INTERNALS=true -I./src -I./deps/v8/include -I./deps/uv/include -E src/async_wrap.cc 353 | ``` 354 | Which produces: 355 | ```C++ 356 | void _register_external_reference_async_wrap(node::ExternalReferenceRegistry* registry) { 357 | node::AsyncWrap::RegisterExternalReferences(registry); 358 | } 359 | 360 | void AsyncWrap::RegisterExternalReferences(ExternalReferenceRegistry* registry) { 361 | registry->Register(SetupHooks); 362 | registry->Register(SetCallbackTrampoline); 363 | registry->Register(PushAsyncContext); 364 | registry->Register(PopAsyncContext); 365 | registry->Register(ExecutionAsyncResource); 366 | registry->Register(ClearAsyncIdStack); 367 | registry->Register(QueueDestroyAsyncId); 368 | registry->Register(EnablePromiseHook); 369 | registry->Register(DisablePromiseHook); 370 | registry->Register(RegisterDestroyHook); 371 | registry->Register(AsyncWrap::GetAsyncId); 372 | registry->Register(AsyncWrap::AsyncReset); 373 | registry->Register(AsyncWrap::GetProviderType); 374 | registry->Register(PromiseWrap::GetAsyncId); 375 | registry->Register(PromiseWrap::GetTriggerAsyncId); 376 | } 377 | ``` 378 | Now, in `node_external_reference.h` we have a number of types that can be 379 | registered, for example `SetupHooks` is of type v8::FunctionCallback (it's a 380 | function pointer) so this is what will be called: 381 | ```c++ 382 | void Register(v8::FunctionCallback addr) { 383 | RegisterT(addr); 384 | } 385 | ``` 386 | And `RegisterT` is a private function in ExternalReferenceRegistry: 387 | ```c++ 388 | void RegisterT(T* address) { 389 | external_references_.push_back(reinterpret_cast(address)); 390 | } 391 | std::vector external_references_; 392 | ``` 393 | And as we can see `external_references_` is a vector of addresses. So this how 394 | the addresses to these functions are collected. 395 | 396 | ### EnvSerializeInfo 397 | This is a struct declared in env.h and looks like this: 398 | ```c++ 399 | struct EnvSerializeInfo { 400 | std::vector native_modules; 401 | AsyncHooks::SerializeInfo async_hooks; 402 | TickInfo::SerializeInfo tick_info; 403 | ImmediateInfo::SerializeInfo immediate_info; 404 | performance::PerformanceState::SerializeInfo performance_state; 405 | AliasedBufferInfo stream_base_state; 406 | AliasedBufferInfo should_abort_on_uncaught_toggle; 407 | 408 | std::vector persistent_templates; 409 | std::vector persistent_values; 410 | 411 | SnapshotIndex context; 412 | friend std::ostream& operator<<(std::ostream& o, const EnvSerializeInfo& i); 413 | }; 414 | ``` 415 | So first we have the native modules (which are the javascript modules under 416 | lib. 417 | Next, we have `AsyncHooks::SerializeInfo` which is a struct in AsyncHooks: 418 | ```c++ 419 | struct SerializeInfo { 420 | AliasedBufferInfo async_ids_stack; 421 | AliasedBufferInfo fields; 422 | AliasedBufferInfo async_id_fields; 423 | SnapshotIndex js_execution_async_resources; 424 | std::vector native_execution_async_resources; 425 | }; 426 | ``` 427 | `src/aliased_buffer.h` has a typedef `AliasedBufferInfo`: 428 | ```c++ 429 | typedef size_t AliasedBufferInfo; 430 | ``` 431 | So async_ids_stack, fields, and async_id_fields are just unsigned integers. 432 | Likewise `SnapshotIndex` is also a typedef: 433 | ```c++ 434 | typedef size_t SnapshotIndex; 435 | ``` 436 | 437 | ### Snapshot usage during startup 438 | When Node.js starts it will check if the is a startup blob (StartupData) 439 | available when calling `NodeMainInstance::GetEmbeddedSnapshotBlob` 440 | (in src/node.cc): 441 | ```c++ 442 | int Start(int argc, char** argv) { 443 | ... 444 | v8::StartupData* blob = NodeMainInstance::GetEmbeddedSnapshotBlob(); 445 | if (blob != nullptr) { 446 | params.snapshot_blob = blob; 447 | indexes = NodeMainInstance::GetIsolateDataIndexes(); 448 | env_info = NodeMainInstance::GetEnvSerializeInfo(); 449 | } 450 | ``` 451 | These functions are the ones that were generated by node_mksnapshot and 452 | can be found in out/{BUILD_TYPE}/obj/gen/node_snapshot.cc: 453 | ```c++ 454 | static const int blob_size = 539943; 455 | static v8::StartupData blob = { blob_data, blob_size }; 456 | v8::StartupData* NodeMainInstance::GetEmbeddedSnapshotBlob() { 457 | return &blob; 458 | } 459 | 460 | const EnvSerializeInfo* NodeMainInstance::GetEnvSerializeInfo() { 461 | return &env_info; 462 | } 463 | ``` 464 | Notice that the `Isolate::CreateParams` field `snapshot_blob` is set, and 465 | after this we retrieve the indexes from NodeMainInstance. Now, this will be 466 | from the generated node_snapshot.cc (in out/Debug/obj/gen/node_snapshot.cc) 467 | which was generated by node_mksnapshot. The same goes for env_info. 468 | 469 | After this a new NodeMainInstance will be created: 470 | ```c++ 471 | NodeMainInstance main_instance(¶ms, 472 | uv_default_loop(), 473 | per_process::v8_platform.Platform(), 474 | result.args, 475 | result.exec_args, 476 | indexes); 477 | ``` 478 | This brings us into `src/node_main_instance.cc` and the constructor that takes 479 | indexes: 480 | ```c++ 481 | if (deserialize_mode_) { 482 | const std::vector& external_references = CollectExternalReferences(); 483 | params->external_references = external_references.data(); 484 | } 485 | ``` 486 | See [ExternalReferenceRegistry](#externalreferenceregistry) for details about 487 | ExternalReferenceRegistry. Notice that we are setting the pointer on 488 | `params` which will be passed to V8 later. 489 | 490 | After this is done, a new Isolate will be allocated, registered with the 491 | platform, and initialized. 492 | 493 | Next we have the following call: 494 | ```c++ 495 | SetIsolateCreateParamsForNode(params); 496 | Isolate::Initialize(isolate_, *params); 497 | ``` 498 | The last call will set the blob on the isolate (in src/api/api.cc): 499 | ```c++ 500 | if (params.snapshot_blob != nullptr) { 501 | i_isolate->set_snapshot_blob(params.snapshot_blob); 502 | } else { 503 | i_isolate->set_snapshot_blob(i::Snapshot::DefaultSnapshotBlob()); 504 | } 505 | ``` 506 | And a little further down we have: 507 | ```c++ 508 | i_isolate->set_api_external_references(params.external_references); 509 | ``` 510 | And this is where we set the list of external references/addresses on the 511 | isolate. 512 | Next, `Isolate::Initialize` will call `StartupDeserializer::DeserializeInto`: 513 | ```c++ 514 | void StartupDeserializer::DeserializeInto(Isolate* isolate) { 515 | ... 516 | { 517 | isolate->heap()->IterateRoots( 518 | this, 519 | base::EnumSet{SkipRoot::kUnserializable, SkipRoot::kWeak}); 520 | } 521 | ``` 522 | ```c++ 523 | void Heap::IterateRoots(RootVisitor* v, base::EnumSet options) { 524 | v->VisitRootPointers(Root::kStrongRootList, nullptr, 525 | roots_table().strong_roots_begin(), 526 | roots_table().strong_roots_end()); 527 | ``` 528 | Now, `v` is of type `v8::internal::StartupDeserializer` which extends 529 | `Deserializer` which implements `VisitRootPointers`: 530 | ```c++ 531 | void Deserializer::VisitRootPointers(Root root, const char* description, 532 | FullObjectSlot start, FullObjectSlot end) { 533 | ReadData(FullMaybeObjectSlot(start), FullMaybeObjectSlot(end), kNullAddress); 534 | } 535 | ``` 536 | And `ReadData` looks like this (also in deserializer.cc): 537 | ```c++ 538 | template 539 | void Deserializer::ReadData(TSlot current, TSlot limit, 540 | Address current_object_address) { 541 | while (current < limit) { 542 | byte data = source_.Get(); 543 | current = ReadSingleBytecodeData(data, current, current_object_address); 544 | } 545 | CHECK_EQ(limit, current); 546 | } 547 | ``` 548 | `source_` is what reads the snapshot (src/snapshot/snapshot-source-sink.h): 549 | ```c++ 550 | class SnapshotByteSource final { 551 | public: 552 | byte Get() { 553 | DCHECK(position_ < length_); 554 | return data_[position_++]; 555 | } 556 | private: 557 | const byte* data_; 558 | int length_; 559 | int position_; 560 | }; 561 | ``` 562 | So we will get the next byte from the snapshot and then call 563 | `ReadSingleBytecodeData`: 564 | ```c++ 565 | template 566 | TSlot Deserializer::ReadSingleBytecodeData(byte data, TSlot current, 567 | Address current_object_address) { 568 | switch (data) { 569 | ... 570 | case kSandboxedApiReference: 571 | case kApiReference: { 572 | uint32_t reference_id = static_cast(source_.GetInt()); 573 | Address address; 574 | if (isolate()->api_external_references()) { 575 | DCHECK_WITH_MSG(reference_id < num_api_references_, 576 | "too few external references provided through the API"); 577 | address = static_cast
( 578 | isolate()->api_external_references()[reference_id]); 579 | } else { 580 | address = reinterpret_cast
(NoExternalReferencesCallback); 581 | } 582 | if (V8_HEAP_SANDBOX_BOOL && data == kSandboxedApiReference) { 583 | return WriteExternalPointer(current, address); 584 | } else { 585 | DCHECK(!V8_HEAP_SANDBOX_BOOL); 586 | return WriteAddress(current, address); 587 | } 588 | } 589 | ... 590 | } 591 | ``` 592 | `kApiReference` is an element of an enum defined in 593 | src/snapshot/serializer-deserializer.h: 594 | ```c++ 595 | enum Bytecode : byte { 596 | // 0x00..0x03 Allocate new object, in specified space. 597 | kNewObject = 0x00, 598 | // Reference to previously allocated object. 599 | kBackref = 0x04, 600 | ... 601 | // Used to encode external references provided through the API. 602 | kApiReference, 603 | ... 604 | 605 | ``` 606 | Just note that this case also allows kSandboxedApiReference to fall through so 607 | you might expect `data` to match `kApiReference` but that might not always be 608 | the case: 609 | ```console 610 | (lldb) expr -f oct -- this->Bytecode::kApiReference 611 | (v8::internal::SerializerDeserializer::Bytecode) $12 = 037 612 | (lldb) expr --format bin -- data 613 | (v8::internal::byte) $26 = 0b01010000 614 | (lldb) expr --format bin -- this->Bytecode::kSandboxedApiReference 615 | (v8::internal::SerializerDeserializer::Bytecode) $27 = 0b00100001 616 | ``` 617 | Lets look at the content of this case and remove the debug checks: 618 | ```c++ 619 | case kApiReference: { 620 | uint32_t reference_id = static_cast(source_.GetInt()); 621 | Address address; 622 | if (isolate()->api_external_references()) { 623 | address = static_cast
(isolate()->api_external_references()[reference_id]); 624 | } else { 625 | address = reinterpret_cast
(NoExternalReferencesCallback); 626 | } 627 | if (V8_HEAP_SANDBOX_BOOL && data == kSandboxedApiReference) { 628 | return WriteExternalPointer(current, address); 629 | } else { 630 | return WriteAddress(current, address); 631 | } 632 | } 633 | ``` 634 | Notice the first thing that happens is that an int is read from the `source_` 635 | which is the reference id that should be used to lookup the reference in the 636 | list of external references. This address will then be written to the `current` 637 | position. So that is how external references are handled. 638 | 639 | Next, a new IsolateData instance will be created: 640 | ```c++ 641 | isolate_data_ = std::make_unique(isolate_, 642 | event_loop, 643 | platform, 644 | array_buffer_allocator_.get(), 645 | per_isolate_data_indexes); 646 | ``` 647 | IsolateData is of type `node::IsolateData` declared in src/env.h (not to be 648 | confused with v8::internal::IsolateData). The members that are of interest to 649 | us are the following: 650 | ```c++ 651 | class IsolateData : public MemoryRetainer { 652 | public: 653 | IsolateData(v8::Isolate* isolate, 654 | uv_loop_t* event_loop, 655 | MultiIsolatePlatform* platform = nullptr, 656 | ArrayBufferAllocator* node_allocator = nullptr, 657 | const std::vector* indexes = nullptr); 658 | 659 | std::vector Serialize(v8::SnapshotCreator* creator); 660 | ``` 661 | Notice that there is no field that stores `indexes`. If we look the definition 662 | of the constructor above we see how this is used (in src/env.cc): 663 | ```c++ 664 | IsolateData::IsolateData(Isolate* isolate, 665 | uv_loop_t* event_loop, 666 | MultiIsolatePlatform* platform, 667 | ArrayBufferAllocator* node_allocator, 668 | const std::vector* indexes) 669 | : isolate_(isolate), event_loop_(event_loop), 670 | node_allocator_(node_allocator == nullptr ? nullptr : node_allocator->GetImpl()), 671 | platform_(platform) { 672 | options_.reset(new PerIsolateOptions(*(per_process::cli_options->per_isolate))); 673 | 674 | if (indexes == nullptr) { 675 | CreateProperties(); 676 | } else { 677 | DeserializeProperties(indexes); 678 | } 679 | ``` 680 | In our case `indexes` will not be null so we will be calling 681 | `DeserializeProperties` which can be found in src/env.cc. This function uses 682 | a macro and below is just showing a few of them expanded: 683 | ```c++ 684 | void IsolateData::DeserializeProperties(const std::vector* indexes) { 685 | size_t i = 0; 686 | HandleScope handle_scope(isolate_); 687 | do { 688 | MaybeLocal field = isolate_->GetDataFromSnapshotOnce((*indexes)[i++]); 689 | if (field.IsEmpty()) { 690 | fprintf(stderr, "Failed to deserialize " "alpn_buffer_private_symbol" "\n"); 691 | } 692 | alpn_buffer_private_symbol_.Set(isolate_, field.ToLocalChecked()); 693 | } while (0); 694 | do { 695 | MaybeLocal field = isolate_->GetDataFromSnapshotOnce((*indexes)[i++]); 696 | if (field.IsEmpty()) { 697 | fprintf(stderr, "Failed to deserialize " "arrow_message_private_symbol" "\n"); 698 | } 699 | arrow_message_private_symbol_.Set(isolate_, field.ToLocalChecked()); 700 | } while (0); 701 | ... 702 | } 703 | ``` 704 | What is happening here is that we are extracting data from the snapshot, 705 | specifying the indexes that were returned when `AddData` was called, and 706 | then populating External's. These are defined using macros in IsolateData: 707 | ```c++ 708 | #define VP(PropertyName, StringValue) V(v8::Private, PropertyName) 709 | #define V(TypeName, PropertyName) \ 710 | inline v8::Local PropertyName() const; 711 | PER_ISOLATE_PRIVATE_SYMBOL_PROPERTIES(VP) 712 | ``` 713 | So for example arrow_message_private_symbol_ would be defined as: 714 | ```c++ 715 | v8::Eternal arrow_message_private_symbol_; 716 | ``` 717 | Back in node.cc and `node::Start` after returning from NodeMainInstance constructor 718 | we have: 719 | ```c++ 720 | result.exit_code = main_instance.Run(env_info); 721 | ``` 722 | NodeMainInstance::Run will create a new Environment: 723 | ```c++ 724 | int NodeMainInstance::Run(const EnvSerializeInfo* env_info) { 725 | ... 726 | DeleteFnPtr env = CreateMainEnvironment(&exit_code, env_info); 727 | } 728 | ``` 729 | And in CreateMainEnvironment we have the following: 730 | ```c++ 731 | if (deserialize_mode_) { 732 | env.reset(new Environment(isolate_data_.get(), 733 | isolate_, 734 | args_, 735 | exec_args_, 736 | env_info, 737 | EnvironmentFlags::kDefaultFlags, 738 | {})); 739 | context = Context::FromSnapshot(isolate_, 740 | kNodeContextIndex, 741 | {DeserializeNodeInternalFields, env.get()}) 742 | .ToLocalChecked(); 743 | 744 | InitializeContextRuntime(context); 745 | SetIsolateErrorHandlers(isolate_, {}); 746 | } 747 | ... 748 | env->InitializeMainContext(context, env_info); 749 | ``` 750 | This time we are passing in a non-null `env_info` which was not the case when 751 | we looked at node_mksnapshot. 752 | Next the context will be created from the snapshot and then `InitializeMainContext` 753 | will be called. 754 | ```c++ 755 | void Environment::InitializeMainContext(Local context, 756 | const EnvSerializeInfo* env_info) { 757 | ... 758 | if (env_info != nullptr) { 759 | DeserializeProperties(env_info); 760 | } else { 761 | CreateProperties(); 762 | } 763 | ... 764 | ``` 765 | In this case `env_info` will be non-null so `DeserializeProperties` will be called 766 | which will read data from the isolate snapshot and the context snapshot to 767 | populate the persistent tempalates and values. 768 | ```c++ 769 | void Environment::DeserializeProperties(const EnvSerializeInfo* info) { 770 | Local ctx = context(); 771 | 772 | async_hooks_.Deserialize(ctx); 773 | immediate_info_.Deserialize(ctx); 774 | tick_info_.Deserialize(ctx); 775 | performance_state_->Deserialize(ctx); 776 | stream_base_state_.Deserialize(ctx); 777 | should_abort_on_uncaught_toggle_.Deserialize(ctx); 778 | 779 | ... 780 | MaybeLocal maybe_ctx_from_snapshot = 781 | ctx->GetDataFromSnapshotOnce(info->context); 782 | Local ctx_from_snapshot; 783 | if (!maybe_ctx_from_snapshot.ToLocal(&ctx_from_snapshot)) { 784 | fprintf(stderr, 785 | "Failed to deserialize context back reference from the snapshot\n"); 786 | } 787 | CHECK_EQ(ctx_from_snapshot, ctx); 788 | ``` 789 | Lets take a look at one of these, `async_hooks_.Deserialize` and focus on 790 | `async_hooks_.async_ids_stack. Prior to calling the Derialize call this field 791 | contains the following data: 792 | ```console 793 | (node::AliasedBufferBase) $5 = { 794 | isolate_ = 0x0000000005db88e0 795 | count_ = 32 796 | byte_offset_ = 0 797 | buffer_ = 0x0000000000000000 798 | js_array_ = { 799 | v8::PersistentBase = (val_ = 0x0000000000000000) 800 | } 801 | index_ = 0x0000000005d415b8 802 | } 803 | ``` 804 | Notice that the `buffer_` and `js_array_` fields are null. If we step-into 805 | async_hooks_Deserialize we have: 806 | ```c++ 807 | void AsyncHooks::Deserialize(Local context) { 808 | async_ids_stack_.Deserialize(context); 809 | ... 810 | ``` 811 | And stepping into again will land us in aliased_buffer.h: 812 | ```c++ 813 | inline void Deserialize(v8::Local context) { 814 | v8::Local arr = context->GetDataFromSnapshotOnce(*info_).ToLocalChecked(); 815 | uint8_t* raw = static_cast(arr->Buffer()->GetBackingStore()->Data()); 816 | buffer_ = reinterpret_cast(raw + byte_offset_); 817 | js_array_.Reset(isolate_, arr); 818 | info_ = nullptr; 819 | } 820 | ``` 821 | The way I've used GetDataFromSnapshotOnce was with a size_t type index to that 822 | was retured when calling SnapshotCreator::AddData. 823 | Notice that `info_` is of type `AliasedBufferInfo` which is a typedef: 824 | ```c++ 825 | typedef size_t AliasedBufferInfo; 826 | const AliasedBufferInfo* info_ = nullptr; 827 | ``` 828 | This was a little surprising as I was expecting something like index. I've 829 | created a PR with a suggestion to change this to AliasedBufferIndex and see if 830 | other agree or not. 831 | 832 | So, we are using the "index" to retreive the data from the context snapshot, 833 | then getting the BackingStore for the array and setting the `buffer_` field to 834 | that value. Finally the js_array is reset to the array read from the snapshot: 835 | ```console 836 | (node::AliasedBufferBase) $6 = { 837 | isolate_ = 0x0000000005db88e0 838 | count_ = 32 839 | byte_offset_ = 0 840 | buffer_ = 0x0000000005e414f0 841 | js_array_ = { 842 | v8::PersistentBase = (val_ = 0x0000000005e44060) 843 | } 844 | index_ = 0x0000000005d415b8 845 | } 846 | ``` 847 | 848 | ### BaseObject data 849 | This section is going to look at the a work by Joyee which is about being 850 | able to snapshot BaseObject data. 851 | 852 | Notice that the following has been added to `lib/internal/bootstrap/node.js`: 853 | ```javascript 854 | internalBinding('fs'); 855 | ``` 856 | This will casue the native module `fs` to be initialized using 857 | `GetInternalBindings` in `node_binding.cc`. This function calls `InitModule` 858 | which will call the initalizlier functions specified in `node_file.cc`: 859 | ```c++ 860 | void Initialize(Local target, 861 | Local unused, 862 | Local context, 863 | void* priv) { 864 | ... 865 | Environment* env = Environment::GetCurrent(context); 866 | Isolate* isolate = env->isolate(); 867 | BindingData* const binding_data = 868 | env->AddBindingData(context, target); 869 | ``` 870 | What is `BindingData`? 871 | This is a class the extends `BaseObject` and is declared in `node_file.h`: 872 | ```c++ 873 | class BindingData : public BaseObject { 874 | public: 875 | AliasedFloat64Array stats_field_array; 876 | AliasedBigUint64Array stats_field_bigint_array; 877 | ``` 878 | So it looks like these two arrays that are members of BindingData hold file 879 | status stats (as in data like [uv_stat_t](http://docs.libuv.org/en/v1.x/fs.html#c.uv_stat_t)). 880 | 881 | And `AddBindingData` can be found in env-inl.h: 882 | ```c++ 883 | template 884 | inline T* Environment::AddBindingData( 885 | v8::Local context, 886 | v8::Local target) { 887 | DCHECK_EQ(GetCurrent(context), this); 888 | BaseObjectPtr item = MakeDetachedBaseObject(this, target); 889 | BindingDataStore* map = static_cast( 890 | context->GetAlignedPointerFromEmbedderData( 891 | ContextEmbedderIndex::kBindingListIndex)); 892 | auto result = map->emplace(T::binding_data_name, item); 893 | return item.get(); 894 | ``` 895 | AddBindingData will create a new BindingData instance and in the proceses call 896 | BaseObject's constructor which will add an entry in the `cleanup_hooks_` map. 897 | This is how/why we have an entry when we call `Environment::Serialize` later 898 | (which we did not previously). 899 | 900 | Next, we get an object from the context using the 901 | ContextEmbedderIndex::kBindingListIndex and note that `BindingDataStore` is 902 | just an unordered_map: 903 | ```c++ 904 | typedef std::unordered_map< 905 | FastStringKey, 906 | BaseObjectPtr, 907 | FastStringKey::Hash> BindingDataStore; 908 | ``` 909 | 910 | If we take a look at tools/snapshot/snapshot_builder.cc we see that 911 | SerializeNodeContextInternalFields has been updated to contain the following 912 | macro: 913 | ```c++ 914 | static v8::StartupData SerializeNodeContextInternalFields(Local holder, 915 | int index, 916 | void* env) { 917 | ... 918 | BaseObject* obj = static_cast(ptr); 919 | switch (obj->type()) { 920 | #define V(TypeName, NativeType) \ 921 | case InternalFieldType::k##TypeName: { \ 922 | NativeType* ptr = static_cast(obj); \ 923 | InternalFieldInfo* info = ptr->Serialize(); \ 924 | per_process::Debug(DebugCategory::MKSNAPSHOT, \ 925 | "Serializing " #NativeType "at %p, length=%d\n", \ 926 | ptr, \ 927 | static_cast(info->length)); \ 928 | return StartupData{reinterpret_cast(info), \ 929 | static_cast(info->length)}; \ 930 | } 931 | 932 | INTERNAL_FIELD_TYPES(V) 933 | #undef V 934 | default: { UNREACHABLE(); } 935 | ``` 936 | The first thing to note is that every BaseObject has a InternalFieldType 937 | associated with it. And we can find the types in INTERNAL_FIELD_TYPES: 938 | ```c++ 939 | #define INTERNAL_FIELD_TYPES(V) \ 940 | V(FSBindingData, fs::BindingData) 941 | #undef V 942 | ``` 943 | 944 | We can expand this macro using: 945 | ```console 946 | $ g++ -DNODE_WANT_INTERNALS=true -E -Ideps/uv/include -Ideps/v8/include -Isrc tools/snapshot/snapshot_builder.cc 947 | ``` 948 | ```c++ 949 | switch (obj->type()) { 950 | case InternalFieldType::kFSBindingData: { 951 | fs::BindingData* ptr = static_cast(obj); 952 | InternalFieldInfo* info = ptr->Serialize(); 953 | return StartupData{reinterpret_cast(info), static_cast(info->length)}; } 954 | ``` 955 | Note that BindingData::Serialize() is defined in node_file.cc and that is 956 | should not be confused with 957 | BindingData::Serialize(Local context, v8::SnapshotCreator* creator): 958 | ```c++ 959 | InternalFieldInfo* BindingData::Serialize() { 960 | InternalFieldInfo* info = InternalFieldInfo::New(type()); 961 | return info; 962 | } 963 | ``` 964 | InternalFieldInfo is a struct with two members, an InternalFieldType and 965 | a size_t length. 966 | 967 | ```c++ 968 | switch (obj->type()) { 969 | case InternalFieldType::kFSBindingData: { 970 | fs::BindingData* ptr = static_cast(obj); 971 | InternalFieldInfo* info = ptr->Serialize(); 972 | return StartupData{reinterpret_cast(info), static_cast(info->length)}; 973 | } 974 | ``` 975 | So this is casting to the concrete type of the BaseObject and then calling it's 976 | `Serialize` function which is then added as StartupData to be stored in the 977 | snapshot. Just to be clear that what is getting added to the startup data 978 | in this case if the InternalFieldInfo so just the information about the type 979 | and and the size of this type. 980 | 981 | Later we see that `Environment::Serialize` has be updated with the following 982 | code: 983 | ```c++ 984 | size_t i = 0; 985 | ForEachBaseObject([&](BaseObject* obj) { 986 | switch (obj->type()) { 987 | case InternalFieldType::kFSBindingData: { 988 | size_t index = creator->AddData(ctx, obj->object()); 989 | info.bindings.push_back({"FSBindingData", i++, index}); 990 | fs::BindingData* ptr = static_cast(obj); 991 | ptr->Serialize(ctx, creator); 992 | break; 993 | } 994 | default: { UNREACHABLE(); } 995 | } 996 | }); 997 | ``` 998 | Notice that the closure is passed to `ForEachBaseObject` which will call it 999 | for each BaseObject in cleanup_hooks_. And also note that this time we are 1000 | calling BindingData::Serialize(Context, SnapshotCreator). 1001 | 1002 | Recall that we arrived here from 1003 | Environment::Serialialze and we are in the process of serializing this 1004 | BaseObject: 1005 | ```console 1006 | (lldb) expr *obj 1007 | (node::BaseObject) $16 = { 1008 | type_ = kFSBindingData 1009 | persistent_handle_ = { 1010 | v8::PersistentBase = (val_ = 0x0000000005a89b00) 1011 | } 1012 | env_ = 0x0000000005a7f930 1013 | pointer_data_ = 0x0000000005b34540 1014 | } 1015 | ``` 1016 | First this is we are going to add the v8::Global persistent_handle_ 1017 | to the snapshot context using `AddData`: 1018 | ```console 1019 | (lldb) jlh obj->object() 1020 | 0x218677edf831: [JS_API_OBJECT_TYPE] 1021 | - map: 0x296d42c9aa51 [DictionaryProperties] 1022 | - prototype: 0x24367d204821 1023 | - elements: 0x1fd3c5ec1309 [HOLEY_ELEMENTS] 1024 | - embedder fields: 2 1025 | - properties: 0x218677ee0929 { 1026 | StatWatcher: 0x12df1ab22811 (data, dict_index: 40, attrs: [WEC]) 1027 | bigintStatValues: 0x218677edf941 (data, dict_index: 39, attrs: [WEC]) 1028 | openFileHandle: 0x12df1ab20099 (data, dict_index: 4, attrs: [WEC]) 1029 | statValues: 0x218677edf8a1 (data, dict_index: 38, attrs: [WEC]) 1030 | fdatasync: 0x12df1ab203f1 (data, dict_index: 7, attrs: [WEC]) 1031 | internalModuleReadJSON: 0x12df1ab20bc9 (data, dict_index: 14, attrs: [WEC]) 1032 | writeString: 0x12df1ab21841 (data, dict_index: 25, attrs: [WEC]) 1033 | fsync: 0x12df1ab20519 (data, dict_index: 8, attrs: [WEC]) 1034 | close: 0x12df1ab1fe89 (data, dict_index: 2, attrs: [WEC]) 1035 | ... 1036 | ``` 1037 | Next, we have : 1038 | ```c++ 1039 | info.bindings.push_back({"FSBindingData", i++, index}); 1040 | fs::BindingData* ptr = static_cast(obj); 1041 | ptr->Serialize(ctx, creator); 1042 | ``` 1043 | `info` is of type EnvSerializeInfo which has a bindings field: 1044 | ```c++ 1045 | struct EnvSerializeInfo { 1046 | std::vector bindings; 1047 | ``` 1048 | So we are adding an entry to this vector. After that we cast `obj` from BaseObject 1049 | to `fs::BindingData` and call it's `Serialize` function which will land in 1050 | node_file.cc `BindingData::Serialize`: 1051 | ```c++ 1052 | void BindingData::Serialize(Local context, v8::SnapshotCreator* creator) { 1053 | HandleScope scope(context->GetIsolate()); 1054 | object()->SetPrivate(context, env()->fs_stats_field_array_symbol(), stats_field_array.GetJSArray()).FromJust(); 1055 | stats_field_array.Release(); 1056 | 1057 | object()->SetPrivate(context, env()->fs_stats_field_bigint_array_symbol(), stats_field_bigint_array.GetJSArray()).FromJust(); 1058 | stats_field_bigint_array.Release(); 1059 | } 1060 | ``` 1061 | So this is setting Private properties on the peristent_handle, note that the 1062 | SnapshotCreator is not used. Also the arrays are released. 1063 | After that we are done and will break out of the iteration of all the 1064 | base objects. 1065 | 1066 | In `Snapshot::NewContextFromSnapshot` we have the following call: 1067 | ```c++ 1068 | MaybeHandle Snapshot::NewContextFromSnapshot( 1069 | Isolate* isolate, Handle global_proxy, size_t context_index, 1070 | v8::DeserializeEmbedderFieldsCallback embedder_fields_deserializer) { 1071 | const v8::StartupData* blob = isolate->snapshot_blob(); 1072 | bool can_rehash = ExtractRehashability(blob); 1073 | Vector context_data = SnapshotImpl::ExtractContextData( 1074 | blob, static_cast(context_index)); 1075 | SnapshotData snapshot_data(MaybeDecompress(context_data)); 1076 | 1077 | MaybeHandle maybe_result = ContextDeserializer::DeserializeContext( 1078 | isolate, &snapshot_data, can_rehash, global_proxy, 1079 | embedder_fields_deserializer); 1080 | ... 1081 | } 1082 | ``` 1083 | `ContextDeserializer::DeserializeContext` will call: 1084 | ```c++ 1085 | MaybeHandle ContextDeserializer::DeserializeContext( 1086 | Isolate* isolate, const SnapshotData* data, bool can_rehash, 1087 | Handle global_proxy, 1088 | v8::DeserializeEmbedderFieldsCallback embedder_fields_deserializer) { 1089 | ContextDeserializer d(data); 1090 | d.SetRehashability(can_rehash); 1091 | 1092 | MaybeHandle maybe_result = 1093 | d.Deserialize(isolate, global_proxy, embedder_fields_deserializer); 1094 | ``` 1095 | Which will call 1096 | ```c++ 1097 | MaybeHandle ContextDeserializer::Deserialize( 1098 | Isolate* isolate, Handle global_proxy, 1099 | v8::DeserializeEmbedderFieldsCallback embedder_fields_deserializer) { 1100 | Handle result; 1101 | { 1102 | ... 1103 | DeserializeEmbedderFields(embedder_fields_deserializer); 1104 | ... 1105 | } 1106 | 1107 | void ContextDeserializer::DeserializeEmbedderFields( 1108 | v8::DeserializeEmbedderFieldsCallback embedder_fields_deserializer) { 1109 | ... 1110 | for (int code = source()->Get(); code != kSynchronize; code = source()->Get()) { 1111 | HandleScope scope(isolate()); 1112 | SnapshotSpace space = NewObject::Decode(code); 1113 | Handle obj(JSObject::cast(GetBackReferencedObject(space)), isolate()); 1114 | int index = source()->GetInt(); 1115 | int size = source()->GetInt(); 1116 | byte* data = new byte[size]; 1117 | source()->CopyRaw(data, size); 1118 | embedder_fields_deserializer.callback(v8::Utils::ToLocal(obj), index, 1119 | {reinterpret_cast(data), size}, 1120 | embedder_fields_deserializer.data); 1121 | delete[] data; 1122 | } 1123 | ``` 1124 | This will land in node_main_instance.cc in DeserializeNodeInternalFields. 1125 | ```c++ 1126 | void DeserializeNodeInternalFields(Local holder, 1127 | int index, 1128 | v8::StartupData payload, 1129 | void* env) { 1130 | ... 1131 | Environment* env_ptr = static_cast(env); 1132 | const InternalFieldInfo* info = 1133 | reinterpret_cast(payload.data); 1134 | 1135 | switch (info->type) { 1136 | case InternalFieldType::kFSBindingData: { 1137 | env_ptr->EnqueueDeserializeRequest({fs::BindingData::Deserialize, 1138 | {env_ptr->isolate(), holder}, 1139 | info->Copy()}); 1140 | break; 1141 | } 1142 | default: { UNREACHABLE(); } 1143 | } 1144 | ``` 1145 | `EnqueueDeserializeRequest` does the following: 1146 | ```c++ 1147 | void Environment::EnqueueDeserializeRequest(DeserializeRequest request) { 1148 | deserialize_requests_.push_back(std::move(request)); 1149 | } 1150 | ``` 1151 | And this is a std::list 1152 | ```c++ 1153 | std::list deserialize_requests_; 1154 | ``` 1155 | ```c++ 1156 | struct DeserializeRequest { 1157 | DeserializeRequestCallback cb; 1158 | v8::Global holder; 1159 | InternalFieldInfo* info; // Owned by the request 1160 | } 1161 | 1162 | typedef void (*DeserializeRequestCallback)(v8::Local, 1163 | v8::Local holder, 1164 | InternalFieldInfo* info); 1165 | ``` 1166 | Later when Environment::InitializeMainContext is called, DeserializeProperties 1167 | will be called: 1168 | ```c++ 1169 | void Environment::DeserializeProperties(const EnvSerializeInfo* info) { 1170 | Local ctx = context(); 1171 | 1172 | RunDeserializeRequests(); 1173 | ``` 1174 | ```c++ 1175 | void Environment::RunDeserializeRequests() { 1176 | HandleScope scope(isolate()); 1177 | Local ctx = context(); 1178 | Isolate* is = isolate(); 1179 | while (!deserialize_requests_.empty()) { 1180 | DeserializeRequest request(std::move(deserialize_requests_.front())); 1181 | deserialize_requests_.pop_front(); 1182 | Local holder = request.holder.Get(is); 1183 | request.cb(ctx, holder, request.info); 1184 | request.holder.Reset(); // unnecessary? 1185 | request.info->Delete(); 1186 | } 1187 | } 1188 | ``` 1189 | `request.cb` will land in node_file.cc BindingData::BindingData: 1190 | ```c++ 1191 | void BindingData::Deserialize(Local context, 1192 | Local holder, 1193 | InternalFieldInfo* info) { 1194 | HandleScope scope(context->GetIsolate()); 1195 | Environment* env = Environment::GetCurrent(context); 1196 | BindingData* binding = env->AddBindingData(context, holder); 1197 | 1198 | Local stats_arr = holder->GetPrivate(context, env->fs_stats_field_array_symbol()) 1199 | .ToLocalChecked(); 1200 | binding->stats_field_array.Deserialize(stats_arr.As()); 1201 | ``` 1202 | Notice that we are now extracting the Private value that were added when 1203 | serializing, and then passing that to AliasedBufferBase::Deserialize in 1204 | aliased_buffer.h which will populate the binding...TODO: 1205 | -------------------------------------------------------------------------------- /src/add.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const buffer = fs.readFileSync('../scripts/helloworld.wasm'); 3 | WebAssembly.validate(buffer); 4 | 5 | const promise = WebAssembly.instantiate(buffer); 6 | 7 | promise.then((result) => { 8 | const instance = result.instance; 9 | const module = result.module; 10 | const add = instance.exports.add; 11 | console.log(add(1, 2)); 12 | console.log(instance.exports.addTwo(2, 3)); 13 | }); 14 | -------------------------------------------------------------------------------- /src/add.wasm: -------------------------------------------------------------------------------- 1 | asm`addaddTwo 2 |  j   -------------------------------------------------------------------------------- /src/add.wat: -------------------------------------------------------------------------------- 1 | (module 2 | (func $add (export "add") (param $first i32) (param $second i32) (result i32) 3 | get_local $first 4 | get_local $second 5 | (i32.add) 6 | ) 7 | (func $addTwo (param i32) (param i32) (result i32) 8 | get_local 0 9 | get_local 1 10 | call $add 11 | ) 12 | (export "addTwo" (func $addTwo)) 13 | ) 14 | -------------------------------------------------------------------------------- /src/import.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const buffer = fs.readFileSync('src/import.wasm'); 3 | 4 | WebAssembly.validate(buffer); 5 | 6 | var importObject = { 7 | imports: { 8 | imported_func: arg => console.log('imported_func:', arg) 9 | } 10 | }; 11 | const promise = WebAssembly.instantiate(buffer, importObject); 12 | 13 | promise.then((result) => { 14 | result.instance.exports.exported_func() 15 | }); 16 | -------------------------------------------------------------------------------- /src/import.wasm: -------------------------------------------------------------------------------- 1 | asm``imports imported_func exported_func 2 | A -------------------------------------------------------------------------------- /src/import.wat: -------------------------------------------------------------------------------- 1 | ;; $ wat2wasm import.wat -o import.wasm 2 | (module 3 | (func $imported (import "imports" "imported_func") (param i32)) 4 | (func $exported (export "exported_func") 5 | i32.const 18 6 | call $imported 7 | ) 8 | ) 9 | -------------------------------------------------------------------------------- /src/mem.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const buffer = fs.readFileSync('./src/mem.wasm'); 3 | 4 | const memory = new WebAssembly.Memory({initial:1}); 5 | 6 | function consoleLogString(offset, length) { 7 | const bytes = new Uint8Array(memory.buffer, offset, length); 8 | console.log(Buffer.from(bytes, 'utf8').toString()); 9 | } 10 | 11 | const importObj = { 12 | console: { log: consoleLogString }, 13 | js: { mem: memory } 14 | }; 15 | 16 | const promise = WebAssembly.instantiate(buffer, importObj); 17 | promise.then((result) => { 18 | result.instance.exports.hi(); 19 | }); 20 | -------------------------------------------------------------------------------- /src/mem.wasm: -------------------------------------------------------------------------------- 1 | asm ``consolelogjsmemhi 2 | 3 | AA A hi -------------------------------------------------------------------------------- /src/mem.wat: -------------------------------------------------------------------------------- 1 | (module 2 | (import "console" "log" (func $log (param i32 i32))) 3 | (import "js" "mem" (memory 1)) 4 | (data (i32.const 0) "hi") 5 | (func (export "hi") 6 | i32.const 0 ;; pass offset 0 to log 7 | i32.const 2 ;; pass length 2 to log 8 | call $log)) 9 | -------------------------------------------------------------------------------- /src/start.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const buffer = fs.readFileSync('src/start.wasm'); 3 | 4 | WebAssembly.validate(buffer); 5 | 6 | var importObject = { 7 | imports: { 8 | imported_func: arg => console.log('imported_func:', arg) 9 | } 10 | }; 11 | const promise = WebAssembly.instantiate(buffer, importObject); 12 | 13 | promise.then((result) => { 14 | result.instance.exports.exported_func() 15 | }); 16 | -------------------------------------------------------------------------------- /src/start.wasm: -------------------------------------------------------------------------------- 1 | asm``imports imported_funcA  exported_func 2 | # A$ -------------------------------------------------------------------------------- /src/start.wat: -------------------------------------------------------------------------------- 1 | ;; $ wat2wasm start.wat -o start.wasm 2 | (module 3 | (func $imported (import "imports" "imported_func") (param i32)) 4 | (func $exported (export "exported_func") 5 | get_global 0 6 | call $imported 7 | ) 8 | (func $start_function 9 | i32.const 2 10 | set_global 0) 11 | (global (mut i32) (i32.const 0)) 12 | (start $start_function) 13 | ) 14 | -------------------------------------------------------------------------------- /test/base-object_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "gtest/gtest.h" 3 | #include "v8.h" 4 | #include "libplatform/libplatform.h" 5 | #include "base-object.h" 6 | #include "base-object-inl.h" 7 | 8 | #ifndef ARRAY_BUFFER_ALLOCATOR_ 9 | #define ARRAY_BUFFER_ALLOCATOR_ 10 | class ArrayBufferAllocator : public v8::ArrayBuffer::Allocator { 11 | public: 12 | virtual void* Allocate(size_t length) { 13 | void* data = AllocateUninitialized(length); 14 | return data == NULL ? data : memset(data, 0, length); 15 | } 16 | virtual void* AllocateUninitialized(size_t length) { 17 | return malloc(length); 18 | } 19 | virtual void Free(void* data, size_t) { 20 | free(data); 21 | } 22 | }; 23 | #endif 24 | 25 | /** 26 | * The goal of this test is simply to learn about 27 | * the BaseObject type. 28 | */ 29 | TEST(BaseObject, baseObject) { 30 | v8::Platform* platform = v8::platform::CreateDefaultPlatform(); 31 | v8::V8::InitializePlatform(platform); 32 | v8::V8::Initialize(); 33 | v8::Isolate::CreateParams params; 34 | ArrayBufferAllocator allocator; 35 | params.array_buffer_allocator = &allocator; 36 | v8::Isolate* isolate = v8::Isolate::New(params); 37 | v8::Isolate::Scope isolate_scope(isolate); 38 | v8::HandleScope handle_scope(isolate); 39 | v8::Local context = v8::Context::New(isolate); 40 | v8::Context::Scope context_scope(context); 41 | uv_loop_t* event_loop = uv_default_loop(); 42 | node::IsolateData* isolateData = new node::IsolateData(isolate, event_loop); 43 | node::Environment* env = new node::Environment(isolateData, context); 44 | v8::Local handle = v8::String::NewFromUtf8(isolate, "testing", v8::NewStringType::kNormal).ToLocalChecked(); 45 | v8::Local obj = v8::Local::Cast(handle); 46 | node::BaseObject bo {env, obj}; 47 | EXPECT_EQ(false, bo.persistent().IsEmpty()); 48 | bo.persistent().Reset(); 49 | } 50 | -------------------------------------------------------------------------------- /test/environment_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "gtest/gtest.h" 3 | #include "v8.h" 4 | #include "libplatform/libplatform.h" 5 | #include "env.h" 6 | 7 | #ifndef ARRAY_BUFFER_ALLOCATOR_ 8 | #define ARRAY_BUFFER_ALLOCATOR_ 9 | class ArrayBufferAllocator : public v8::ArrayBuffer::Allocator { 10 | public: 11 | virtual void* Allocate(size_t length) { 12 | void* data = AllocateUninitialized(length); 13 | return data == NULL ? data : memset(data, 0, length); 14 | } 15 | virtual void* AllocateUninitialized(size_t length) { 16 | return malloc(length); 17 | } 18 | virtual void Free(void* data, size_t) { 19 | free(data); 20 | } 21 | }; 22 | #endif 23 | 24 | TEST(Environment, env) { 25 | v8::Platform* platform = v8::platform::CreateDefaultPlatform(); 26 | v8::V8::InitializePlatform(platform); 27 | v8::V8::Initialize(); 28 | v8::Isolate::CreateParams params; 29 | ArrayBufferAllocator allocator; 30 | params.array_buffer_allocator = &allocator; 31 | v8::Isolate* isolate = v8::Isolate::New(params); 32 | v8::Local ctx = isolate->GetCurrentContext(); 33 | EXPECT_EQ(true, ctx.IsEmpty()) << "Context should be empty"; 34 | } 35 | -------------------------------------------------------------------------------- /test/main.cc: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | int main(int argc, char* argv[]) { 4 | ::testing::InitGoogleTest(&argc, argv); 5 | return RUN_ALL_TESTS(); 6 | } 7 | --------------------------------------------------------------------------------