├── .env.sample ├── .eslintrc.js ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .nvmrc ├── .prettierignore ├── .prettierrc.js ├── LICENSE ├── README.md ├── config.example.json ├── config.json ├── package.json ├── packages ├── assertion-monitor │ ├── README.md │ ├── __test__ │ │ ├── monitoring.test.ts │ │ └── testConfigs.ts │ ├── abi.ts │ ├── alerts.ts │ ├── blockchain.ts │ ├── chains.ts │ ├── constants.ts │ ├── errors.ts │ ├── index.ts │ ├── main.ts │ ├── monitoring.ts │ ├── package.json │ ├── reportAssertionMonitorAlertToSlack.ts │ ├── tsconfig.json │ ├── types.ts │ ├── utils.ts │ └── vitest.config.ts ├── batch-poster-monitor │ ├── README.md │ ├── chains.ts │ ├── index.ts │ ├── package.json │ ├── reportBatchPosterAlertToSlack.ts │ ├── tsconfig.json │ └── types.ts ├── retryable-monitor │ ├── README.md │ ├── core │ │ ├── depositEventFetcher.ts │ │ ├── reportGenerator.ts │ │ ├── retryableChecker.ts │ │ ├── retryableCheckerMode.ts │ │ ├── tokenDataFetcher.ts │ │ └── types.ts │ ├── handlers │ │ ├── handleFailedRetryablesFound.ts │ │ ├── handleRedeemedRetryablesFound.ts │ │ ├── notion │ │ │ ├── alertUntriagedRetraybles.ts │ │ │ ├── createNotionClient.ts │ │ │ └── syncRetryableToNotion.ts │ │ ├── reportFailedRetryables.ts │ │ └── slack │ │ │ ├── postSlackMessage.ts │ │ │ ├── slackMessageFormattingUtils.ts │ │ │ └── slackMessageGenerator.ts │ ├── index.ts │ ├── package.json │ └── tsconfig.json └── utils │ ├── config.ts │ ├── getExplorerUrlPrefixes.ts │ ├── index.ts │ ├── package.json │ ├── postSlackMessage.ts │ ├── sanitizeSlackMessage.ts │ ├── tsconfig.json │ └── types.ts ├── tsconfig.base.json └── yarn.lock /.env.sample: -------------------------------------------------------------------------------- 1 | NODE_ENV= 2 | 3 | RETRYABLE_MONITORING_SLACK_TOKEN= 4 | RETRYABLE_MONITORING_SLACK_CHANNEL= 5 | 6 | BATCH_POSTER_MONITORING_SLACK_TOKEN= 7 | BATCH_POSTER_MONITORING_SLACK_CHANNEL= 8 | 9 | ASSERTION_MONITORING_SLACK_TOKEN= 10 | ASSERTION_MONITORING_SLACK_CHANNEL= 11 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | commonjs: true, 4 | es6: true, 5 | node: true, 6 | }, 7 | plugins: ['prettier'], 8 | extends: [ 9 | 'eslint:recommended', 10 | 'plugin:prettier/recommended', // Enables eslint-plugin-prettier and displays prettier errors as ESLint errors. Make sure this is always the last configuration in the extends array. 11 | ], 12 | parserOptions: { 13 | ecmaVersion: 2018, // Allows for the parsing of modern ECMAScript features 14 | sourceType: 'module', // Allows for the use of imports 15 | }, 16 | rules: { 17 | 'prettier/prettier': 'error', 18 | 'no-unused-vars': 'off', 19 | 'prefer-const': [2, { destructuring: 'all' }], 20 | 'object-curly-spacing': ['error', 'always'], 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | node-version: [18.x] 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - name: Use Node.js ${{ matrix.node-version }} 21 | uses: actions/setup-node@v4 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | cache: 'yarn' 25 | 26 | - name: Install dependencies 27 | run: yarn install --frozen-lockfile 28 | 29 | - name: Run tests 30 | run: yarn test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | yarn-error.log 4 | .envrc 5 | dist 6 | ./*.js 7 | .DS_Store 8 | logfile.log 9 | config.json -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 20 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules/** -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | semi: false, 3 | trailingComma: 'es5', 4 | singleQuote: true, 5 | printWidth: 80, 6 | tabWidth: 2, 7 | arrowParens: 'avoid', 8 | bracketSpacing: true, 9 | overrides: [ 10 | { 11 | files: '*.sol', 12 | options: { 13 | printWidth: 100, 14 | tabWidth: 4, 15 | useTabs: false, 16 | singleQuote: false, 17 | bracketSpacing: true, 18 | explicitTypes: 'always', 19 | }, 20 | }, 21 | ], 22 | } 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Arbitrum Monitoring 2 | 3 | ## Overview 4 | 5 | This monitoring suite helps you track the health and performance of your Arbitrum chains through three specialized monitors: 6 | 7 | 1. [**Retryable Monitor**](./packages/retryable-monitor/README.md) - Tracks ParentChain->ChildChain message execution and retryable ticket lifecycle 8 | 2. [**Batch Poster Monitor**](./packages/batch-poster-monitor/README.md) - Monitors batch posting and data availability 9 | 3. [**Assertion Monitor**](./packages/assertion-monitor/README.md) - Monitor assertion creation and validation on Arbitrum chains 10 | 11 | Each monitor has its own detailed documentation with technical specifics and implementation details. 12 | 13 | ## Prerequisites 14 | 15 | - Node.js v18 or greater 16 | - Yarn package manager 17 | - Access to Arbitrum chain RPC endpoints 18 | - Access to parent chain RPC endpoints 19 | - Slack workspace for alerts (optional) 20 | 21 | ## Installation 22 | 23 | 1. Clone and install dependencies: 24 | 25 | ```bash 26 | git clone https://github.com/OffchainLabs/arbitrum-monitoring.git 27 | cd arbitrum-monitoring 28 | yarn install 29 | ``` 30 | 31 | ## Configuration 32 | 33 | ### Chain Configuration 34 | 35 | 1. Copy and edit the config file: 36 | 37 | ```bash 38 | cp config.example.json config.json 39 | ``` 40 | 41 | 2. Configure your chains in `config.json`: 42 | 43 | ```json 44 | { 45 | "childChains": [ 46 | { 47 | "name": "Your Chain Name", 48 | "chainId": 421614, 49 | "parentChainId": 11155111, 50 | "confirmPeriodBlocks": 45818, 51 | "parentRpcUrl": "https://your-parent-chain-rpc", 52 | "orbitRpcUrl": "https://your-chain-rpc", 53 | "ethBridge": { 54 | "bridge": "0x...", 55 | "inbox": "0x...", 56 | "outbox": "0x...", 57 | "rollup": "0x...", 58 | "sequencerInbox": "0x..." 59 | } 60 | } 61 | ] 62 | } 63 | ``` 64 | 65 | ### Alert Configuration 66 | 67 | 1. Copy and configure the environment file: 68 | 69 | ```bash 70 | cp .env.sample .env 71 | ``` 72 | 73 | 2. Set up Slack alerts in `.env` (optional): 74 | 75 | ```bash 76 | NODE_ENV=CI 77 | RETRYABLE_MONITORING_SLACK_TOKEN=your-slack-token 78 | RETRYABLE_MONITORING_SLACK_CHANNEL=your-slack-channel 79 | BATCH_POSTER_MONITORING_SLACK_TOKEN=your-slack-token 80 | BATCH_POSTER_MONITORING_SLACK_CHANNEL=your-slack-channel 81 | ASSERTION_MONITORING_SLACK_TOKEN=your-slack-token 82 | ASSERTION_MONITORING_SLACK_CHANNEL=your-slack-channel 83 | ``` 84 | 85 | Required environment variables: 86 | 87 | - `RETRYABLE_MONITORING_NOTION_TOKEN`: Notion API token for database integration 88 | - `RETRYABLE_MONITORING_NOTION_DB_ID`: Notion database ID for storing retryable tickets 89 | 90 | ## Usage 91 | 92 | All monitors support these base options: 93 | 94 | - `--configPath`: Path to configuration file (default: "config.json") 95 | - `--enableAlerting`: Enable Slack alerts (default: false) 96 | 97 | ### Quick Start Commands 98 | 99 | ```bash 100 | # Monitor retryable tickets 101 | yarn retryable-monitor [options] 102 | 103 | # Monitor batch posting 104 | yarn batch-poster-monitor [options] 105 | 106 | # Monitor chain assertions 107 | yarn assertion-monitor [options] 108 | ``` 109 | 110 | See individual monitor READMEs for specific options and features: 111 | 112 | - [Retryable Monitor Details](./packages/retryable-monitor/README.md) 113 | - [Batch Poster Monitor Details](./packages/batch-poster-monitor/README.md) 114 | - [Assertion Monitor Details](./packages/assertion-monitor/README.md) 115 | 116 | ### Notion Integration 117 | 118 | When `--writeToNotion` is enabled, the monitor will: 119 | 120 | - Create new pages in the Notion database for each retryable ticket 121 | - Update existing pages when ticket status changes 122 | - Run a daily sweep to mark expired tickets 123 | - Track ticket status, creation time, expiration time, and transaction hashes 124 | 125 | The Notion database should have the following properties: 126 | 127 | - Ticket ID (title) 128 | - Status (select) 129 | - Created At (date) 130 | - Expires At (date) 131 | - Transaction Hash (url) 132 | - Last Updated (date) 133 | -------------------------------------------------------------------------------- /config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "childChains": [ 3 | { 4 | "name": "Arbitrum Sepolia", 5 | "chainId": 421614, 6 | "parentChainId": 11155111, 7 | "confirmPeriodBlocks": 45818, 8 | "parentRpcUrl": "https://sepolia.drpc.org", 9 | "orbitRpcUrl": "https://sepolia-rollup.arbitrum.io/rpc", 10 | "explorerUrl": "https://sepolia.arbiscan.io", 11 | "parentExplorerUrl": "https://sepolia.etherscan.io", 12 | "ethBridge": { 13 | "bridge": "0x38f918D0E9F1b721EDaA41302E399fa1B79333a9", 14 | "inbox": "0xaAe29B0366299461418F5324a79Afc425BE5ae21", 15 | "outbox": "0x65f07C7D521164a4d5DaC6eB8Fac8DA067A3B78F", 16 | "rollup": "0x042b2e6c5e99d4c521bd49beed5e99651d9b0cf4", 17 | "sequencerInbox": "0x6c97864CE4bE1C2C8bB6aFe3A115E6D7Dca82E71" 18 | } 19 | }, 20 | { 21 | "name": "Xai Testnet", 22 | "chainId": 37714555429, 23 | "parentChainId": 421614, 24 | "confirmPeriodBlocks": 150, 25 | "parentRpcUrl": "https://sepolia-rollup.arbitrum.io/rpc", 26 | "orbitRpcUrl": "https://testnet-v2.xai-chain.net/rpc", 27 | "explorerUrl": "https://testnet-explorer-v2.xai-chain.net/", 28 | "parentExplorerUrl": "https://sepolia.arbiscan.io/", 29 | "ethBridge": { 30 | "bridge": "0x6c7FAC4edC72E86B3388B48979eF37Ecca5027e6", 31 | "inbox": "0x6396825803B720bc6A43c63caa1DcD7B31EB4dd0", 32 | "outbox": "0xc7491a559b416540427f9f112C5c98b1412c5d51", 33 | "rollup": "0xeedE9367Df91913ab149e828BDd6bE336df2c892", 34 | "sequencerInbox": "0x529a2061A1973be80D315770bA9469F3Da40D938" 35 | }, 36 | "nativeToken": "0x4e6f41acbfa8eb4a3b25e151834d9a14b49b69d2", 37 | "tokenBridge": { 38 | "parentCustomGateway": "0x04e14E04949D49ae9c551ca8Cc3192310Ce65D88", 39 | "parentErc20Gateway": "0xCcB451C4Df22addCFe1447c58bC6b2f264Bb1256", 40 | "parentGatewayRouter": "0x185b868DBBF41554465fcb99C6FAb9383E15f47A", 41 | "parentMultiCall": "0xA115146782b7143fAdB3065D86eACB54c169d092", 42 | "parentProxyAdmin": "0x022c515aEAb29aaFf82e86A10950cE14eA89C9c5", 43 | "parentWeth": "0x0000000000000000000000000000000000000000", 44 | "parentWethGateway": "0x0000000000000000000000000000000000000000", 45 | "childCustomGateway": "0xea1ce1CC75C948488515A3058E10aa82da40cE8F", 46 | "childErc20Gateway": "0xD840761a09609394FaFA3404bEEAb312059AC558", 47 | "childGatewayRouter": "0x3B8ba769a43f34cdD67a20aF60d08D54C9C8f1AD", 48 | "childMultiCall": "0x5CBd60Ae5Af80A42FA8b0F20ADF95A8879844984", 49 | "childProxyAdmin": "0x7C1BA251d812fb34aF5C2566040C3C30585aFed9", 50 | "childWeth": "0x0000000000000000000000000000000000000000", 51 | "childWethGateway": "0x0000000000000000000000000000000000000000" 52 | } 53 | } 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "childChains": [ 3 | { 4 | "name": "Arbitrum Sepolia", 5 | "chainId": 421614, 6 | "parentChainId": 11155111, 7 | "confirmPeriodBlocks": 45818, 8 | "parentRpcUrl": "https://sepolia.drpc.org", 9 | "orbitRpcUrl": "https://sepolia-rollup.arbitrum.io/rpc", 10 | "explorerUrl": "https://sepolia.arbiscan.io", 11 | "parentExplorerUrl": "https://sepolia.etherscan.io", 12 | "ethBridge": { 13 | "bridge": "0x38f918D0E9F1b721EDaA41302E399fa1B79333a9", 14 | "inbox": "0xaAe29B0366299461418F5324a79Afc425BE5ae21", 15 | "outbox": "0x65f07C7D521164a4d5DaC6eB8Fac8DA067A3B78F", 16 | "rollup": "0x042b2e6c5e99d4c521bd49beed5e99651d9b0cf4", 17 | "sequencerInbox": "0x6c97864CE4bE1C2C8bB6aFe3A115E6D7Dca82E71" 18 | } 19 | }, 20 | { 21 | "chainId": 37714555429, 22 | "confirmPeriodBlocks": 150, 23 | "ethBridge": { 24 | "bridge": "0x6c7FAC4edC72E86B3388B48979eF37Ecca5027e6", 25 | "inbox": "0x6396825803B720bc6A43c63caa1DcD7B31EB4dd0", 26 | "outbox": "0xc7491a559b416540427f9f112C5c98b1412c5d51", 27 | "rollup": "0xeedE9367Df91913ab149e828BDd6bE336df2c892", 28 | "sequencerInbox": "0x529a2061A1973be80D315770bA9469F3Da40D938" 29 | }, 30 | "nativeToken": "0x4e6f41acbfa8eb4a3b25e151834d9a14b49b69d2", 31 | "explorerUrl": "https://testnet-explorer-v2.xai-chain.net/", 32 | "rpcUrl": "https://testnet-v2.xai-chain.net/rpc", 33 | "name": "Xai Testnet", 34 | "slug": "xai-testnet", 35 | "parentChainId": 421614, 36 | "retryableLifetimeSeconds": 604800, 37 | "isCustom": true, 38 | "tokenBridge": { 39 | "parentCustomGateway": "0x04e14E04949D49ae9c551ca8Cc3192310Ce65D88", 40 | "parentErc20Gateway": "0xCcB451C4Df22addCFe1447c58bC6b2f264Bb1256", 41 | "parentGatewayRouter": "0x185b868DBBF41554465fcb99C6FAb9383E15f47A", 42 | "parentMultiCall": "0xA115146782b7143fAdB3065D86eACB54c169d092", 43 | "parentProxyAdmin": "0x022c515aEAb29aaFf82e86A10950cE14eA89C9c5", 44 | "parentWeth": "0x0000000000000000000000000000000000000000", 45 | "parentWethGateway": "0x0000000000000000000000000000000000000000", 46 | "childCustomGateway": "0xea1ce1CC75C948488515A3058E10aa82da40cE8F", 47 | "childErc20Gateway": "0xD840761a09609394FaFA3404bEEAb312059AC558", 48 | "childGatewayRouter": "0x3B8ba769a43f34cdD67a20aF60d08D54C9C8f1AD", 49 | "childMultiCall": "0x5CBd60Ae5Af80A42FA8b0F20ADF95A8879844984", 50 | "childProxyAdmin": "0x7C1BA251d812fb34aF5C2566040C3C30585aFed9", 51 | "childWeth": "0x0000000000000000000000000000000000000000", 52 | "childWethGateway": "0x0000000000000000000000000000000000000000" 53 | }, 54 | "bridgeUiConfig": { 55 | "color": "#F30019", 56 | "network": { 57 | "name": "Xai Testnet", 58 | "logo": "/images/XaiLogo.svg", 59 | "description": "The testnet for Xai’s gaming chain." 60 | }, 61 | "nativeTokenData": { 62 | "name": "Xai", 63 | "symbol": "sXAI", 64 | "decimals": 18, 65 | "logoUrl": "/images/XaiLogo.svg" 66 | } 67 | }, 68 | "orbitRpcUrl": "https://testnet-v2.xai-chain.net/rpc", 69 | "parentRpcUrl": "https://sepolia-rollup.arbitrum.io/rpc", 70 | "parentExplorerUrl": "https://sepolia.arbiscan.io/" 71 | } 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arbitrum-monitoring", 3 | "workspaces": [ 4 | "packages/*" 5 | ], 6 | "private": true, 7 | "license": "Apache-2.0", 8 | "scripts": { 9 | "retryable-monitor": "yarn workspace retryable-monitor dev", 10 | "batch-poster-monitor": "yarn workspace batch-poster-monitor dev", 11 | "assertion-monitor": "yarn workspace assertion-monitor dev", 12 | "test": "vitest" 13 | }, 14 | "devDependencies": { 15 | "@types/node": "^20.14.10", 16 | "@types/yargs": "^17.0.19", 17 | "dotenv": "^10.0.0", 18 | "eslint": "^8.15.0", 19 | "eslint-config-prettier": "^8.3.0", 20 | "eslint-plugin-mocha": "^9.0.0", 21 | "eslint-plugin-prettier": "^4.0.0", 22 | "prettier": "^2.3.2", 23 | "prettier-plugin-solidity": "^1.0.0-beta.17", 24 | "ts-node": "^10.8.1", 25 | "typescript": "^4.5", 26 | "vitest": "^3.0.5", 27 | "yargs": "^17.5.1" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/assertion-monitor/README.md: -------------------------------------------------------------------------------- 1 | # Assertion Monitor 2 | 3 | > For installation and general configuration, see the [main README](../../README.md). 4 | 5 | ## Overview 6 | 7 | The Assertion Monitor monitors the lifecycle of assertions in both BoLD (Bounded Liquidity Delay) and pre-BoLD rollup chains. [Learn more](https://docs.arbitrum.io/how-arbitrum-works/inside-arbitrum-nitro#arbitrum-rollup-protocol). 8 | 9 | ## Command-Line Interface 10 | 11 | ```bash 12 | yarn assertion-monitor [options] 13 | 14 | Monitor assertion creation and validation on Arbitrum chains 15 | 16 | Options: 17 | --help Show help [boolean] 18 | --version Show version number [boolean] 19 | --configPath Path to config file [string] [default: "config.json"] 20 | --enableAlerting Enable Slack alerts [boolean] [default: false] 21 | 22 | Examples: 23 | yarn assertion-monitor Run with default config 24 | yarn assertion-monitor --enableAlerting Enable Slack notifications 25 | yarn assertion-monitor --configPath=custom.json Use custom config file 26 | 27 | Environment Variables: 28 | ASSERTION_MONITORING_SLACK_TOKEN Slack API token for alerts 29 | ASSERTION_MONITORING_SLACK_CHANNEL Slack channel for alerts 30 | ``` 31 | 32 | ## Monitor Details 33 | 34 | The Assertion Monitor tracks assertions through their lifecycle, implementing distinct strategies for BoLD and pre-BoLD rollup chains. 35 | 36 | ### Critical Events Monitored 37 | 38 | The monitor tracks five categories of blockchain events: 39 | 40 | - **Creation Events**: Records when new assertions and nodes are created on the chain to verify transaction execution 41 | - **Confirmation Events**: Identifies when assertions are confirmed on the parent chain after challenge periods end 42 | - **Validator Events**: Tracks validator participation metrics including stakes, challenges, and whitelist status 43 | - **Block Events**: Monitors block production rates, finalization timing, and synchronization between chains 44 | - **Chain State**: Analyzes the overall consistency between on-chain state and expected protocol behavior 45 | 46 | ### Alert Scenarios 47 | 48 | The monitor triggers alerts when these conditions are detected: 49 | 50 | #### Creation Issues 51 | 52 | - No assertion creation events within configured time window 53 | - Chain activity without corresponding recent assertions 54 | - Extended node creation gaps on non-BoLD chains 55 | - Validator participation below required thresholds 56 | 57 | #### Confirmation Issues 58 | 59 | - Parent chain block threshold exceeded 60 | - Assertions stuck in challenge period 61 | - Data inconsistencies between confirmation events and confirmed blocks 62 | - Confirmation events missing despite available creation events 63 | 64 | #### Other Issues 65 | 66 | - Validator whitelist disabled on pre-BoLD chains 67 | - Base stake below 1 ETH threshold on BoLD chains 68 | - Parent-child chain synchronization anomalies 69 | - State inconsistencies between expected and observed chain state 70 | 71 | For implementation details and thresholds, see `alerts.ts` and `monitoring.ts`. 72 | -------------------------------------------------------------------------------- /packages/assertion-monitor/__test__/monitoring.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, test, vi, beforeEach, afterEach } from 'vitest' 2 | import { analyzeAssertionEvents } from '../monitoring' 3 | import type { ChainState } from '../types' 4 | import { Block } from 'viem' 5 | import { 6 | NO_CREATION_EVENTS_ALERT, 7 | CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT, 8 | NO_CONFIRMATION_EVENTS_ALERT, 9 | CONFIRMATION_DELAY_ALERT, 10 | CREATION_EVENT_STUCK_ALERT, 11 | NON_BOLD_NO_RECENT_CREATION_ALERT, 12 | VALIDATOR_WHITELIST_DISABLED_ALERT, 13 | NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT, 14 | BOLD_LOW_BASE_STAKE_ALERT, 15 | } from '../alerts' 16 | 17 | // Mock constants to avoid importing from the actual constants file 18 | vi.mock('../constants', () => ({ 19 | MAXIMUM_SEARCH_DAYS: 7, 20 | RECENT_CREATION_CHECK_HOURS: 4, 21 | CHALLENGE_PERIOD_SECONDS: 6.4 * 24 * 60 * 60, // 6.4 days in seconds 22 | SEARCH_WINDOW_SECONDS: 7 * 24 * 60 * 60, // 7 days in seconds 23 | RECENT_ACTIVITY_SECONDS: 4 * 60 * 60, // 4 hours in seconds 24 | VALIDATOR_AFK_BLOCKS: 50, // Add the validator AFK blocks constant 25 | })) 26 | 27 | // Mock chain info 28 | const mockChainInfo = { 29 | name: 'Test Chain', 30 | confirmPeriodBlocks: 100, 31 | ethBridge: { 32 | rollup: '0x1234567890123456789012345678901234567890', 33 | }, 34 | } as any 35 | 36 | // Base timestamp for tests (current time) 37 | const NOW = 1672531200n // 2023-01-01 00:00:00 UTC as bigint 38 | 39 | describe('Assertion Health Monitoring', () => { 40 | // Mock Date.now() to return a consistent timestamp 41 | let originalDateNow: () => number 42 | 43 | beforeEach(() => { 44 | originalDateNow = Date.now 45 | Date.now = vi.fn(() => Number(NOW) * 1000) // Convert to milliseconds 46 | }) 47 | 48 | afterEach(() => { 49 | Date.now = originalDateNow 50 | }) 51 | 52 | // Helper function to create a basic chain state 53 | function createBaseChainState(): ChainState { 54 | return { 55 | childCurrentBlock: { 56 | number: 2000n, 57 | timestamp: NOW - 50n, 58 | hash: '0x5678' as `0x${string}`, 59 | parentHash: '0x0000' as `0x${string}`, 60 | } as Block, 61 | childLatestCreatedBlock: { 62 | number: 900n, 63 | timestamp: NOW - 3600n, // 1 hour ago 64 | hash: '0xabcd' as `0x${string}`, 65 | parentHash: '0x0000' as `0x${string}`, 66 | } as Block, 67 | childLatestConfirmedBlock: { 68 | number: 850n, 69 | timestamp: NOW - 7200n, // 2 hours ago 70 | hash: '0xef01' as `0x${string}`, 71 | parentHash: '0x0000' as `0x${string}`, 72 | } as Block, 73 | // Parent chain block information 74 | parentCurrentBlock: { 75 | number: 150n, 76 | timestamp: NOW, 77 | hash: '0xparent1' as `0x${string}`, 78 | parentHash: '0x0000' as `0x${string}`, 79 | } as Block, 80 | parentBlockAtCreation: { 81 | number: 140n, 82 | timestamp: NOW - 3600n, // 1 hour ago 83 | hash: '0xparent2' as `0x${string}`, 84 | parentHash: '0x0000' as `0x${string}`, 85 | } as Block, 86 | parentBlockAtConfirmation: { 87 | number: 130n, 88 | timestamp: NOW - 7200n, // 2 hours ago 89 | hash: '0xparent3' as `0x${string}`, 90 | parentHash: '0x0000' as `0x${string}`, 91 | } as Block, 92 | recentCreationEvent: null, 93 | recentConfirmationEvent: null, 94 | isValidatorWhitelistDisabled: false, 95 | isBaseStakeBelowThreshold: false 96 | } 97 | } 98 | 99 | describe('BOLD Chain Tests', () => { 100 | test('should not alert when everything is normal', async () => { 101 | const chainState = createBaseChainState() 102 | 103 | // Update the blocks to have a normal confirmation delay 104 | chainState.childCurrentBlock = { 105 | ...chainState.childCurrentBlock!, 106 | number: 2000n, 107 | } as Block 108 | 109 | chainState.childLatestConfirmedBlock = { 110 | ...chainState.childLatestConfirmedBlock!, 111 | number: 1950n, // Only 50 blocks behind, less than threshold 112 | } as Block 113 | 114 | // Make sure creation events are recent 115 | chainState.childLatestCreatedBlock = { 116 | ...chainState.childLatestCreatedBlock!, 117 | timestamp: NOW - 1000n, // Very recent 118 | number: 1980n, // Between latest and confirmed 119 | } as Block 120 | 121 | const alerts = await analyzeAssertionEvents( 122 | chainState, 123 | mockChainInfo, 124 | true 125 | ) 126 | 127 | // Should have no alerts 128 | expect(alerts.length).toBe(0) 129 | }) 130 | 131 | test('should alert when no creation events are found', async () => { 132 | const chainState = createBaseChainState() 133 | chainState.childLatestCreatedBlock = undefined 134 | 135 | const alerts = await analyzeAssertionEvents( 136 | chainState, 137 | mockChainInfo, 138 | true 139 | ) 140 | 141 | expect(alerts[0]).toBe(NO_CREATION_EVENTS_ALERT) 142 | }) 143 | 144 | test('should alert when chain has activity but no recent creation events', async () => { 145 | const chainState = createBaseChainState() 146 | // Set creation event to be older than the recent activity threshold (4 hours) 147 | chainState.childLatestCreatedBlock = { 148 | ...chainState.childLatestCreatedBlock!, 149 | timestamp: NOW - BigInt(5 * 60 * 60), // 5 hours ago 150 | } as Block 151 | 152 | const alerts = await analyzeAssertionEvents( 153 | chainState, 154 | mockChainInfo, 155 | true 156 | ) 157 | 158 | // Check if alerts array exists and has at least one element 159 | expect(alerts.length).toBeGreaterThan(0) 160 | 161 | // Check for expected alert 162 | expect(alerts).toContain(CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT) 163 | }) 164 | 165 | test('should alert when no confirmation events exist', async () => { 166 | const chainState = createBaseChainState() 167 | chainState.childLatestConfirmedBlock = undefined 168 | 169 | const alerts = await analyzeAssertionEvents( 170 | chainState, 171 | mockChainInfo, 172 | true 173 | ) 174 | 175 | // Check if alerts array exists and has at least one element 176 | expect(alerts.length).toBeGreaterThan(0) 177 | 178 | // Check for expected alert 179 | expect(alerts).toContain(NO_CONFIRMATION_EVENTS_ALERT) 180 | }) 181 | 182 | test('should alert when confirmation delay exceeds period', async () => { 183 | const chainState = createBaseChainState() 184 | 185 | // Set values to trigger confirmation delay 186 | chainState.childCurrentBlock = { 187 | ...chainState.childCurrentBlock!, 188 | number: 2000n, 189 | } as Block 190 | 191 | chainState.childLatestConfirmedBlock = { 192 | ...chainState.childLatestConfirmedBlock!, 193 | number: 1700n, // 300 blocks behind, exceeds threshold 194 | } as Block 195 | 196 | // Set parent chain blocks to indicate a delay 197 | chainState.parentCurrentBlock = { 198 | ...chainState.parentCurrentBlock!, 199 | number: 300n, 200 | } as Block 201 | 202 | chainState.parentBlockAtConfirmation = { 203 | ...chainState.parentBlockAtConfirmation!, 204 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 205 | } as Block 206 | 207 | const alerts = await analyzeAssertionEvents( 208 | chainState, 209 | mockChainInfo, 210 | true 211 | ) 212 | 213 | // Check for expected alert 214 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 215 | }) 216 | 217 | test('should alert when creation event is stuck in challenge period', async () => { 218 | const chainState = createBaseChainState() 219 | // Set creation event to be older than the challenge period 220 | chainState.childLatestCreatedBlock = { 221 | ...chainState.childLatestCreatedBlock!, 222 | timestamp: NOW - BigInt(7 * 24 * 60 * 60), // 7 days ago 223 | } as Block 224 | 225 | const alerts = await analyzeAssertionEvents( 226 | chainState, 227 | mockChainInfo, 228 | true 229 | ) 230 | 231 | // Check if alerts array exists and has at least one element 232 | expect(alerts.length).toBeGreaterThan(0) 233 | 234 | // Check for expected alert 235 | expect(alerts).toContain(CREATION_EVENT_STUCK_ALERT) 236 | }) 237 | 238 | test('should include validator whitelist status in confirmation delay alerts', async () => { 239 | const chainState = createBaseChainState() 240 | 241 | // Set values to trigger confirmation delay 242 | chainState.childCurrentBlock = { 243 | ...chainState.childCurrentBlock!, 244 | number: 2000n, 245 | } as Block 246 | 247 | chainState.childLatestConfirmedBlock = { 248 | ...chainState.childLatestConfirmedBlock!, 249 | number: 1700n, // 300 blocks behind, exceeds threshold 250 | } as Block 251 | 252 | // Set parent chain blocks to indicate a delay 253 | chainState.parentCurrentBlock = { 254 | ...chainState.parentCurrentBlock!, 255 | number: 300n, 256 | } as Block 257 | 258 | chainState.parentBlockAtConfirmation = { 259 | ...chainState.parentBlockAtConfirmation!, 260 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 261 | } as Block 262 | 263 | chainState.isBaseStakeBelowThreshold = true 264 | chainState.isValidatorWhitelistDisabled = true 265 | 266 | const alerts = await analyzeAssertionEvents( 267 | chainState, 268 | mockChainInfo, 269 | true 270 | ) 271 | 272 | // Check for expected alert 273 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 274 | 275 | const alertsWithWhitelistDisabled = await analyzeAssertionEvents( 276 | chainState, 277 | mockChainInfo, 278 | true 279 | ) 280 | 281 | // Should have both alerts 282 | expect(alertsWithWhitelistDisabled).toContain(CONFIRMATION_DELAY_ALERT) 283 | expect(alertsWithWhitelistDisabled).toContain( 284 | BOLD_LOW_BASE_STAKE_ALERT 285 | ) 286 | }) 287 | 288 | test('should generate multiple alerts when multiple conditions are met', async () => { 289 | const chainState = createBaseChainState() 290 | 291 | // Set creation event to be older than the recent activity threshold 292 | chainState.childLatestCreatedBlock = { 293 | ...chainState.childLatestCreatedBlock!, 294 | timestamp: NOW - BigInt(5 * 60 * 60), // 5 hours ago 295 | number: 1800n, 296 | } as Block 297 | 298 | // Set values to trigger confirmation delay 299 | chainState.childCurrentBlock = { 300 | ...chainState.childCurrentBlock!, 301 | number: 2000n, // Activity since last creation 302 | } as Block 303 | 304 | chainState.childLatestConfirmedBlock = { 305 | ...chainState.childLatestConfirmedBlock!, 306 | number: 1700n, // 300 blocks behind, exceeds threshold 307 | } as Block 308 | 309 | // Set parent chain blocks to indicate a delay 310 | chainState.parentCurrentBlock = { 311 | ...chainState.parentCurrentBlock!, 312 | number: 300n, 313 | } as Block 314 | 315 | chainState.parentBlockAtConfirmation = { 316 | ...chainState.parentBlockAtConfirmation!, 317 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 318 | } as Block 319 | 320 | const alerts = await analyzeAssertionEvents( 321 | chainState, 322 | mockChainInfo, 323 | true 324 | ) 325 | 326 | // Check that we have multiple alerts 327 | expect(alerts.length).toBeGreaterThan(1) 328 | 329 | // Check for expected alerts 330 | expect(alerts).toContain(CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT) 331 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 332 | }) 333 | 334 | test('should use parent chain blocks for confirmation delay when available', async () => { 335 | const chainState = createBaseChainState() 336 | 337 | // Set child chain blocks to normal values (no delay based on child blocks) 338 | chainState.childCurrentBlock = { 339 | ...chainState.childCurrentBlock!, 340 | number: 2000n, 341 | } as Block 342 | 343 | chainState.childLatestConfirmedBlock = { 344 | ...chainState.childLatestConfirmedBlock!, 345 | number: 1950n, // Only 50 blocks behind, less than threshold 346 | } as Block 347 | 348 | // But set parent blocks to indicate a delay 349 | chainState.parentCurrentBlock = { 350 | ...chainState.parentCurrentBlock!, 351 | number: 300n, 352 | } as Block 353 | 354 | chainState.parentBlockAtConfirmation = { 355 | ...chainState.parentBlockAtConfirmation!, 356 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 357 | } as Block 358 | 359 | const alerts = await analyzeAssertionEvents( 360 | chainState, 361 | mockChainInfo, 362 | true 363 | ) 364 | 365 | // Should alert due to parent chain block delay, despite child chain blocks being normal 366 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 367 | }) 368 | 369 | test('should not generate confirmation delay alert when parent chain block gap is zero', async () => { 370 | const chainState = createBaseChainState() 371 | 372 | // Set parent chain blocks to have no gap 373 | chainState.parentCurrentBlock = { 374 | ...chainState.parentCurrentBlock!, 375 | number: 200n, 376 | } as Block 377 | 378 | chainState.parentBlockAtConfirmation = { 379 | ...chainState.parentBlockAtConfirmation!, 380 | number: 200n, // Same as current block, so no delay 381 | } as Block 382 | 383 | // Set child blocks to indicate a delay (which would have triggered an alert in the old implementation) 384 | chainState.childCurrentBlock = { 385 | ...chainState.childCurrentBlock!, 386 | number: 2000n, 387 | } as Block 388 | 389 | chainState.childLatestConfirmedBlock = { 390 | ...chainState.childLatestConfirmedBlock!, 391 | number: 1800n, // 200 blocks behind, would have exceeded threshold for BOLD in old implementation 392 | } as Block 393 | 394 | const alerts = await analyzeAssertionEvents( 395 | chainState, 396 | mockChainInfo, 397 | true 398 | ) 399 | 400 | // Should NOT alert since parent chain blocks have no gap 401 | expect(alerts).not.toContain(CONFIRMATION_DELAY_ALERT) 402 | }) 403 | 404 | test('should alert when confirmation events exist but no confirmation blocks found', async () => { 405 | const chainState = createBaseChainState() 406 | 407 | // Set up the inconsistent state: confirmation event exists but no confirmed block 408 | chainState.childLatestConfirmedBlock = undefined 409 | 410 | // Add a mock confirmation event with the correct structure 411 | chainState.recentConfirmationEvent = { 412 | blockNumber: 130n, 413 | args: { 414 | blockHash: '0xef01' as `0x${string}`, 415 | sendRoot: '0xabcd' as `0x${string}`, 416 | assertionHash: '0x1234' as `0x${string}`, 417 | }, 418 | // Add minimal required properties to satisfy the type 419 | address: '0x1234' as `0x${string}`, 420 | data: '0x' as `0x${string}`, 421 | topics: ['0x1234' as `0x${string}`, '0x5678' as `0x${string}`], 422 | transactionHash: '0x5678' as `0x${string}`, 423 | logIndex: 0, 424 | blockHash: '0xparent3' as `0x${string}`, 425 | transactionIndex: 0, 426 | removed: false, 427 | eventName: 'AssertionConfirmed', 428 | } 429 | 430 | const alerts = await analyzeAssertionEvents( 431 | chainState, 432 | mockChainInfo, 433 | true 434 | ) 435 | 436 | // Should contain both the standard no confirmation events alert and the specific inconsistency alert 437 | expect(alerts).toContain(NO_CONFIRMATION_EVENTS_ALERT) 438 | expect(alerts).toContain(NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT) 439 | 440 | }) 441 | 442 | test('should alert when base stake is below threshold and whitelist is disabled for BoLD chain', async () => { 443 | const chainState = createBaseChainState() 444 | 445 | // Set both conditions to trigger alert 446 | chainState.isBaseStakeBelowThreshold = true 447 | chainState.isValidatorWhitelistDisabled = true 448 | 449 | const alerts = await analyzeAssertionEvents( 450 | chainState, 451 | mockChainInfo, 452 | true // isBold 453 | ) 454 | 455 | expect(alerts).toContain(BOLD_LOW_BASE_STAKE_ALERT) 456 | 457 | // Test that alert is not generated when base stake is adequate 458 | const chainStateWithAdequateStake = { 459 | ...chainState, 460 | isBaseStakeBelowThreshold: false 461 | } 462 | 463 | const alertsWithAdequateStake = await analyzeAssertionEvents( 464 | chainStateWithAdequateStake, 465 | mockChainInfo, 466 | true // isBold 467 | ) 468 | 469 | expect(alertsWithAdequateStake).not.toContain(BOLD_LOW_BASE_STAKE_ALERT) 470 | }) 471 | 472 | test('should not alert for whitelist being disabled on BoLD chain', async () => { 473 | const chainState = createBaseChainState() 474 | 475 | // Enable whitelist disabled flag 476 | chainState.isValidatorWhitelistDisabled = true 477 | 478 | const alerts = await analyzeAssertionEvents( 479 | chainState, 480 | mockChainInfo, 481 | true // isBold 482 | ) 483 | 484 | // Should not contain whitelist alert since this is a BoLD chain 485 | expect(alerts).not.toContain(VALIDATOR_WHITELIST_DISABLED_ALERT) 486 | }) 487 | }) 488 | 489 | describe('Non-BOLD Chain Tests', () => { 490 | test('should not alert when everything is normal for non-BOLD chain', async () => { 491 | const chainState = createBaseChainState() 492 | 493 | // Update the blocks to have a normal confirmation delay 494 | chainState.childCurrentBlock = { 495 | ...chainState.childCurrentBlock!, 496 | number: 2000n, 497 | } as Block 498 | 499 | chainState.childLatestConfirmedBlock = { 500 | ...chainState.childLatestConfirmedBlock!, 501 | number: 1950n, // Only 50 blocks behind, less than threshold 502 | } as Block 503 | 504 | // Make sure creation events are recent 505 | chainState.childLatestCreatedBlock = { 506 | ...chainState.childLatestCreatedBlock!, 507 | timestamp: NOW - 1000n, // Very recent 508 | number: 1980n, // Between latest and confirmed 509 | } as Block 510 | 511 | const alerts = await analyzeAssertionEvents( 512 | chainState, 513 | mockChainInfo, 514 | false 515 | ) 516 | 517 | // Our implementation now generates no alerts for normal operation 518 | expect(alerts.length).toBe(0) 519 | }) 520 | 521 | test('should alert when no creation events are found for non-BOLD chain', async () => { 522 | const chainState = createBaseChainState() 523 | chainState.childLatestCreatedBlock = undefined 524 | 525 | const alerts = await analyzeAssertionEvents( 526 | chainState, 527 | mockChainInfo, 528 | false 529 | ) 530 | 531 | expect(alerts[0]).toBe(NO_CREATION_EVENTS_ALERT) 532 | }) 533 | 534 | test('should alert when no recent creation events for non-BOLD chain', async () => { 535 | const chainState = createBaseChainState() 536 | // Set creation event to be older than the recent activity threshold (4 hours) 537 | chainState.childLatestCreatedBlock = { 538 | ...chainState.childLatestCreatedBlock!, 539 | timestamp: NOW - BigInt(5 * 60 * 60), // 5 hours ago 540 | } as Block 541 | 542 | const alerts = await analyzeAssertionEvents( 543 | chainState, 544 | mockChainInfo, 545 | false 546 | ) 547 | 548 | // Check if alerts array exists and has at least one element 549 | expect(alerts.length).toBeGreaterThan(0) 550 | 551 | // Check for expected alert 552 | expect(alerts).toContain(NON_BOLD_NO_RECENT_CREATION_ALERT) 553 | }) 554 | 555 | test('should alert when no confirmation events exist for non-BOLD chain', async () => { 556 | const chainState = createBaseChainState() 557 | chainState.childLatestConfirmedBlock = undefined 558 | 559 | const alerts = await analyzeAssertionEvents( 560 | chainState, 561 | mockChainInfo, 562 | false 563 | ) 564 | 565 | // Check if alerts array exists and has at least one element 566 | expect(alerts.length).toBeGreaterThan(0) 567 | 568 | // Check for expected alert 569 | expect(alerts).toContain(NO_CONFIRMATION_EVENTS_ALERT) 570 | }) 571 | 572 | test('should alert when confirmation delay exceeds very high threshold for non-BOLD chain', async () => { 573 | const chainState = createBaseChainState() 574 | 575 | // Set parent chain blocks to indicate a delay 576 | chainState.parentCurrentBlock = { 577 | ...chainState.parentCurrentBlock!, 578 | number: 300n, 579 | } as Block 580 | 581 | chainState.parentBlockAtConfirmation = { 582 | ...chainState.parentBlockAtConfirmation!, 583 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 584 | } as Block 585 | 586 | // Also set child blocks to have a huge gap (but this shouldn't matter anymore) 587 | chainState.childCurrentBlock = { 588 | ...chainState.childCurrentBlock!, 589 | number: 7000n, 590 | } as Block 591 | 592 | chainState.childLatestConfirmedBlock = { 593 | ...chainState.childLatestConfirmedBlock!, 594 | number: 1800n, // 5200 blocks behind 595 | } as Block 596 | 597 | const alerts = await analyzeAssertionEvents( 598 | chainState, 599 | mockChainInfo, 600 | false // non-BOLD 601 | ) 602 | 603 | // Check for expected alert - should be triggered by parent chain blocks 604 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 605 | }) 606 | 607 | test('should not alert for challenge period on non-BOLD chain', async () => { 608 | const chainState = createBaseChainState() 609 | // Set creation event to be older than the challenge period (6.4 days) 610 | chainState.childLatestCreatedBlock = { 611 | ...chainState.childLatestCreatedBlock!, 612 | timestamp: NOW - BigInt(7 * 24 * 60 * 60), // 7 days ago 613 | } as Block 614 | 615 | const alerts = await analyzeAssertionEvents( 616 | chainState, 617 | mockChainInfo, 618 | false 619 | ) 620 | 621 | // The implementation will generate other alerts, but not CREATION_EVENT_STUCK_ALERT 622 | expect(alerts).not.toContain(CREATION_EVENT_STUCK_ALERT) 623 | 624 | // But it should contain the NON_BOLD_NO_RECENT_CREATION_ALERT 625 | expect(alerts).toContain(NON_BOLD_NO_RECENT_CREATION_ALERT) 626 | }) 627 | 628 | test('should generate alerts when extreme conditions are met for non-BOLD chain', async () => { 629 | const chainState = createBaseChainState() 630 | // Set creation event to be older than the recent activity threshold 631 | chainState.childLatestCreatedBlock = { 632 | ...chainState.childLatestCreatedBlock!, 633 | timestamp: NOW - BigInt(5 * 60 * 60), // 5 hours ago 634 | number: 900n, 635 | } as Block 636 | 637 | // Set parent chain blocks to indicate a delay 638 | chainState.parentCurrentBlock = { 639 | ...chainState.parentCurrentBlock!, 640 | number: 300n, 641 | } as Block 642 | 643 | chainState.parentBlockAtConfirmation = { 644 | ...chainState.parentBlockAtConfirmation!, 645 | number: 100n, // 200 blocks behind, exceeds confirmPeriodBlocks(100) + VALIDATOR_AFK_BLOCKS(50) 646 | } as Block 647 | 648 | // Also set child blocks to have a huge gap (but this shouldn't matter anymore) 649 | chainState.childCurrentBlock = { 650 | ...chainState.childCurrentBlock!, 651 | number: 7000n, 652 | } as Block 653 | 654 | chainState.childLatestConfirmedBlock = { 655 | ...chainState.childLatestConfirmedBlock!, 656 | number: 1800n, // 5200 blocks behind 657 | } as Block 658 | 659 | const alerts = await analyzeAssertionEvents( 660 | chainState, 661 | mockChainInfo, 662 | false // non-BOLD 663 | ) 664 | 665 | // Check for required alerts 666 | expect(alerts).toContain(CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT) 667 | expect(alerts).toContain(NON_BOLD_NO_RECENT_CREATION_ALERT) 668 | expect(alerts).toContain(CONFIRMATION_DELAY_ALERT) 669 | }) 670 | 671 | test('should alert when validator whitelist is disabled for non-BOLD chain', async () => { 672 | const chainState = createBaseChainState() 673 | 674 | chainState.isValidatorWhitelistDisabled = true 675 | 676 | // Test with validator whitelist disabled 677 | const alerts = await analyzeAssertionEvents( 678 | chainState, 679 | mockChainInfo, 680 | false 681 | ) 682 | 683 | // Check if alerts array exists and has at least one element 684 | expect(alerts.length).toBeGreaterThan(0) 685 | 686 | // Check for the validator whitelist disabled alert 687 | expect(alerts).toContain(VALIDATOR_WHITELIST_DISABLED_ALERT) 688 | 689 | chainState.isValidatorWhitelistDisabled = false 690 | 691 | // Test with whitelist enabled to confirm no alert is generated 692 | const alertsWithWhitelist = await analyzeAssertionEvents( 693 | chainState, 694 | mockChainInfo, 695 | false 696 | ) 697 | expect(alertsWithWhitelist).not.toContain( 698 | VALIDATOR_WHITELIST_DISABLED_ALERT 699 | ) 700 | }) 701 | 702 | test('should alert when confirmation events exist but no confirmation blocks found for non-BOLD chains', async () => { 703 | const chainState = createBaseChainState() 704 | 705 | // Set up the inconsistent state: confirmation event exists but no confirmed block 706 | chainState.childLatestConfirmedBlock = undefined 707 | 708 | // Add a mock confirmation event with the correct structure for NODE_CONFIRMED_EVENT 709 | chainState.recentConfirmationEvent = { 710 | blockNumber: 130n, 711 | args: { 712 | blockHash: '0xef01' as `0x${string}`, 713 | sendRoot: '0xabcd' as `0x${string}`, 714 | nodeNum: 42n, 715 | }, 716 | // Add minimal required properties to satisfy the type 717 | address: '0x1234' as `0x${string}`, 718 | data: '0x' as `0x${string}`, 719 | topics: ['0x1234' as `0x${string}`, '0x5678' as `0x${string}`], 720 | transactionHash: '0x5678' as `0x${string}`, 721 | logIndex: 0, 722 | blockHash: '0xparent3' as `0x${string}`, 723 | transactionIndex: 0, 724 | removed: false, 725 | eventName: 'NodeConfirmed', 726 | } 727 | 728 | const alerts = await analyzeAssertionEvents( 729 | chainState, 730 | mockChainInfo, 731 | false // isBold = false for non-BOLD chain 732 | ) 733 | 734 | // Should contain both the standard no confirmation events alert and the specific inconsistency alert 735 | expect(alerts).toContain(NO_CONFIRMATION_EVENTS_ALERT) 736 | expect(alerts).toContain(NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT) 737 | }) 738 | }) 739 | 740 | describe('Classic Chain Tests', () => { 741 | test('should alert when validator whitelist is disabled on Classic chain', async () => { 742 | const chainState = createBaseChainState() 743 | 744 | // Enable whitelist disabled flag 745 | chainState.isValidatorWhitelistDisabled = true 746 | 747 | const alerts = await analyzeAssertionEvents( 748 | chainState, 749 | mockChainInfo, 750 | false // isBold 751 | ) 752 | 753 | expect(alerts).toContain(VALIDATOR_WHITELIST_DISABLED_ALERT) 754 | 755 | // Test that alert is not generated when whitelist is enabled 756 | const chainStateWithWhitelist = { 757 | ...chainState, 758 | isValidatorWhitelistDisabled: false 759 | } 760 | 761 | const alertsWithWhitelist = await analyzeAssertionEvents( 762 | chainStateWithWhitelist, 763 | mockChainInfo, 764 | false // isBold 765 | ) 766 | 767 | expect(alertsWithWhitelist).not.toContain(VALIDATOR_WHITELIST_DISABLED_ALERT) 768 | }) 769 | 770 | test('should not alert for base stake on Classic chain', async () => { 771 | const chainState = createBaseChainState() 772 | 773 | // Set base stake below threshold 774 | chainState.isBaseStakeBelowThreshold = true 775 | 776 | const alerts = await analyzeAssertionEvents( 777 | chainState, 778 | mockChainInfo, 779 | false // isBold 780 | ) 781 | 782 | // Should not contain base stake alert since this is a Classic chain 783 | expect(alerts).not.toContain(BOLD_LOW_BASE_STAKE_ALERT) 784 | }) 785 | }) 786 | }) 787 | -------------------------------------------------------------------------------- /packages/assertion-monitor/__test__/testConfigs.ts: -------------------------------------------------------------------------------- 1 | // Test chain configurations 2 | export const boldChainInfo = { 3 | name: 'Arbitrum Sepolia', 4 | chainId: 421614, 5 | parentChainId: 11155111, 6 | confirmPeriodBlocks: 45818, 7 | parentRpcUrl: 'https://sepolia.drpc.org', 8 | orbitRpcUrl: 'https://sepolia-rollup.arbitrum.io/rpc', 9 | ethBridge: { 10 | bridge: '0x38f918D0E9F1b721EDaA41302E399fa1B79333a9', 11 | inbox: '0xaAe29B0366299461418F5324a79Afc425BE5ae21', 12 | outbox: '0x65f07C7D521164a4d5DaC6eB8Fac8DA067A3B78F', 13 | rollup: '0x042b2e6c5e99d4c521bd49beed5e99651d9b0cf4', 14 | sequencerInbox: '0x6c97864CE4bE1C2C8bB6aFe3A115E6D7Dca82E71', 15 | }, 16 | explorerUrl: 'https://sepolia.arbiscan.io', 17 | parentExplorerUrl: 'https://sepolia.etherscan.io', 18 | isCustom: false, 19 | severity: 'critical', 20 | } 21 | 22 | export const classicChainInfo = { 23 | name: 'Xai Testnet', 24 | chainId: 37714555429, 25 | parentChainId: 421614, 26 | confirmPeriodBlocks: 150, 27 | parentRpcUrl: 'https://arbitrum-sepolia.drpc.org', 28 | orbitRpcUrl: 'https://testnet-v2.xai-chain.net/rpc', 29 | ethBridge: { 30 | bridge: '0x6c7FAC4edC72E86B3388B48979eF37Ecca5027e6', 31 | inbox: '0x6396825803B720bc6A43c63caa1DcD7B31EB4dd0', 32 | outbox: '0xc7491a559b416540427f9f112C5c98b1412c5d51', 33 | rollup: '0xeedE9367Df91913ab149e828BDd6bE336df2c892', 34 | sequencerInbox: '0x529a2061A1973be80D315770bA9469F3Da40D938', 35 | }, 36 | explorerUrl: 'https://testnet-explorer-v2.xai-chain.net', 37 | parentExplorerUrl: 'https://sepolia.arbiscan.io', 38 | isCustom: false, 39 | severity: 'critical', 40 | } -------------------------------------------------------------------------------- /packages/assertion-monitor/abi.ts: -------------------------------------------------------------------------------- 1 | import { parseAbiItem } from 'viem' 2 | 3 | export const rollupABI = [ 4 | { 5 | inputs: [], 6 | name: 'validatorWhitelistDisabled', 7 | outputs: [{ internalType: 'bool', name: '', type: 'bool' }], 8 | stateMutability: 'view', 9 | type: 'function', 10 | } 11 | ] as const 12 | 13 | export const boldABI = [ 14 | { 15 | inputs: [], 16 | name: 'genesisAssertionHash', 17 | outputs: [{ internalType: 'bytes32', name: '', type: 'bytes32' }], 18 | stateMutability: 'view', 19 | type: 'function', 20 | }, 21 | { 22 | inputs: [], 23 | name: 'baseStake', 24 | outputs: [{ internalType: 'uint256', name: '', type: 'uint256' }], 25 | stateMutability: 'view', 26 | type: 'function', 27 | }, 28 | ] as const 29 | 30 | export const ASSERTION_CREATED_EVENT = { 31 | ...parseAbiItem( 32 | 'event AssertionCreated(bytes32 indexed assertionHash, bytes32 indexed parentAssertionHash, ((bytes32 prevPrevAssertionHash, bytes32 sequencerBatchAcc, (bytes32 wasmModuleRoot, uint256 requiredStake, address challengeManager, uint64 confirmPeriodBlocks, uint64 nextInboxPosition)), ((bytes32[2] globalStateBytes32Vals, uint64[2] globalStateU64Vals), uint8 beforeStateMachineStatus, bytes32 beforeStateEndHistoryRoot), ((bytes32[2] globalStateBytes32Vals, uint64[2] globalStateU64Vals), uint8 afterStateMachineStatus, bytes32 afterStateEndHistoryRoot)) assertion, bytes32 afterInboxBatchAcc, uint256 inboxMaxCount, bytes32 wasmModuleRoot, uint256 requiredStake, address challengeManager, uint64 confirmPeriodBlocks)' 33 | ), 34 | name: 'AssertionCreated', 35 | type: 'event', 36 | } as const 37 | 38 | export const ASSERTION_CONFIRMED_EVENT = { 39 | ...parseAbiItem( 40 | 'event AssertionConfirmed(bytes32 indexed assertionHash, bytes32 blockHash, bytes32 sendRoot)' 41 | ), 42 | name: 'AssertionConfirmed', 43 | type: 'event', 44 | } as const 45 | 46 | export const NODE_CREATED_EVENT = { 47 | ...parseAbiItem( 48 | 'event NodeCreated(uint64 indexed nodeNum, bytes32 indexed parentNodeHash, bytes32 indexed nodeHash, bytes32 executionHash, (((bytes32[2] bytes32Vals, uint64[2] u64Vals) globalState, uint8 machineStatus) beforeState, ((bytes32[2] bytes32Vals, uint64[2] u64Vals) globalState, uint8 machineStatus) afterState, uint64 numBlocks) assertion, bytes32 afterInboxBatchAcc, bytes32 wasmModuleRoot, uint256 inboxMaxCount)' 49 | ), 50 | name: 'NodeCreated', 51 | type: 'event', 52 | } as const 53 | 54 | export const NODE_CONFIRMED_EVENT = { 55 | ...parseAbiItem( 56 | 'event NodeConfirmed(uint64 indexed nodeNum, bytes32 blockHash, bytes32 sendRoot)' 57 | ), 58 | name: 'NodeConfirmed', 59 | type: 'event', 60 | } as const 61 | -------------------------------------------------------------------------------- /packages/assertion-monitor/alerts.ts: -------------------------------------------------------------------------------- 1 | export const NO_CREATION_EVENTS_ALERT = `No assertion creation events found` 2 | 3 | export const CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT = `Chain activity detected but no assertions created recently` 4 | 5 | export const NO_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation events found` 6 | 7 | export const CONFIRMATION_DELAY_ALERT = `Confirmation period exceeded` 8 | 9 | export const CREATION_EVENT_STUCK_ALERT = `Assertion event stuck in challenge period` 10 | 11 | export const NON_BOLD_NO_RECENT_CREATION_ALERT = `No recent node creation events detected for non-BOLD chain` 12 | 13 | export const VALIDATOR_WHITELIST_DISABLED_ALERT = `Validator whitelist disabled - this may indicate security concerns for Classic chains` 14 | 15 | export const BOLD_LOW_BASE_STAKE_ALERT = `BoLD chain has low base stake (below 1 ETH) which may indicate restricted validation` 16 | 17 | export const NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation blocks found but confirmation events detected` 18 | -------------------------------------------------------------------------------- /packages/assertion-monitor/blockchain.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AbiEvent, 3 | PublicClient, 4 | createPublicClient, 5 | defineChain, 6 | getContract, 7 | http, 8 | type Block, 9 | type Log, 10 | } from 'viem' 11 | import { ChildNetwork as ChainInfo, sleep } from '../utils' 12 | import { 13 | ASSERTION_CONFIRMED_EVENT, 14 | ASSERTION_CREATED_EVENT, 15 | NODE_CONFIRMED_EVENT, 16 | NODE_CREATED_EVENT, 17 | boldABI, 18 | rollupABI, 19 | } from './abi' 20 | import { CHUNK_SIZE, MIN_BASE_STAKE_THRESHOLD } from './constants' 21 | import { AssertionDataError } from './errors' 22 | import { ChainState, ConfirmationEvent, CreationEvent } from './types' 23 | import { extractBoldBlockHash, extractClassicBlockHash } from './utils' 24 | 25 | /** 26 | * Queries the rollup contract to determine if the validator whitelist feature is disabled. 27 | * Controls which validators can post assertions in non-permissionless mode. 28 | */ 29 | export async function getValidatorWhitelistDisabled( 30 | client: PublicClient, 31 | rollupAddress: string 32 | ): Promise { 33 | const contract = getContract({ 34 | address: rollupAddress as `0x${string}`, 35 | abi: rollupABI, 36 | client, 37 | }) 38 | 39 | return contract.read.validatorWhitelistDisabled() 40 | } 41 | 42 | /** 43 | * Checks if the baseStake for a BoLD chain is below a threshold that would indicate 44 | * permissionless validation might be disabled or restricted. 45 | * A very low baseStake could indicate that validation is not intended to be permissionless. 46 | */ 47 | export async function fetchIsBaseStakeBelowThreshold( 48 | client: PublicClient, 49 | rollupAddress: string, 50 | isBold: boolean, 51 | thresholdInWei: bigint = MIN_BASE_STAKE_THRESHOLD 52 | ): Promise { 53 | if (!isBold) { 54 | return false 55 | } 56 | 57 | try { 58 | const contract = getContract({ 59 | address: rollupAddress as `0x${string}`, 60 | abi: boldABI, 61 | client, 62 | }) 63 | 64 | const baseStake = await contract.read.baseStake() 65 | console.log(`Base stake for rollup ${rollupAddress}: ${baseStake} wei`) 66 | return baseStake < thresholdInWei 67 | } catch (error) { 68 | console.error(`Error checking baseStake: ${error}`) 69 | // Default to false if we can't check 70 | return false 71 | } 72 | } 73 | 74 | /** 75 | * Retrieves the latest block number that has been processed by the assertion chain. 76 | * Uses block hash from assertion data to track L2/L3 state progression. 77 | */ 78 | export async function getLatestCreationBlock( 79 | childChainClient: PublicClient, 80 | latestCreationLog: CreationEvent | null, 81 | isBold: boolean 82 | ): Promise { 83 | if (!latestCreationLog) { 84 | throw new AssertionDataError('No assertion logs found') 85 | } 86 | 87 | const assertionData = latestCreationLog.args.assertion 88 | const lastProcessedBlockHash = isBold 89 | ? extractBoldBlockHash(assertionData) 90 | : extractClassicBlockHash(assertionData) 91 | 92 | const block = await childChainClient.getBlock({ 93 | blockHash: lastProcessedBlockHash, 94 | }) 95 | console.log(`Last processed child chain block: ${block.number}`) 96 | return block 97 | } 98 | 99 | /** 100 | * Gets the latest confirmed block number from assertion logs by finding the corresponding child block. 101 | * Returns undefined if no confirmed logs are found or if the corresponding block cannot be found. 102 | */ 103 | export async function getLatestConfirmedBlock( 104 | childChainClient: PublicClient, 105 | confirmationEvent: ConfirmationEvent | null 106 | ): Promise { 107 | try { 108 | let childLastConfirmedBlock 109 | if (confirmationEvent) { 110 | const lastConfirmedBlockhash = confirmationEvent.args.blockHash 111 | childLastConfirmedBlock = await childChainClient.getBlock({ 112 | blockHash: lastConfirmedBlockhash, 113 | }) 114 | } 115 | if (childLastConfirmedBlock) { 116 | console.log( 117 | 'Found confirmed child block:', 118 | childLastConfirmedBlock?.number 119 | ) 120 | return childLastConfirmedBlock 121 | } else { 122 | console.log('No confirmed child block found') 123 | return undefined 124 | } 125 | } catch (error) { 126 | console.error('Failed to get confirmed block from child chain:', error) 127 | return undefined 128 | } 129 | } 130 | 131 | /** 132 | * Determines if the rollup contract is using BOLD mode by checking for a genesis assertion hash. 133 | * BOLD mode uses a different assertion format and validation process than Classic mode. 134 | */ 135 | export async function isBoldEnabled( 136 | client: PublicClient, 137 | rollupAddress: string 138 | ): Promise { 139 | try { 140 | const contract = getContract({ 141 | address: rollupAddress as `0x${string}`, 142 | abi: boldABI, 143 | client, 144 | }) 145 | 146 | const genesisHash = await contract.read.genesisAssertionHash() 147 | return !!genesisHash 148 | } catch (error) { 149 | return false 150 | } 151 | } 152 | 153 | /** 154 | * Configures and creates a viem PublicClient instance for interacting with the child chain. 155 | * Used to monitor L2/L3 chain state and transaction activity. 156 | */ 157 | export function createChildChainClient( 158 | childChainInfo: ChainInfo 159 | ): PublicClient { 160 | const childChain = defineChain({ 161 | id: childChainInfo.chainId, 162 | name: childChainInfo.name, 163 | network: 'childChain', 164 | nativeCurrency: { 165 | name: 'ETH', 166 | symbol: 'ETH', 167 | decimals: 18, 168 | }, 169 | rpcUrls: { 170 | default: { 171 | http: [childChainInfo.orbitRpcUrl], 172 | }, 173 | public: { 174 | http: [childChainInfo.orbitRpcUrl], 175 | }, 176 | }, 177 | }) 178 | 179 | return createPublicClient({ 180 | chain: childChain, 181 | transport: http(childChainInfo.orbitRpcUrl), 182 | }) 183 | } 184 | 185 | /** 186 | * Generic function to fetch the most recent event of a specific type within a block range. 187 | * Uses exponential backoff and retries to ensure robustness. 188 | */ 189 | export async function fetchMostRecentEvent< 190 | T extends Log 191 | >( 192 | fromBlock: bigint, 193 | toBlock: bigint, 194 | client: PublicClient, 195 | rollupAddress: string, 196 | event: AbiEvent, 197 | chunkSize: bigint = CHUNK_SIZE, 198 | eventName?: string 199 | ): Promise { 200 | let currentToBlock = toBlock 201 | 202 | while (currentToBlock >= fromBlock) { 203 | const currentFromBlock = 204 | currentToBlock - chunkSize + 1n > fromBlock 205 | ? currentToBlock - chunkSize + 1n 206 | : fromBlock 207 | 208 | try { 209 | const logs = await client.getLogs({ 210 | address: rollupAddress as `0x${string}`, 211 | fromBlock: currentFromBlock, 212 | toBlock: currentToBlock, 213 | event, 214 | }) 215 | 216 | if (logs.length > 0) { 217 | const eventType = eventName || event.name || 'event' 218 | console.log( 219 | `Found ${eventType} in block range ${currentFromBlock} to ${currentToBlock}` 220 | ) 221 | // Return the most recent event (last in the array) 222 | return logs[logs.length - 1] as T 223 | } 224 | 225 | // If we've searched all blocks, stop 226 | if (currentFromBlock === fromBlock) break 227 | 228 | // Move to the next chunk 229 | currentToBlock = currentFromBlock - 1n 230 | 231 | // Add a small delay between chunks to avoid rate limiting 232 | await sleep(100) 233 | } catch (error) { 234 | console.error( 235 | `Error in fetchMostRecentEvent for ${eventName || event.name}:`, 236 | error 237 | ) 238 | // If we get an error, try a smaller chunk size 239 | if (chunkSize > 100n) { 240 | console.log(`Retrying with smaller chunk size: ${chunkSize / 2n}`) 241 | return fetchMostRecentEvent( 242 | currentFromBlock, 243 | currentToBlock, 244 | client, 245 | rollupAddress, 246 | event, 247 | chunkSize / 2n, 248 | eventName 249 | ) 250 | } 251 | throw error 252 | } 253 | } 254 | 255 | return null 256 | } 257 | 258 | /** 259 | * Fetches the most recent creation event (assertion or node) within a block range. 260 | * Uses exponential backoff and retries to ensure robustness. 261 | */ 262 | export async function fetchMostRecentCreationEvent( 263 | fromBlock: bigint, 264 | toBlock: bigint, 265 | client: PublicClient, 266 | rollupAddress: string, 267 | isBold: boolean, 268 | chunkSize: bigint = CHUNK_SIZE 269 | ): Promise { 270 | const event = isBold ? ASSERTION_CREATED_EVENT : NODE_CREATED_EVENT 271 | const eventName = isBold ? 'creation event' : 'node creation event' 272 | 273 | return fetchMostRecentEvent( 274 | fromBlock, 275 | toBlock, 276 | client, 277 | rollupAddress, 278 | event, 279 | chunkSize, 280 | eventName 281 | ) 282 | } 283 | 284 | /** 285 | * Fetches the most recent confirmation event (assertion or node) within a block range. 286 | * Uses exponential backoff and retries to ensure robustness. 287 | */ 288 | export async function fetchMostRecentConfirmationEvent< 289 | T extends ConfirmationEvent 290 | >( 291 | fromBlock: bigint, 292 | toBlock: bigint, 293 | client: PublicClient, 294 | rollupAddress: string, 295 | isBold: boolean, 296 | chunkSize: bigint = CHUNK_SIZE 297 | ): Promise { 298 | const event = isBold ? ASSERTION_CONFIRMED_EVENT : NODE_CONFIRMED_EVENT 299 | const eventName = isBold ? 'confirmation event' : 'node confirmation event' 300 | 301 | return fetchMostRecentEvent( 302 | fromBlock, 303 | toBlock, 304 | client, 305 | rollupAddress, 306 | event, 307 | chunkSize, 308 | eventName 309 | ) 310 | } 311 | 312 | /** 313 | * Fetches the latest blocks and events to build the `ChainState` object 314 | */ 315 | export const fetchChainState = async ({ 316 | childChainClient, 317 | parentClient, 318 | childChainInfo, 319 | isBold, 320 | fromBlock, 321 | toBlock, 322 | }: { 323 | childChainClient: PublicClient 324 | parentClient: PublicClient 325 | childChainInfo: ChainInfo 326 | isBold: boolean 327 | fromBlock: bigint 328 | toBlock: bigint 329 | }): Promise => { 330 | const childCurrentBlock = await childChainClient.getBlock({ 331 | blockTag: 'latest', 332 | }) 333 | 334 | const parentCurrentBlock = await parentClient.getBlock({ 335 | blockTag: 'latest', 336 | }) 337 | 338 | const recentCreationEvent = await fetchMostRecentCreationEvent( 339 | fromBlock, 340 | toBlock, 341 | parentClient, 342 | childChainInfo.ethBridge.rollup, 343 | isBold 344 | ) 345 | 346 | const recentConfirmationEvent = await fetchMostRecentConfirmationEvent( 347 | fromBlock, 348 | toBlock, 349 | parentClient, 350 | childChainInfo.ethBridge.rollup, 351 | isBold 352 | ) 353 | 354 | const childLatestConfirmedBlock = await getLatestConfirmedBlock( 355 | childChainClient, 356 | recentConfirmationEvent 357 | ) 358 | 359 | const childLatestCreatedBlock = await getLatestCreationBlock( 360 | childChainClient, 361 | recentCreationEvent, 362 | isBold 363 | ) 364 | 365 | // Get parent blocks at creation and confirmation 366 | let parentBlockAtCreation 367 | if (recentCreationEvent) { 368 | parentBlockAtCreation = await parentClient.getBlock({ 369 | blockNumber: recentCreationEvent.blockNumber, 370 | }) 371 | } 372 | 373 | let parentBlockAtConfirmation 374 | if (recentConfirmationEvent) { 375 | parentBlockAtConfirmation = await parentClient.getBlock({ 376 | blockNumber: recentConfirmationEvent.blockNumber, 377 | }) 378 | } 379 | 380 | const isValidatorWhitelistDisabled = await getValidatorWhitelistDisabled( 381 | parentClient, 382 | childChainInfo.ethBridge.rollup 383 | ) 384 | const isBaseStakeBelowThreshold = await fetchIsBaseStakeBelowThreshold( 385 | parentClient, 386 | childChainInfo.ethBridge.rollup, 387 | isBold 388 | ) 389 | 390 | const chainState: ChainState = { 391 | childCurrentBlock, 392 | childLatestCreatedBlock, 393 | childLatestConfirmedBlock, 394 | parentCurrentBlock, 395 | parentBlockAtCreation, 396 | parentBlockAtConfirmation, 397 | recentCreationEvent, 398 | recentConfirmationEvent, 399 | isValidatorWhitelistDisabled, 400 | isBaseStakeBelowThreshold, 401 | } 402 | 403 | console.log('Built chain state blocks:', { 404 | childCurrentBlock: childCurrentBlock.number, 405 | childLatestCreatedBlock: childLatestCreatedBlock?.number, 406 | childLatestConfirmedBlock: childLatestConfirmedBlock?.number, 407 | parentCurrentBlock: parentCurrentBlock.number, 408 | parentBlockAtCreation: parentBlockAtCreation?.number, 409 | parentBlockAtConfirmation: parentBlockAtConfirmation?.number, 410 | }) 411 | 412 | return chainState 413 | } 414 | -------------------------------------------------------------------------------- /packages/assertion-monitor/chains.ts: -------------------------------------------------------------------------------- 1 | import { Chain } from 'viem' 2 | import { 3 | mainnet, 4 | arbitrum, 5 | arbitrumNova, 6 | base, 7 | sepolia, 8 | holesky, 9 | arbitrumSepolia, 10 | baseSepolia, 11 | } from 'viem/chains' 12 | 13 | export const supportedParentChains = [ 14 | mainnet, 15 | arbitrum, 16 | arbitrumNova, 17 | base, 18 | sepolia, 19 | holesky, 20 | arbitrumSepolia, 21 | baseSepolia, 22 | ] 23 | 24 | export const getChainFromId = (chainId: number): Chain => { 25 | const chain = supportedParentChains.filter(chain => chain.id === chainId) 26 | return chain[0] ?? null 27 | } 28 | 29 | export const getBlockTimeForChain = (chain: Chain): number => { 30 | switch (chain) { 31 | case mainnet: 32 | case sepolia: 33 | case holesky: 34 | return 12 35 | 36 | case base: 37 | case baseSepolia: 38 | return 2 39 | 40 | case arbitrum: 41 | case arbitrumNova: 42 | case arbitrumSepolia: 43 | return 0.25 44 | 45 | default: 46 | return 1 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /packages/assertion-monitor/constants.ts: -------------------------------------------------------------------------------- 1 | /** Maximum number of days to look back when scanning for assertions */ 2 | export const MAXIMUM_SEARCH_DAYS = 7 3 | 4 | /** Number of hours to check for recent creation events */ 5 | export const RECENT_CREATION_CHECK_HOURS = 4 6 | 7 | /** Number of hours to consider an event "recent" for confirmation checks */ 8 | export const RECENT_EVENT_HOURS = 24 9 | 10 | /** Maximum number of blocks a validator can be inactive before alerts are triggered */ 11 | export const VALIDATOR_AFK_BLOCKS = 45818 12 | 13 | /** Buffer period in days to avoid scanning too close to the current block */ 14 | export const SAFETY_BUFFER_DAYS = 4 15 | 16 | /** Number of blocks to process in each chunk when fetching logs to avoid RPC timeouts */ 17 | export const CHUNK_SIZE = 800n 18 | 19 | /** Number of seconds in a day */ 20 | export const SECONDS_IN_A_DAY = 24 * 60 * 60 21 | 22 | /** Challenge period in seconds (6.4 days) */ 23 | export const CHALLENGE_PERIOD_SECONDS = 6.4 * SECONDS_IN_A_DAY 24 | 25 | /** Search window in seconds (7 days) */ 26 | export const SEARCH_WINDOW_SECONDS = MAXIMUM_SEARCH_DAYS * SECONDS_IN_A_DAY 27 | 28 | /** Recent activity threshold in seconds (4 hours) */ 29 | export const RECENT_ACTIVITY_SECONDS = RECENT_CREATION_CHECK_HOURS * 60 * 60 30 | 31 | /** Minimum base stake threshold for BoLD chains (1 ETH in wei) */ 32 | export const MIN_BASE_STAKE_THRESHOLD = BigInt('1000000000000000000') 33 | 34 | /** Convert hours to seconds for timestamp comparison */ 35 | export const hoursToSeconds = (hours: number) => hours * 60 * 60 -------------------------------------------------------------------------------- /packages/assertion-monitor/errors.ts: -------------------------------------------------------------------------------- 1 | export class AssertionDataError extends Error { 2 | constructor(message: string, public readonly rawData?: any) { 3 | super(message) 4 | this.name = 'AssertionDataError' 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/assertion-monitor/index.ts: -------------------------------------------------------------------------------- 1 | import { PublicClient, createPublicClient, http } from 'viem' 2 | import yargs from 'yargs' 3 | import { 4 | ChildNetwork as ChainInfo, 5 | DEFAULT_CONFIG_PATH, 6 | getConfig, 7 | } from '../utils' 8 | import { 9 | createChildChainClient, 10 | fetchChainState, 11 | isBoldEnabled, 12 | } from './blockchain' 13 | import { getBlockTimeForChain, getChainFromId } from './chains' 14 | import { 15 | MAXIMUM_SEARCH_DAYS, 16 | SAFETY_BUFFER_DAYS, 17 | VALIDATOR_AFK_BLOCKS, 18 | } from './constants' 19 | import { analyzeAssertionEvents } from './monitoring' 20 | import { reportAssertionMonitorErrorToSlack } from './reportAssertionMonitorAlertToSlack' 21 | import { BlockRange } from './types' 22 | 23 | /** Retrieves and validates the monitor configuration from the config file. */ 24 | export const getMonitorConfig = (configPath: string = DEFAULT_CONFIG_PATH) => { 25 | const options = yargs(process.argv.slice(2)) 26 | .options({ 27 | configPath: { type: 'string', default: configPath }, 28 | enableAlerting: { type: 'boolean', default: false }, 29 | writeToNotion: { type: 'boolean', default: false }, 30 | }) 31 | .strict() 32 | .parseSync() 33 | 34 | const config = getConfig(options) 35 | 36 | if (!Array.isArray(config.childChains) || config.childChains.length === 0) { 37 | throw new Error('Error: Chains not found in the config file.') 38 | } 39 | 40 | return { config, options } 41 | } 42 | 43 | /** Calculates the appropriate block window for monitoring assertions based on chain characteristics. */ 44 | function calculateSearchWindow( 45 | childChainInfo: ChainInfo, 46 | parentChain: ReturnType 47 | ): { days: number; blocks: number } { 48 | const blockTime = getBlockTimeForChain(parentChain) 49 | 50 | // Return zero days and blocks if block time is zero to avoid division by zero 51 | if (blockTime === 0) { 52 | return { 53 | days: 0, 54 | blocks: 0, 55 | } 56 | } 57 | 58 | const initialBlocksToSearch = 59 | childChainInfo.confirmPeriodBlocks + VALIDATOR_AFK_BLOCKS 60 | const timespan = blockTime * initialBlocksToSearch 61 | 62 | const blocksInDays = timespan / (60 * 60 * 24) 63 | const blocksInDaysMinusSafety = Math.max(blocksInDays - SAFETY_BUFFER_DAYS, 0) 64 | const daysAdjustedForMax = Math.min( 65 | Math.ceil(blocksInDaysMinusSafety), 66 | MAXIMUM_SEARCH_DAYS 67 | ) 68 | 69 | // Calculate the maximum number of blocks for the maximum search days 70 | const maxSearchableBlocks = Math.floor( 71 | (MAXIMUM_SEARCH_DAYS * 24 * 60 * 60) / blockTime 72 | ) 73 | 74 | // Adjust blocks to the maximum of 7 days 75 | const adjustedBlocks = Math.min(initialBlocksToSearch, maxSearchableBlocks) 76 | 77 | return { 78 | days: daysAdjustedForMax, 79 | blocks: adjustedBlocks, 80 | } 81 | } 82 | 83 | /** 84 | * Determines the block range to scan for assertions based on chain configuration. 85 | * Uses chain-specific parameters to calculate an appropriate range that covers potential confirmation delays. 86 | */ 87 | export const getBlockRange = async ( 88 | client: PublicClient, 89 | childChainInfo: ChainInfo 90 | ) => { 91 | const latestBlockNumber = await client.getBlockNumber() 92 | const parentChain = getChainFromId(childChainInfo.parentChainId) 93 | const { blocks: blockRange } = calculateSearchWindow( 94 | childChainInfo, 95 | parentChain 96 | ) 97 | 98 | const fromBlock = await client.getBlock({ 99 | blockNumber: latestBlockNumber - BigInt(blockRange), 100 | }) 101 | 102 | return { fromBlock: fromBlock.number, toBlock: latestBlockNumber } 103 | } 104 | 105 | /** 106 | * Main monitoring function for a single chain's assertion health. 107 | * Follows a specific flow to analyze chain activity and assertion health. 108 | */ 109 | export const checkChainForAssertionIssues = async ( 110 | childChainInfo: ChainInfo, 111 | blockRange?: BlockRange, 112 | options?: { enableAlerting: boolean } 113 | ) => { 114 | console.log(`\nMonitoring ${childChainInfo.name}...`) 115 | 116 | const parentChain = getChainFromId(childChainInfo.parentChainId) 117 | const parentClient = createPublicClient({ 118 | chain: parentChain, 119 | transport: http(childChainInfo.parentRpcUrl), 120 | }) 121 | 122 | const isBold = await isBoldEnabled( 123 | parentClient, 124 | childChainInfo.ethBridge.rollup 125 | ) 126 | console.log(`Chain type: ${isBold ? 'BoLD' : 'Classic'} rollup`) 127 | 128 | const { fromBlock, toBlock } = 129 | blockRange || (await getBlockRange(parentClient, childChainInfo)) 130 | console.log( 131 | `Scanning blocks ${fromBlock} to ${toBlock} (${toBlock - fromBlock} blocks)` 132 | ) 133 | 134 | const childChainClient = createChildChainClient(childChainInfo) 135 | 136 | const chainState = await fetchChainState({ 137 | childChainClient, 138 | parentClient, 139 | childChainInfo, 140 | isBold, 141 | fromBlock, 142 | toBlock, 143 | }) 144 | 145 | const alerts = await analyzeAssertionEvents( 146 | chainState, 147 | childChainInfo, 148 | isBold 149 | ) 150 | if (alerts.length > 0) { 151 | console.log(`Generated ${alerts.length} alerts for ${childChainInfo.name}`) 152 | return `${childChainInfo.name}:\n- ${alerts.join('\n- ')}` 153 | } else { 154 | console.log(`No issues found for ${childChainInfo.name}`) 155 | } 156 | return 157 | } 158 | 159 | /** 160 | * Entry point for the assertion monitoring system. 161 | * Reports issues to Slack when alerting is enabled. 162 | */ 163 | export const main = async () => { 164 | try { 165 | const { config, options } = getMonitorConfig() 166 | const alerts: string[] = [] 167 | console.log('Starting assertion monitoring...') 168 | 169 | for (const chainInfo of config.childChains) { 170 | const result = await checkChainForAssertionIssues(chainInfo) 171 | if (result) { 172 | alerts.push(result) 173 | } 174 | } 175 | 176 | if (alerts.length > 0) { 177 | const alertMessage = `Assertion Monitor Alert Summary:\n\n${alerts}` 178 | console.log(alertMessage) 179 | 180 | if (options.enableAlerting) { 181 | console.log('Sending alerts to Slack...') 182 | await reportAssertionMonitorErrorToSlack({ message: alertMessage }) 183 | } 184 | } else { 185 | console.log('\nMonitoring complete - all chains healthy') 186 | } 187 | } catch (error: unknown) { 188 | const errorMessage = error instanceof Error ? error.message : String(error) 189 | const errorStr = `Error processing chain data for assertion monitoring: ${errorMessage}` 190 | const { options } = getMonitorConfig() 191 | if (options.enableAlerting) { 192 | reportAssertionMonitorErrorToSlack({ message: errorStr }) 193 | } 194 | console.error(errorStr) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /packages/assertion-monitor/main.ts: -------------------------------------------------------------------------------- 1 | import { main } from './index' 2 | 3 | main() 4 | .then(() => process.exit(0)) 5 | .catch(error => { 6 | console.error(error) 7 | process.exit(1) 8 | }) -------------------------------------------------------------------------------- /packages/assertion-monitor/monitoring.ts: -------------------------------------------------------------------------------- 1 | import { ChildNetwork as ChainInfo } from '../utils' 2 | import { 3 | BOLD_LOW_BASE_STAKE_ALERT, 4 | CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT, 5 | CONFIRMATION_DELAY_ALERT, 6 | CREATION_EVENT_STUCK_ALERT, 7 | NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT, 8 | NO_CONFIRMATION_EVENTS_ALERT, 9 | NO_CREATION_EVENTS_ALERT, 10 | NON_BOLD_NO_RECENT_CREATION_ALERT, 11 | VALIDATOR_WHITELIST_DISABLED_ALERT, 12 | } from './alerts' 13 | import { 14 | CHALLENGE_PERIOD_SECONDS, 15 | RECENT_ACTIVITY_SECONDS, 16 | VALIDATOR_AFK_BLOCKS, 17 | } from './constants' 18 | import type { ChainState } from './types' 19 | import { isEventRecent } from './utils' 20 | 21 | /** 22 | * Analyzes chain state to detect assertion and confirmation issues 23 | * 24 | * Evaluates BOLD chains for: 25 | * - Challenge/confirmation system health 26 | * - Validator activity and challenge detection 27 | * - Bounded finality guarantee issues 28 | * - Whitelist security concerns 29 | * 30 | * Evaluates Classic chains with adjusted thresholds for: 31 | * - Basic validator activity 32 | * - Confirmation patterns 33 | * 34 | */ 35 | export const analyzeAssertionEvents = async ( 36 | chainState: ChainState, 37 | chainInfo: ChainInfo, 38 | isBold: boolean = true 39 | ): Promise => { 40 | const alerts: string[] = [] 41 | 42 | const { 43 | doesLatestChildCreatedBlockExist, 44 | doesLatestChildConfirmedBlockExist, 45 | hasActivityWithoutRecentAssertions, 46 | noConfirmationsWithCreationEvents, 47 | noConfirmedBlocksWithConfirmationEvents, 48 | confirmationDelayExceedsPeriod, 49 | creationEventStuckInChallengePeriod, 50 | nonBoldMissingRecentCreation, 51 | isValidatorWhitelistDisabledOnClassic, 52 | isBaseStakeBelowThresholdOnBold, 53 | } = generateConditionsForAlerts(chainInfo, chainState, isBold) 54 | 55 | if (isValidatorWhitelistDisabledOnClassic) { 56 | alerts.push(VALIDATOR_WHITELIST_DISABLED_ALERT) 57 | } 58 | 59 | if (isBaseStakeBelowThresholdOnBold) { 60 | alerts.push(BOLD_LOW_BASE_STAKE_ALERT) 61 | } 62 | 63 | if (!doesLatestChildCreatedBlockExist) { 64 | alerts.push(NO_CREATION_EVENTS_ALERT) 65 | } 66 | 67 | if (!doesLatestChildConfirmedBlockExist) { 68 | alerts.push(NO_CONFIRMATION_EVENTS_ALERT) 69 | } 70 | 71 | if (noConfirmedBlocksWithConfirmationEvents) { 72 | alerts.push(NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT) 73 | } 74 | 75 | if (hasActivityWithoutRecentAssertions) { 76 | alerts.push(CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT) 77 | } 78 | 79 | if (noConfirmationsWithCreationEvents) { 80 | alerts.push(NO_CONFIRMATION_EVENTS_ALERT) 81 | } 82 | 83 | if (confirmationDelayExceedsPeriod) { 84 | alerts.push(CONFIRMATION_DELAY_ALERT) 85 | } 86 | 87 | if (creationEventStuckInChallengePeriod) { 88 | alerts.push(CREATION_EVENT_STUCK_ALERT) 89 | } 90 | 91 | if (nonBoldMissingRecentCreation) { 92 | alerts.push(NON_BOLD_NO_RECENT_CREATION_ALERT) 93 | } 94 | 95 | return alerts 96 | } 97 | 98 | /** 99 | * Generates boolean conditions for chain health alerts 100 | * 101 | * @throws If essential chain state data is missing 102 | */ 103 | export const generateConditionsForAlerts = ( 104 | chainInfo: ChainInfo, 105 | chainState: ChainState, 106 | isBold: boolean 107 | ) => { 108 | const currentTimestamp = BigInt(Date.now()) 109 | const currentTimeSeconds = Number(currentTimestamp / 1000n) 110 | 111 | const { 112 | childCurrentBlock, 113 | childLatestCreatedBlock, 114 | childLatestConfirmedBlock, 115 | parentCurrentBlock, 116 | parentBlockAtConfirmation, 117 | recentCreationEvent, 118 | recentConfirmationEvent, 119 | } = chainState 120 | 121 | /** 122 | * Critical for both chain types as assertions are fundamental to the rollup mechanism 123 | * No assertions indicates severe validator issues or extreme chain inactivity 124 | */ 125 | const doesLatestChildCreatedBlockExist = !!childLatestCreatedBlock 126 | 127 | /** 128 | * For BOLD: Critical for bounded finality guarantees 129 | * For Classic: Indicates active validation 130 | * 131 | * Always compare with current timestamp, not child chain latest block timestamp 132 | */ 133 | const hasRecentCreationEvents = 134 | childLatestCreatedBlock && 135 | isEventRecent( 136 | childLatestCreatedBlock.timestamp, 137 | currentTimestamp / 1000n, 138 | RECENT_ACTIVITY_SECONDS 139 | ) 140 | 141 | /** 142 | * Missing confirmations may indicate challenge period in progress or 143 | * may be normal for low-activity chains where no assertions need confirmation yet 144 | */ 145 | const doesLatestChildConfirmedBlockExist = !!childLatestConfirmedBlock 146 | 147 | /** 148 | * Detects transaction processing in child chain not yet asserted in parent chain 149 | * Normal in small amounts due to batching, concerning in large amounts 150 | */ 151 | const hasActivityWithoutAssertions = 152 | childCurrentBlock?.number && 153 | childLatestCreatedBlock?.number && 154 | childCurrentBlock.number > childLatestCreatedBlock.number 155 | 156 | /** 157 | * Critical for BOLD due to finality implications 158 | * Indicates validator issues for both chain types 159 | */ 160 | const hasActivityWithoutRecentAssertions = 161 | hasActivityWithoutAssertions && !hasRecentCreationEvents 162 | 163 | /** 164 | * May indicate active challenges or technical issues with confirmation 165 | * Could also be normal in low-activity chains where assertions are waiting for challenge period 166 | */ 167 | const noConfirmationsWithCreationEvents = 168 | doesLatestChildCreatedBlockExist && !doesLatestChildConfirmedBlockExist 169 | 170 | /** 171 | * Detects an inconsistent state where confirmation events exist but no confirmed blocks are recorded 172 | * Indicates a technical issue with confirmation processing or data synchronization 173 | * This should not occur in normal operation and requires investigation 174 | */ 175 | const noConfirmedBlocksWithConfirmationEvents = 176 | recentConfirmationEvent && !doesLatestChildConfirmedBlockExist 177 | 178 | /** 179 | * Parent chain block gap since last confirmation 180 | * This is the direct, accurate measure of confirmation delay in terms of parent chain blocks 181 | */ 182 | const parentBlocksSinceLastConfirmation = 183 | (parentCurrentBlock?.number && 184 | parentBlockAtConfirmation?.number && 185 | parentCurrentBlock.number - parentBlockAtConfirmation.number) ?? 186 | 0n 187 | 188 | /** 189 | * Confirmation threshold in parent chain blocks 190 | * This is the exact, chain-appropriate threshold without any approximation multipliers 191 | */ 192 | const parentConfirmationThreshold = BigInt( 193 | chainInfo.confirmPeriodBlocks + VALIDATOR_AFK_BLOCKS 194 | ) 195 | 196 | /** 197 | * Confirmation delay check using direct parent chain comparison 198 | * We now assume parent chain data is always available 199 | */ 200 | const confirmationDelayExceedsPeriod = 201 | parentBlocksSinceLastConfirmation > parentConfirmationThreshold 202 | 203 | /** 204 | * Identifies assertions exceeding challenge period (6.4 days) without confirmation 205 | * Indicates active challenges or confirmation problems 206 | */ 207 | const creationEventStuckInChallengePeriod = 208 | isBold && 209 | childLatestCreatedBlock && 210 | childLatestCreatedBlock?.timestamp && 211 | childLatestCreatedBlock.timestamp < 212 | BigInt(currentTimeSeconds - CHALLENGE_PERIOD_SECONDS) 213 | 214 | /** 215 | * Only alerts when activity exists without assertions 216 | * May be normal for low-activity chains, hence contextual consideration required 217 | */ 218 | const nonBoldMissingRecentCreation = 219 | !isBold && 220 | (!childLatestCreatedBlock || 221 | (!hasRecentCreationEvents && hasActivityWithoutAssertions)) 222 | 223 | /** 224 | * Whether a Classic chain's validator whitelist is disabled, allowing 225 | * unauthorized validators to post assertions. 226 | */ 227 | const isValidatorWhitelistDisabledOnClassic = 228 | !isBold && chainState.isValidatorWhitelistDisabled 229 | 230 | /** 231 | * Whether a BoLD chain's base stake is below threshold, indicating restricted 232 | * validator participation in dispute resolution. 233 | */ 234 | const isBaseStakeBelowThresholdOnBold = 235 | isBold && 236 | chainState.isBaseStakeBelowThreshold && 237 | chainState.isValidatorWhitelistDisabled 238 | 239 | return { 240 | doesLatestChildCreatedBlockExist, 241 | doesLatestChildConfirmedBlockExist, 242 | hasRecentCreationEvents, 243 | hasActivityWithoutRecentAssertions, 244 | noConfirmationsWithCreationEvents, 245 | noConfirmedBlocksWithConfirmationEvents, 246 | confirmationDelayExceedsPeriod, 247 | creationEventStuckInChallengePeriod, 248 | nonBoldMissingRecentCreation, 249 | isValidatorWhitelistDisabledOnClassic, 250 | isBaseStakeBelowThresholdOnBold, 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /packages/assertion-monitor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "assertion-monitor", 3 | "version": "1.0.0", 4 | "main": "dist/main.js", 5 | "dependencies": { 6 | "@arbitrum/sdk": "^4.0.0", 7 | "@types/yargs": "17.0.32", 8 | "typescript": "^5.4.5", 9 | "utils": "*", 10 | "viem": "^2.7.9", 11 | "yargs": "17.7.2" 12 | }, 13 | "devDependencies": { 14 | "@types/node": "^20.14.1", 15 | "ts-node": "^10.9.2", 16 | "vitest": "^3.0.5" 17 | }, 18 | "scripts": { 19 | "lint": "eslint .", 20 | "build": "rm -rf ./dist && tsc", 21 | "format": "prettier './**/*.{js,json,md,yml,sol,ts}' --write && yarn run lint --fix", 22 | "dev": "yarn build && node ./dist/assertion-monitor/main.js", 23 | "test": "vitest run __test__/*.test.ts", 24 | "test:watch": "vitest watch __test__/*.test.ts" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /packages/assertion-monitor/reportAssertionMonitorAlertToSlack.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | import { postSlackMessage } from '../utils/postSlackMessage' 3 | 4 | dotenv.config() 5 | 6 | const slackToken = process.env.ASSERTION_MONITORING_SLACK_TOKEN 7 | const slackChannel = process.env.ASSERTION_MONITORING_SLACK_CHANNEL 8 | 9 | export const reportAssertionMonitorErrorToSlack = ({ 10 | message, 11 | }: { 12 | message: string 13 | }) => { 14 | if (!slackToken) throw new Error(`Slack token is required.`) 15 | if (!slackChannel) throw new Error(`Slack channel is required.`) 16 | 17 | if (process.env.NODE_ENV === 'DEV') return 18 | if (process.env.NODE_ENV === 'CI' && message === 'success') return 19 | 20 | console.log(`>>> Reporting message to Slack -> ${message}`) 21 | 22 | return postSlackMessage({ 23 | slackToken, 24 | slackChannel, 25 | message, 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /packages/assertion-monitor/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "types": ["node", "vitest"] 6 | }, 7 | "include": ["./**/*.ts", "./**/*.d.ts", "packages/*.ts"], 8 | "exclude": ["node_modules", "dist/"] 9 | } 10 | -------------------------------------------------------------------------------- /packages/assertion-monitor/types.ts: -------------------------------------------------------------------------------- 1 | import { Block, Log } from 'viem' 2 | import { 3 | ASSERTION_CONFIRMED_EVENT, 4 | ASSERTION_CREATED_EVENT, 5 | NODE_CONFIRMED_EVENT, 6 | NODE_CREATED_EVENT, 7 | } from './abi' 8 | 9 | export interface BlockRange { 10 | fromBlock: bigint 11 | toBlock: bigint 12 | } 13 | 14 | export interface AssertionLogs { 15 | createdLogs: Log[] 16 | confirmedLogs: Log[] 17 | } 18 | 19 | /** Type for assertion/node creation events */ 20 | export type CreationEvent = Log< 21 | bigint, 22 | number, 23 | false, 24 | typeof ASSERTION_CREATED_EVENT | typeof NODE_CREATED_EVENT, 25 | true 26 | > & { 27 | args: { 28 | assertionHash: `0x${string}` 29 | parentAssertionHash: `0x${string}` 30 | assertion: { 31 | wasmModuleRoot: `0x${string}` 32 | requiredStake: bigint 33 | challengeManager: `0x${string}` 34 | confirmPeriodBlocks: bigint 35 | } 36 | } 37 | } 38 | 39 | /** Type for assertion/node confirmation events */ 40 | export type ConfirmationEvent = Log< 41 | bigint, 42 | number, 43 | false, 44 | typeof ASSERTION_CONFIRMED_EVENT | typeof NODE_CONFIRMED_EVENT, 45 | true 46 | > & { 47 | args: { 48 | blockHash: `0x${string}` 49 | } 50 | } 51 | 52 | /** Chain state information needed for monitoring */ 53 | export interface ChainState { 54 | childCurrentBlock: Block 55 | childLatestCreatedBlock?: Block 56 | childLatestConfirmedBlock?: Block 57 | parentCurrentBlock?: Block 58 | parentBlockAtCreation?: Block 59 | parentBlockAtConfirmation?: Block 60 | recentCreationEvent: CreationEvent | null 61 | recentConfirmationEvent: ConfirmationEvent | null 62 | isValidatorWhitelistDisabled: boolean 63 | isBaseStakeBelowThreshold: boolean 64 | } 65 | -------------------------------------------------------------------------------- /packages/assertion-monitor/utils.ts: -------------------------------------------------------------------------------- 1 | import { AssertionDataError } from './errors' 2 | 3 | export const jsonStringifyWithBigInt = (obj: any): string => 4 | JSON.stringify( 5 | obj, 6 | (_, value) => (typeof value === 'bigint' ? value.toString() : value), 7 | 2 8 | ) 9 | 10 | export function extractBoldBlockHash(assertionData: any): `0x${string}` { 11 | if (!assertionData?.[2]?.[0]?.globalStateBytes32Vals?.[0]) { 12 | throw new AssertionDataError( 13 | 'Incomplete BOLD assertion data structure', 14 | assertionData 15 | ) 16 | } 17 | return assertionData[2][0].globalStateBytes32Vals[0] 18 | } 19 | 20 | export function extractClassicBlockHash(assertionData: any): `0x${string}` { 21 | if (!assertionData?.afterState?.globalState?.bytes32Vals?.[0]) { 22 | throw new AssertionDataError( 23 | 'Incomplete Classic assertion data structure', 24 | assertionData 25 | ) 26 | } 27 | return assertionData.afterState.globalState.bytes32Vals[0] 28 | } 29 | 30 | /** 31 | * Checks if an event is within a specific time window in seconds 32 | */ 33 | export function isEventRecent( 34 | eventTimestamp: bigint, 35 | currentTimestamp: bigint, 36 | secondsThreshold: number 37 | ): boolean { 38 | const timeSinceEvent = Number(currentTimestamp - eventTimestamp) 39 | return timeSinceEvent <= secondsThreshold 40 | } 41 | -------------------------------------------------------------------------------- /packages/assertion-monitor/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config' 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | environment: 'node', 7 | }, 8 | }) -------------------------------------------------------------------------------- /packages/batch-poster-monitor/README.md: -------------------------------------------------------------------------------- 1 | # Batch Poster Monitor 2 | 3 | > For installation and general configuration, see the [main README](../../README.md). 4 | 5 | ## Overview 6 | 7 | The Batch Poster Monitor tracks batch posting performance, data compression, and poster account balance. 8 | 9 | ## Command-Line Interface 10 | 11 | ```bash 12 | yarn batch-poster-monitor [options] 13 | 14 | Monitor batch posting activity on Arbitrum chains 15 | 16 | Options: 17 | --help Show help [boolean] 18 | --version Show version number [boolean] 19 | --configPath Path to config file [string] [default: "config.json"] 20 | --enableAlerting Enable Slack alerts [boolean] [default: false] 21 | 22 | Examples: 23 | yarn batch-poster-monitor Run with default config 24 | yarn batch-poster-monitor --enableAlerting Enable Slack notifications 25 | yarn batch-poster-monitor --configPath=custom.json Use custom config file 26 | 27 | Environment Variables: 28 | BATCH_POSTER_MONITORING_SLACK_TOKEN Slack API token for alerts 29 | BATCH_POSTER_MONITORING_SLACK_CHANNEL Slack channel for alerts 30 | ``` 31 | 32 | ## Monitor Details 33 | 34 | The Batch Poster Monitor is crucial for ensuring that transaction data is reliably posted to the parent chain, maintaining the chain's data availability guarantees. It monitors the batch posting process, which involves compressing transaction data and submitting it to the parent chain, while also tracking the batch poster's account balance to ensure uninterrupted operation. 35 | 36 | ### Critical Events 37 | 38 | The monitor tracks several key metrics and events: 39 | 40 | - Batch posting frequency and timing 41 | - Data compression ratios 42 | - Poster account balance and gas costs 43 | - AnyTrust committee participation 44 | - Backlog accumulation 45 | 46 | ### Alert Scenarios 47 | 48 | The monitor generates alerts in these critical scenarios: 49 | 50 | #### Batch Posting Delays 51 | - No batches posted within 24 hours AND pending user transactions exist 52 | - Time since last batch post exceeds chain-specific time bounds: 53 | - Default: 4 hours 54 | - Customized based on `maxTimeVariation` contract setting 55 | - Minimum: 1 hour 56 | - Maximum: Chain's time bounds minus buffer 57 | 58 | #### Batch Poster Balance 59 | - Current balance falls below minimum required for 3 days of operation 60 | - Calculation based on: 61 | - Rolling 24-hour gas cost average 62 | - Current balance / daily posting cost estimate 63 | - Fallback threshold: Static check if no recent activity 64 | 65 | #### AnyTrust-Specific 66 | - Committee failure detection: 67 | - Chain reverted to posting full calldata on-chain (0x00 prefix) 68 | - Indicates potential Data Availability Committee issues 69 | - Expected format: DACert (0x88 prefix) 70 | 71 | #### Backlog Detection 72 | - Non-zero block backlog AND last batch posted > time bounds ago 73 | - Backlog measured as: `latestChildChainBlockNumber - lastBlockReported` 74 | 75 | #### Processing Errors 76 | - Chain RPC connectivity issues 77 | - Contract interaction failures 78 | - Invalid response formats 79 | - Rate limiting errors 80 | 81 | Each alert includes: 82 | - Chain name and ID 83 | - Relevant contract addresses 84 | - Specific threshold violations 85 | - Time since last successful operation 86 | - Current system state metrics 87 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/chains.ts: -------------------------------------------------------------------------------- 1 | import { ChildNetwork } from 'utils' 2 | import { Chain } from 'viem' 3 | import { 4 | mainnet, 5 | arbitrum, 6 | arbitrumNova, 7 | base, 8 | sepolia, 9 | holesky, 10 | arbitrumSepolia, 11 | baseSepolia, 12 | } from 'viem/chains' 13 | 14 | // Defaults 15 | export const MAX_TIMEBOUNDS_SECONDS = 60 * 60 * 24 // 24 hours : we don't care about txs older than 24 hours 16 | export const BATCH_POSTING_TIMEBOUNDS_FALLBACK = 60 * 60 * 12 // fallback in case we can't derive the on-chain timebounds for batch posting 17 | export const BATCH_POSTING_TIMEBOUNDS_BUFFER = 60 * 60 * 7 // reduce buffer (secs) from the time bounds for proactive alerting 18 | export const MIN_DAYS_OF_BALANCE_LEFT = 3n // Number of days the batch-poster balance must last, else alert 19 | export const MAX_LOGS_TO_PROCESS_FOR_BALANCE = 50 // Number of logs to process for batch poster balance estimation, there can be 1000+ logs for high activity chains 20 | export const BATCH_POSTER_BALANCE_ALERT_THRESHOLD_FALLBACK = 0.1 // (ETH) Fallback if dynamic balance calculation doesn't go through 21 | 22 | export const supportedParentChains = [ 23 | mainnet, 24 | arbitrum, 25 | arbitrumNova, 26 | base, 27 | 28 | sepolia, 29 | holesky, 30 | arbitrumSepolia, 31 | baseSepolia, 32 | ] 33 | 34 | export const supportedCoreChainIds: number[] = supportedParentChains.map( 35 | chain => chain.id 36 | ) 37 | 38 | export const getChainFromId = (chainId: number): Chain => { 39 | const chain = supportedParentChains.filter(chain => chain.id === chainId) 40 | return chain[0] ?? null 41 | } 42 | 43 | export const getMaxBlockRange = (chain: Chain): bigint => { 44 | switch (chain) { 45 | case mainnet: 46 | case sepolia: 47 | case holesky: 48 | return BigInt(MAX_TIMEBOUNDS_SECONDS / 12) 49 | 50 | case base: 51 | case baseSepolia: 52 | return BigInt(MAX_TIMEBOUNDS_SECONDS / 2) 53 | 54 | case arbitrum: 55 | case arbitrumNova: 56 | case arbitrumSepolia: 57 | return BigInt(MAX_TIMEBOUNDS_SECONDS * 4) 58 | } 59 | 60 | return 0n 61 | } 62 | 63 | // this is different from simple `getParentChainBlockTime` in retryable-tracker because we need to fallback to Ethereum values no matter what the chain 64 | export const getParentChainBlockTimeForBatchPosting = ( 65 | childChain: ChildNetwork 66 | ) => { 67 | const parentChainId = childChain.parentChainId 68 | 69 | // for Base / Base Sepolia 70 | if (parentChainId === 8453 || parentChainId === 84532) return 2 71 | 72 | // for arbitrum networks, return the block-time corresponding to Ethereum 73 | return 12 74 | } 75 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/index.ts: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs' 2 | import { 3 | Log, 4 | PublicClient, 5 | createPublicClient, 6 | decodeFunctionData, 7 | defineChain, 8 | formatEther, 9 | http, 10 | parseAbi, 11 | } from 'viem' 12 | import { arbitrumNova } from 'viem/chains' 13 | import { AbiEvent } from 'abitype' 14 | import { 15 | getBatchPosters, 16 | isAnyTrust as isAnyTrustOrbitChain, 17 | } from '@arbitrum/orbit-sdk' 18 | import { 19 | getChainFromId, 20 | getMaxBlockRange, 21 | getParentChainBlockTimeForBatchPosting, 22 | MAX_TIMEBOUNDS_SECONDS, 23 | BATCH_POSTING_TIMEBOUNDS_FALLBACK, 24 | BATCH_POSTING_TIMEBOUNDS_BUFFER, 25 | MIN_DAYS_OF_BALANCE_LEFT, 26 | MAX_LOGS_TO_PROCESS_FOR_BALANCE, 27 | BATCH_POSTER_BALANCE_ALERT_THRESHOLD_FALLBACK, 28 | supportedCoreChainIds, 29 | } from './chains' 30 | import { BatchPosterMonitorOptions } from './types' 31 | import { reportBatchPosterErrorToSlack } from './reportBatchPosterAlertToSlack' 32 | import { 33 | ChildNetwork as ChainInfo, 34 | DEFAULT_CONFIG_PATH, 35 | getConfig, 36 | getExplorerUrlPrefixes, 37 | } from '../utils' 38 | 39 | // Parsing command line arguments using yargs 40 | const options: BatchPosterMonitorOptions = yargs(process.argv.slice(2)) 41 | .options({ 42 | configPath: { type: 'string', default: DEFAULT_CONFIG_PATH }, 43 | enableAlerting: { type: 'boolean', default: false }, 44 | writeToNotion: { type: 'boolean', default: false }, 45 | }) 46 | .strict() 47 | .parseSync() as BatchPosterMonitorOptions 48 | 49 | const config = getConfig({ configPath: options.configPath }) 50 | 51 | const sequencerBatchDeliveredEventAbi: AbiEvent = { 52 | anonymous: false, 53 | inputs: [ 54 | { 55 | indexed: true, 56 | internalType: 'uint256', 57 | name: 'batchSequenceNumber', 58 | type: 'uint256', 59 | }, 60 | { 61 | indexed: true, 62 | internalType: 'bytes32', 63 | name: 'beforeAcc', 64 | type: 'bytes32', 65 | }, 66 | { 67 | indexed: true, 68 | internalType: 'bytes32', 69 | name: 'afterAcc', 70 | type: 'bytes32', 71 | }, 72 | { 73 | indexed: false, 74 | internalType: 'bytes32', 75 | name: 'delayedAcc', 76 | type: 'bytes32', 77 | }, 78 | { 79 | indexed: false, 80 | internalType: 'uint256', 81 | name: 'afterDelayedMessagesRead', 82 | type: 'uint256', 83 | }, 84 | { 85 | components: [ 86 | { internalType: 'uint64', name: 'minTimestamp', type: 'uint64' }, 87 | { internalType: 'uint64', name: 'maxTimestamp', type: 'uint64' }, 88 | { internalType: 'uint64', name: 'minBlockNumber', type: 'uint64' }, 89 | { internalType: 'uint64', name: 'maxBlockNumber', type: 'uint64' }, 90 | ], 91 | indexed: false, 92 | internalType: 'struct ISequencerInbox.TimeBounds', 93 | name: 'timeBounds', 94 | type: 'tuple', 95 | }, 96 | { 97 | indexed: false, 98 | internalType: 'enum ISequencerInbox.BatchDataLocation', 99 | name: 'dataLocation', 100 | type: 'uint8', 101 | }, 102 | ], 103 | name: 'SequencerBatchDelivered', 104 | type: 'event', 105 | } 106 | 107 | const sequencerInboxAbi = [ 108 | { 109 | inputs: [ 110 | { 111 | internalType: 'uint256', 112 | name: 'sequenceNumber', 113 | type: 'uint256', 114 | }, 115 | { 116 | internalType: 'bytes', 117 | name: 'data', 118 | type: 'bytes', 119 | }, 120 | { 121 | internalType: 'uint256', 122 | name: 'afterDelayedMessagesRead', 123 | type: 'uint256', 124 | }, 125 | { 126 | internalType: 'address', 127 | name: 'gasRefunder', 128 | type: 'address', 129 | }, 130 | { 131 | internalType: 'uint256', 132 | name: 'prevMessageCount', 133 | type: 'uint256', 134 | }, 135 | { 136 | internalType: 'uint256', 137 | name: 'newMessageCount', 138 | type: 'uint256', 139 | }, 140 | ], 141 | name: 'addSequencerL2BatchFromOrigin', 142 | outputs: [], 143 | stateMutability: 'nonpayable', 144 | type: 'function', 145 | }, 146 | ] as const 147 | 148 | const displaySummaryInformation = ({ 149 | childChainInformation, 150 | lastBlockReported, 151 | latestBatchPostedBlockNumber, 152 | latestBatchPostedSecondsAgo, 153 | latestChildChainBlockNumber, 154 | batchPosterBacklogSize, 155 | batchPostingTimeBounds, 156 | }: { 157 | childChainInformation: ChainInfo 158 | lastBlockReported: bigint 159 | latestBatchPostedBlockNumber: bigint 160 | latestBatchPostedSecondsAgo: bigint 161 | latestChildChainBlockNumber: bigint 162 | batchPosterBacklogSize: bigint 163 | batchPostingTimeBounds: number 164 | }) => { 165 | console.log('**********') 166 | console.log(`Batch poster summary of [${childChainInformation.name}]`) 167 | console.log( 168 | `Latest block number on [${childChainInformation.name}] is ${latestChildChainBlockNumber}.` 169 | ) 170 | console.log( 171 | `Latest [${ 172 | childChainInformation.name 173 | }] block included on [Parent chain id: ${ 174 | childChainInformation.parentChainId 175 | }, block-number ${latestBatchPostedBlockNumber}] is ${lastBlockReported} => ${ 176 | latestBatchPostedSecondsAgo / 60n / 60n 177 | } hours, ${(latestBatchPostedSecondsAgo / 60n) % 60n} minutes, ${ 178 | latestBatchPostedSecondsAgo % 60n 179 | } seconds ago.` 180 | ) 181 | 182 | console.log(`Batch poster backlog is ${batchPosterBacklogSize} blocks.`) 183 | console.log(timeBoundsExpectedMessage(batchPostingTimeBounds)) 184 | console.log('**********') 185 | console.log('') 186 | } 187 | 188 | const allBatchedAlertsContent: string[] = [] 189 | 190 | const showAlert = (childChainInformation: ChainInfo, reasons: string[]) => { 191 | const { PARENT_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes( 192 | childChainInformation 193 | ) 194 | 195 | reasons 196 | .reverse() 197 | .push( 198 | `SequencerInbox located at <${ 199 | PARENT_CHAIN_ADDRESS_PREFIX + 200 | childChainInformation.ethBridge.sequencerInbox 201 | }|${childChainInformation.ethBridge.sequencerInbox}> on [chain id ${ 202 | childChainInformation.parentChainId 203 | }]` 204 | ) 205 | 206 | const reasonsString = reasons 207 | .filter(reason => !!reason.trim().length) 208 | .join('\n• ') 209 | 210 | console.log(`Alert on ${childChainInformation.name}:`) 211 | console.log(`• ${reasonsString}`) 212 | console.log('--------------------------------------') 213 | console.log('') 214 | allBatchedAlertsContent.push( 215 | `[${childChainInformation.name}]:\n• ${reasonsString}` 216 | ) 217 | } 218 | 219 | type EventLogs = Log< 220 | bigint, 221 | number, 222 | false, 223 | AbiEvent, 224 | undefined, 225 | [AbiEvent], 226 | string 227 | >[] 228 | 229 | const getBatchPosterFromEventLogs = async ( 230 | eventLogs: EventLogs, 231 | parentChainClient: PublicClient 232 | ) => { 233 | // get the batch-poster for the first event log 234 | const batchPostingTransactionHash = eventLogs[0].transactionHash 235 | const tx = await parentChainClient.getTransaction({ 236 | hash: batchPostingTransactionHash, 237 | }) 238 | return tx.from 239 | } 240 | 241 | const getBatchPosterAddress = async ( 242 | parentChainClient: PublicClient, 243 | childChainInformation: ChainInfo, 244 | sequencerInboxLogs: EventLogs 245 | ) => { 246 | // if we have sequencer inbox logs, then get the batch poster directly 247 | if (sequencerInboxLogs.length > 0) { 248 | return await getBatchPosterFromEventLogs( 249 | sequencerInboxLogs, 250 | parentChainClient 251 | ) 252 | } 253 | 254 | // else derive batch poster from the sdk 255 | const { batchPosters, isAccurate } = await getBatchPosters( 256 | //@ts-ignore - PublicClient that we pass vs PublicClient that orbit-sdk expects is not matching 257 | parentChainClient, 258 | { 259 | rollup: childChainInformation.ethBridge.rollup as `0x${string}`, 260 | sequencerInbox: childChainInformation.ethBridge 261 | .sequencerInbox as `0x${string}`, 262 | } 263 | ) 264 | 265 | if (isAccurate) { 266 | return batchPosters[0] // get the first batch poster 267 | } else { 268 | throw Error('Batch poster information not found') 269 | } 270 | } 271 | 272 | const getBatchPosterLowBalanceAlertMessage = async ( 273 | parentChainClient: PublicClient, 274 | childChainInformation: ChainInfo, 275 | sequencerInboxLogs: EventLogs 276 | ) => { 277 | const { PARENT_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes( 278 | childChainInformation 279 | ) 280 | 281 | const batchPoster = await getBatchPosterAddress( 282 | parentChainClient, 283 | childChainInformation, 284 | sequencerInboxLogs 285 | ) 286 | const currentBalance = await parentChainClient.getBalance({ 287 | address: batchPoster, 288 | }) 289 | 290 | // if there are no logs, add a static check for low balance 291 | if (sequencerInboxLogs.length === 0) { 292 | const bal = Number(formatEther(currentBalance)) 293 | if (bal < BATCH_POSTER_BALANCE_ALERT_THRESHOLD_FALLBACK) { 294 | return `Low Batch poster balance (<${ 295 | PARENT_CHAIN_ADDRESS_PREFIX + batchPoster 296 | }|${batchPoster}>): ${formatEther( 297 | currentBalance 298 | )} ETH (Minimum expected balance: ${BATCH_POSTER_BALANCE_ALERT_THRESHOLD_FALLBACK} ETH). ` 299 | } 300 | return null 301 | } 302 | 303 | // Dynamic balance check based on the logs 304 | // Extract the most recent logs for processing to avoid overloading with too many logs 305 | const recentLogs = [...sequencerInboxLogs].slice( 306 | -MAX_LOGS_TO_PROCESS_FOR_BALANCE 307 | ) 308 | 309 | // Calculate the elapsed time (in seconds) since the first block in the logs 310 | const firstTransaction = await parentChainClient.getTransaction({ 311 | hash: recentLogs[0].transactionHash, 312 | }) 313 | const initialBlock = await parentChainClient.getBlock({ 314 | blockNumber: firstTransaction.blockNumber, 315 | }) 316 | const initialBlockTimestamp = initialBlock.timestamp 317 | 318 | const elapsedTimeSinceFirstBlock = 319 | BigInt(Math.floor(Date.now() / 1000)) - initialBlockTimestamp 320 | 321 | // Loop through each log and calculate the gas cost for posting batches 322 | let postingCost = BigInt(0) 323 | for (const log of recentLogs) { 324 | const tx = await parentChainClient.getTransactionReceipt({ 325 | hash: log.transactionHash, 326 | }) 327 | postingCost += tx.gasUsed * tx.effectiveGasPrice // Accumulate the transaction cost 328 | } 329 | 330 | // Calculate the approximate balance spent over the last 24 hours 331 | const secondsIn1Day = 24n * 60n * 60n 332 | 333 | const timeRatio = 334 | secondsIn1Day / elapsedTimeSinceFirstBlock > 1n 335 | ? secondsIn1Day / elapsedTimeSinceFirstBlock 336 | : 1n // set minimum cap of the ratio to 1, since we are calculating the cost for 24 hours, else bigInt rounds off the ratio to zero 337 | 338 | const dailyPostingCostEstimate = timeRatio * postingCost 339 | 340 | // Estimate how many days the current balance will last based on the daily cost 341 | const daysLeftForCurrentBalance = currentBalance / dailyPostingCostEstimate 342 | console.log( 343 | `The current batch poster balance is ${formatEther( 344 | currentBalance 345 | )} ETH, and balance spent in 24 hours is approx ${formatEther( 346 | dailyPostingCostEstimate 347 | )} ETH. The current balance can last approximately ${daysLeftForCurrentBalance} days.` 348 | ) 349 | 350 | // Determine the minimum expected balance needed to maintain operations for a certain number of days 351 | const minimumExpectedBalance = 352 | MIN_DAYS_OF_BALANCE_LEFT * dailyPostingCostEstimate 353 | 354 | // Check if the current balance is below the minimum expected balance 355 | // Return a warning message if low balance is detected 356 | const lowBalanceDetected = currentBalance < minimumExpectedBalance 357 | 358 | if (lowBalanceDetected) { 359 | return `Low Batch poster balance (<${ 360 | PARENT_CHAIN_ADDRESS_PREFIX + batchPoster 361 | }|${batchPoster}>): ${formatEther( 362 | currentBalance 363 | )} ETH (Minimum expected balance: ${formatEther( 364 | minimumExpectedBalance 365 | )} ETH). The current balance is expected to last for ~${daysLeftForCurrentBalance} days only.` 366 | } 367 | 368 | return null 369 | } 370 | 371 | const checkForUserTransactionBlocks = async ({ 372 | fromBlock, 373 | toBlock, 374 | publicClient, 375 | }: { 376 | fromBlock: number 377 | toBlock: number 378 | publicClient: PublicClient 379 | }) => { 380 | const MINER_OF_USER_TX_BLOCKS = '0xa4b000000000000000000073657175656e636572' // this will be the miner address if a block contains user tx 381 | 382 | let userTransactionBlockFound = false 383 | 384 | for (let i = fromBlock; i <= toBlock; i++) { 385 | const block = await publicClient.getBlock({ blockNumber: BigInt(i) }) 386 | if (block.miner === MINER_OF_USER_TX_BLOCKS) { 387 | userTransactionBlockFound = true 388 | break 389 | } 390 | } 391 | 392 | return userTransactionBlockFound 393 | } 394 | 395 | const getBatchPostingTimeBounds = async ( 396 | childChainInformation: ChainInfo, 397 | parentChainClient: PublicClient 398 | ) => { 399 | let batchPostingTimeBounds = BATCH_POSTING_TIMEBOUNDS_FALLBACK 400 | try { 401 | const maxTimeVariation = await parentChainClient.readContract({ 402 | address: childChainInformation.ethBridge.sequencerInbox as `0x${string}`, 403 | abi: parseAbi([ 404 | 'function maxTimeVariation() view returns (uint256, uint256, uint256, uint256)', 405 | ]), 406 | functionName: 'maxTimeVariation', 407 | }) 408 | 409 | const delayBlocks = Number(maxTimeVariation[0]) 410 | const delaySeconds = Number(maxTimeVariation[2].toString()) 411 | 412 | // use the minimum of delayBlocks or delay seconds 413 | batchPostingTimeBounds = Math.min( 414 | delayBlocks * 415 | getParentChainBlockTimeForBatchPosting(childChainInformation), 416 | delaySeconds 417 | ) 418 | } catch (_) { 419 | // no-op, use the fallback value 420 | } 421 | 422 | // formula : min(50% of x , max(1h, x - buffer)) 423 | // minimum of half of the batchPostingTimeBounds vs [1 hour vs batchPostingTimeBounds - buffer] 424 | return Math.min( 425 | 0.65 * batchPostingTimeBounds, 426 | Math.max(3600, batchPostingTimeBounds - BATCH_POSTING_TIMEBOUNDS_BUFFER) 427 | ) 428 | } 429 | 430 | const timeBoundsExpectedMessage = (batchPostingTimebounds: number) => 431 | `At least 1 batch is expected to be posted every ${ 432 | batchPostingTimebounds / 60 / 60 433 | } hours.` 434 | 435 | const isAnyTrust = async ( 436 | childChainInformation: ChainInfo, 437 | parentChainClient: PublicClient 438 | ) => { 439 | const { chainId } = childChainInformation 440 | 441 | const anyTrustCoreChainIds = [arbitrumNova.id] as number[] // core chains that we know are AnyTrust 442 | 443 | // if chainId being passed is a core chainId 444 | if (supportedCoreChainIds.includes(chainId)) { 445 | // then return true if it's in anyTrust, else false 446 | return anyTrustCoreChainIds.includes(chainId) 447 | } 448 | 449 | return isAnyTrustOrbitChain({ 450 | publicClient: parentChainClient as any, 451 | rollup: childChainInformation.ethBridge.rollup as `0x${string}`, 452 | }) 453 | } 454 | 455 | const monitorBatchPoster = async (childChainInformation: ChainInfo) => { 456 | const alertsForChildChain: string[] = [] 457 | 458 | const parentChain = getChainFromId(childChainInformation.parentChainId) 459 | const childChain = defineChain({ 460 | id: childChainInformation.chainId, 461 | name: childChainInformation.name, 462 | network: 'childChain', 463 | nativeCurrency: { 464 | name: 'ETH', 465 | symbol: 'ETH', 466 | decimals: 18, 467 | }, 468 | rpcUrls: { 469 | default: { 470 | http: [childChainInformation.orbitRpcUrl], 471 | }, 472 | public: { 473 | http: [childChainInformation.orbitRpcUrl], 474 | }, 475 | }, 476 | }) 477 | 478 | const parentChainClient = createPublicClient({ 479 | chain: parentChain, 480 | transport: http(childChainInformation.parentRpcUrl), 481 | }) 482 | const childChainClient = createPublicClient({ 483 | chain: childChain, 484 | transport: http(childChainInformation.orbitRpcUrl), 485 | }) 486 | 487 | // Getting sequencer inbox logs 488 | const latestBlockNumber = await parentChainClient.getBlockNumber() 489 | 490 | const blocksToProcess = getMaxBlockRange(parentChain) 491 | const toBlock = latestBlockNumber 492 | const fromBlock = toBlock - blocksToProcess 493 | 494 | // if the block range provided is >=MAX_BLOCKS_TO_PROCESS, we might get rate limited while fetching logs from the node 495 | // so we break down the range into smaller chunks and process them sequentially 496 | // generate the final ranges' batches to process [ [fromBlock, toBlock], [fromBlock, toBlock], ...] 497 | const ranges = [], 498 | fromBlockNum = Number(fromBlock.toString()), 499 | toBlockNum = Number(toBlock.toString()) 500 | 501 | const MAX_BLOCKS_TO_PROCESS = 502 | childChainInformation.parentChainId === 1 ? 500 : 500000 // for Ethereum, have lower block range to avoid rate limiting 503 | 504 | for (let i = fromBlockNum; i <= toBlockNum; i += MAX_BLOCKS_TO_PROCESS) { 505 | ranges.push([i, Math.min(i + MAX_BLOCKS_TO_PROCESS - 1, toBlockNum)]) 506 | } 507 | 508 | const sequencerInboxLogsArray = [] 509 | for (const range of ranges) { 510 | const logs = await parentChainClient.getLogs({ 511 | address: childChainInformation.ethBridge.sequencerInbox as `0x${string}`, 512 | event: sequencerBatchDeliveredEventAbi, 513 | fromBlock: BigInt(range[0]), 514 | toBlock: BigInt(range[1]), 515 | }) 516 | sequencerInboxLogsArray.push(logs) 517 | } 518 | 519 | // Flatten the array of arrays to get final array of logs 520 | const sequencerInboxLogs = sequencerInboxLogsArray.flat() 521 | 522 | // First, a basic check to get batch poster balance 523 | const batchPosterLowBalanceMessage = 524 | await getBatchPosterLowBalanceAlertMessage( 525 | parentChainClient, 526 | childChainInformation, 527 | sequencerInboxLogs 528 | ) 529 | if (batchPosterLowBalanceMessage) { 530 | alertsForChildChain.push(batchPosterLowBalanceMessage) 531 | } 532 | 533 | const batchPostingTimeBounds = await getBatchPostingTimeBounds( 534 | childChainInformation, 535 | parentChainClient 536 | ) 537 | 538 | // Get the last block of the chain 539 | const latestChildChainBlockNumber = await childChainClient.getBlockNumber() 540 | 541 | if (!sequencerInboxLogs || sequencerInboxLogs.length === 0) { 542 | // get the last block that is 'safe' ie. can be assumed to have been posted 543 | const latestChildChainSafeBlock = await childChainClient.getBlock({ 544 | blockTag: 'safe', 545 | }) 546 | 547 | const blocksPendingToBePosted = 548 | latestChildChainBlockNumber - latestChildChainSafeBlock.number 549 | 550 | const doPendingBlocksContainUserTransactions = 551 | await checkForUserTransactionBlocks({ 552 | fromBlock: Number(latestChildChainSafeBlock.number + 1n), // start checking AFTER the latest 'safe' block 553 | toBlock: Number(latestChildChainBlockNumber), 554 | publicClient: childChainClient, 555 | }) 556 | 557 | const batchPostingBacklog = 558 | blocksPendingToBePosted > 0n && doPendingBlocksContainUserTransactions 559 | 560 | // if alert situation 561 | if (batchPostingBacklog) { 562 | alertsForChildChain.push( 563 | `No batch has been posted in the last ${ 564 | MAX_TIMEBOUNDS_SECONDS / 60 / 60 565 | } hours, and last block number (${latestChildChainBlockNumber}) is greater than the last safe block number (${ 566 | latestChildChainSafeBlock.number 567 | }). ${timeBoundsExpectedMessage(batchPostingTimeBounds)}` 568 | ) 569 | 570 | showAlert(childChainInformation, alertsForChildChain) 571 | } else { 572 | // if no alerting situation, just log the summary 573 | console.log( 574 | `**********\nBatch poster summary of [${childChainInformation.name}]` 575 | ) 576 | console.log( 577 | `No user activity in the last ${ 578 | MAX_TIMEBOUNDS_SECONDS / 60 / 60 579 | } hours, and hence no batch has been posted.\n` 580 | ) 581 | 582 | // in this case show alert only if batch poster balance is low 583 | if (batchPosterLowBalanceMessage) { 584 | showAlert(childChainInformation, alertsForChildChain) 585 | } 586 | } 587 | return 588 | } 589 | 590 | // Get the latest log 591 | const lastSequencerInboxLog = sequencerInboxLogs.pop() 592 | 593 | const isChainAnyTrust = await isAnyTrust( 594 | childChainInformation, 595 | parentChainClient 596 | ) 597 | 598 | if (isChainAnyTrust) { 599 | const alerts = await checkIfAnyTrustRevertedToPostDataOnChain({ 600 | parentChainClient, 601 | childChainInformation, 602 | lastSequencerInboxLog, 603 | }) 604 | alertsForChildChain.push(...alerts) 605 | } 606 | // Get the timestamp of the block where that log was emitted 607 | const lastSequencerInboxBlock = await parentChainClient.getBlock({ 608 | blockNumber: lastSequencerInboxLog!.blockNumber, 609 | }) 610 | const lastBatchPostedTime = lastSequencerInboxBlock.timestamp 611 | const secondsSinceLastBatchPoster = 612 | BigInt(Math.floor(Date.now() / 1000)) - lastBatchPostedTime 613 | 614 | // Get last block that's part of a batch 615 | const lastBlockReported = await parentChainClient.readContract({ 616 | address: childChainInformation.ethBridge.bridge as `0x${string}`, 617 | abi: parseAbi([ 618 | 'function sequencerReportedSubMessageCount() view returns (uint256)', 619 | ]), 620 | functionName: 'sequencerReportedSubMessageCount', 621 | }) 622 | 623 | // Get batch poster backlog 624 | const batchPosterBacklog = latestChildChainBlockNumber - lastBlockReported 625 | 626 | // If there's backlog and last batch posted was 4 hours ago, send alert 627 | if ( 628 | batchPosterBacklog > 0 && 629 | secondsSinceLastBatchPoster > BigInt(batchPostingTimeBounds) 630 | ) { 631 | alertsForChildChain.push( 632 | `Last batch was posted ${ 633 | secondsSinceLastBatchPoster / 60n / 60n 634 | } hours and ${ 635 | (secondsSinceLastBatchPoster / 60n) % 60n 636 | } mins ago, and there's a backlog of ${batchPosterBacklog} blocks in the chain. ${timeBoundsExpectedMessage( 637 | batchPostingTimeBounds 638 | )}` 639 | ) 640 | } 641 | 642 | if (alertsForChildChain.length > 0) { 643 | showAlert(childChainInformation, alertsForChildChain) 644 | return 645 | } 646 | 647 | displaySummaryInformation({ 648 | childChainInformation, 649 | lastBlockReported, 650 | latestBatchPostedBlockNumber: lastSequencerInboxBlock.number, 651 | latestBatchPostedSecondsAgo: secondsSinceLastBatchPoster, 652 | latestChildChainBlockNumber, 653 | batchPosterBacklogSize: batchPosterBacklog, 654 | batchPostingTimeBounds, 655 | }) 656 | } 657 | 658 | const main = async () => { 659 | // log the chains being processed for better debugging in github actions 660 | console.log( 661 | '>>>>>> Processing chains: ', 662 | config.childChains.map((chainInformation: ChainInfo) => ({ 663 | name: chainInformation.name, 664 | chainID: chainInformation.chainId, 665 | rpc: chainInformation.orbitRpcUrl, 666 | })) 667 | ) 668 | 669 | // process each chain sequentially to avoid RPC rate limiting 670 | for (const childChain of config.childChains) { 671 | try { 672 | console.log('>>>>> Processing chain: ', childChain.name) 673 | await monitorBatchPoster(childChain) 674 | } catch (e) { 675 | const errorStr = `Batch Posting alert on [${childChain.name}]:\nError processing chain: ${e.message}` 676 | if (options.enableAlerting) { 677 | await reportBatchPosterErrorToSlack({ 678 | message: errorStr, 679 | }) 680 | } 681 | console.error(errorStr) 682 | } 683 | } 684 | 685 | if (options.enableAlerting && allBatchedAlertsContent.length > 0) { 686 | const finalMessage = `Batch poster monitor summary \n\n${allBatchedAlertsContent.join( 687 | '\n--------------------------------------\n' 688 | )}` 689 | 690 | console.log(finalMessage) 691 | await reportBatchPosterErrorToSlack({ 692 | message: finalMessage, 693 | }) 694 | } 695 | } 696 | 697 | const checkIfAnyTrustRevertedToPostDataOnChain = async ({ 698 | parentChainClient, 699 | childChainInformation, 700 | lastSequencerInboxLog, 701 | }: { 702 | parentChainClient: PublicClient 703 | childChainInformation: ChainInfo 704 | lastSequencerInboxLog: 705 | | Log 706 | | undefined 707 | }): Promise => { 708 | const alerts = [] 709 | 710 | try { 711 | // Get the transaction that emitted `lastSequencerInboxLog` 712 | const transaction = await parentChainClient.getTransaction({ 713 | hash: lastSequencerInboxLog?.transactionHash as `0x${string}`, 714 | }) 715 | 716 | const { args } = decodeFunctionData({ 717 | abi: sequencerInboxAbi, 718 | data: transaction.input, 719 | }) 720 | 721 | // Extract the 'data' field 722 | const batchData = args[1] as `0x${string}` 723 | 724 | // Check the first byte of the data 725 | const firstByte = batchData.slice(0, 4) 726 | 727 | if (firstByte === '0x00') { 728 | alerts.push( 729 | `AnyTrust chain [${childChainInformation.name}] has fallen back to posting calldata on-chain. This indicates a potential issue with the Data Availability Committee.` 730 | ) 731 | } else if (firstByte === '0x88') { 732 | console.log( 733 | `Chain [${childChainInformation.name}] is using AnyTrust DACert as expected.` 734 | ) 735 | } else { 736 | console.log( 737 | `Chain [${childChainInformation.name}] is using an unknown data format. First byte: ${firstByte}` 738 | ) 739 | } 740 | } catch (e) { 741 | console.log( 742 | `Chain [${childChainInformation.name}]: Error checking if AnyTrust reverted to posting calldata on-chain: ${e.message}` 743 | ) 744 | } 745 | 746 | return alerts 747 | } 748 | 749 | main() 750 | .then(() => process.exit(0)) 751 | .catch(error => { 752 | console.error(error) 753 | process.exit(1) 754 | }) 755 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "batch-poster-monitor", 3 | "version": "1.0.0", 4 | "main": "index.ts", 5 | "scripts": { 6 | "lint": "eslint .", 7 | "build": "rm -rf ./dist && tsc", 8 | "format": "prettier './**/*.{js,json,md,yml,sol,ts}' --write && yarn run lint --fix", 9 | "dev": "yarn build && node ./dist/batch-poster-monitor/index.js" 10 | }, 11 | "dependencies": { 12 | "@arbitrum/orbit-sdk": "^0.23.1", 13 | "abitype": "^1.0.5", 14 | "typescript": "^5.4.5", 15 | "utils": "*", 16 | "viem": "1.20.0" 17 | }, 18 | "devDependencies": { 19 | "@types/node": "^20.14.1", 20 | "ts-node": "^10.9.2" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/reportBatchPosterAlertToSlack.ts: -------------------------------------------------------------------------------- 1 | import { postSlackMessage } from '../utils/postSlackMessage' 2 | 3 | const slackToken = process.env.BATCH_POSTER_MONITORING_SLACK_TOKEN 4 | const slackChannel = process.env.BATCH_POSTER_MONITORING_SLACK_CHANNEL 5 | 6 | export const reportBatchPosterErrorToSlack = ({ 7 | message, 8 | }: { 9 | message: string 10 | }) => { 11 | if (!slackToken) throw new Error(`Slack token is required.`) 12 | if (!slackChannel) throw new Error(`Slack channel is required.`) 13 | 14 | if (process.env.NODE_ENV === 'DEV') return 15 | if (process.env.NODE_ENV === 'CI' && message === 'success') return 16 | 17 | console.log(`>>> Reporting message to Slack -> ${message}`) 18 | 19 | return postSlackMessage({ 20 | slackToken, 21 | slackChannel, 22 | message, 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "./dist" 5 | }, 6 | "include": ["./**/*.ts", "./**/*.d.ts", "packages/*.ts"], 7 | "exclude": ["node_modules", "dist/"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/batch-poster-monitor/types.ts: -------------------------------------------------------------------------------- 1 | export type BatchPosterMonitorOptions = { 2 | configPath: string 3 | enableAlerting: boolean 4 | writeToNotion: boolean 5 | } 6 | -------------------------------------------------------------------------------- /packages/retryable-monitor/README.md: -------------------------------------------------------------------------------- 1 | # Retryable Monitor 2 | 3 | > For installation and general configuration, see the [main README](../../README.md). 4 | 5 | ## Overview 6 | 7 | The Retryable Monitor tracks ParentChain->ChildChain message execution through retryable tickets. These tickets are the primary mechanism for cross-chain communication in Arbitrum. [Learn more](https://docs.arbitrum.io/arbos/l1-to-l2-messaging). 8 | 9 | ## Command-Line Interface 10 | 11 | ```bash 12 | yarn retryable-monitor [options] 13 | 14 | Monitor retryable tickets on Arbitrum chains 15 | 16 | Options: 17 | --help Show help [boolean] 18 | --version Show version number [boolean] 19 | --configPath Path to config file [string] [default: "config.json"] 20 | --enableAlerting Enable Slack alerts [boolean] [default: false] 21 | --fromBlock Starting block number for monitoring [number] 22 | --toBlock Ending block number for monitoring [number] 23 | --continuous Run monitor continuously [boolean] [default: false] 24 | --writeToNotion Sync ticket metadata to Notion [boolean] [default: false] 25 | 26 | Examples: 27 | yarn retryable-monitor --continuous Run continuous monitoring 28 | yarn retryable-monitor --fromBlock=1000 --toBlock=2000 Check specific block range 29 | yarn retryable-monitor --enableAlerting --writeToNotion Enables Slack alerts and syncs retryable data to Notion 30 | 31 | Environment Variables: 32 | RETRYABLE_MONITORING_SLACK_TOKEN Slack API token for alerts 33 | RETRYABLE_MONITORING_SLACK_CHANNEL Slack channel for alerts 34 | RETRYABLE_MONITORING_NOTION_TOKEN Notion integration token 35 | RETRYABLE_MONITORING_NOTION_DB_ID Notion database ID 36 | ``` 37 | 38 | ## Monitoring Behavior 39 | 40 | By default, the monitor runs once. You can pass `--continuous` to keep it running. 41 | 42 | ✅ Finds retryable tickets 43 | ✅ Writes to Notion if `--writeToNotion` is used: 44 |   • New tickets are marked `Untriaged` 45 |   • Existing tickets update metadata only 46 | ✅ Sends Slack alerts for new tickets added to Notion if both `--writeToNotion` and `--enableAlerting` are used 47 | 48 | When `--continuous` is on, it also: 49 | 50 | ✅ Checks for new tickets every 3 minutes 51 | ✅ Sweeps the Notion DB every 24 hours to: 52 |   • Mark tickets as "Expired" after 7 days 53 |   • Alert on tickets expiring soon and still `Untriaged` or `Investigating` 54 | 55 | ## Monitor Details 56 | 57 | Retryable tickets are Arbitrum’s mechanism for guaranteed ParentChain → ChildChain message delivery. When a message is sent from the parent chain to the child chain, it creates a retryable ticket that must be executed within 7 days. This monitor tracks those tickets from creation through execution, ensuring that no messages are lost or expire unexecuted. 58 | 59 | The monitoring process spans both parent and child chains: 60 | 61 | - On the parent chain, it listens for `MessageDelivered` events that indicate a retryable ticket has been created. 62 | 63 | - On the child chain, it checks the status of each ticket, including whether it was successfully redeemed (automatically or manually), still pending, or failed. 64 | 65 | If `--writeToNotion` is enabled, each detected ticket is written to a Notion database with metadata such as creation time, gas information, callvalue, token deposit amount, and expiration timestamp. 66 | 67 | - If both --`writeToNotion` and `--enableAlerting` are enabled, the monitor sends a Slack alert the first time a new retryable is added to Notion with status `Untriaged`. 68 | 69 | When running in `--continuous` mode, the monitor also performs a Notion sweep every 24 hours to: 70 | 71 | - Mark tickets as `Expired` if more than 7 days have passed without redemption. 72 | 73 | - Alert on tickets that are close to expiring (less than 2 days left) and still marked as `Untriaged` or `Investigating`. 74 | 75 | This dual-layer monitoring ensures cross-chain messages are reliably delivered and that at-risk tickets are surfaced for action before expiration. 76 | 77 | ### Critical Events 78 | 79 | The monitor tracks five key events that represent state transitions: 80 | 81 | - `RetryableTicketCreated`: A new ParentChain->ChildChain message has been created and funded 82 | - `RedeemScheduled`: A redemption attempt has been initiated 83 | - `TicketRedeemed`: The message has been successfully executed on ChildChain 84 | - `AutoRedemptionSuccess`: Automatic redemption system successfully executed the message 85 | - `AutoRedemptionFailed`: Automatic redemption attempt failed, manual intervention may be needed 86 | 87 | ### Alert Scenarios 88 | 89 | By default, Slack alerts are not sent. 90 | 91 | If `--enableAlerting` is used without `--writeToNotion`, alerts are only sent when errors occur during processing (e.g., RPC failures or unexpected exceptions). 92 | 93 | When used with `--writeToNotion`, Slack alerts are more advanced. See the next section for details — including alerts for new retryables and tickets close to expiration based on Notion status. 94 | 95 | ## About the Notion Database 96 | 97 | The Notion database acts as a shared triage board for tracking retryable ticket status and metadata across Orbit chains. 98 | 99 | When the monitor is run with `--writeToNotion`: 100 | 101 | - Each unredeemed ticket is written to Notion once, with structured metadata like gas info, deposited tokens, callvalue, and expiration time. 102 | 103 | - If the ticket already exists, its metadata is updated — but the Status field is preserved unless it's still `Untriaged` or blank. 104 | 105 | This prevents overwriting any manual updates (e.g., `Investigating` or `Resolved`). 106 | This enables teams to track and resolve at-risk retryables without losing triage state between runs. 107 | 108 | ### Slack Alerts (when `--enableAlerting` is also used) 109 | 110 | Slack alerts are sent only for tickets that: 111 | 112 | - Are marked as `Untriaged` or `Investigating`, 113 | 114 | - Have less than 2 days left before expiration. 115 | 116 | - Are not already marked as `Resolved` or `Expired` 117 | 118 | - Successfully redeemed tickets are skipped, keeping the database focused on retryables that are stuck, failed, or at risk. 119 | 120 | ### Required Columns 121 | 122 | The Notion database should be configured with the following columns: 123 | 124 | | **Column** | **Type** | **Description** | 125 | | -------------------- | -------- | ------------------------------------------------------------------------- | 126 | | `ParentTx` | URL | Link to the parent chain transaction that created the retryable | 127 | | `ChildTx` | URL | Link to the child chain transaction (if available) | 128 | | `CreatedAt` | Date | Timestamp (ms) when the retryable was created | 129 | | `Timeout` | Number | Expiration timestamp in milliseconds | 130 | | `Status` | Select | Workflow status (`Untriaged`, `Investigating`, `Expired`, `Resolved`.) | 131 | | `Priority` | Select | Optional manual priority (`High`, `Medium`, `Low`, `Unset`) | 132 | | `TokensDeposited` | Text | Amount, symbol, and token address (e.g. `1.23 USDC ($1.23) (0xToken...)`) | 133 | | `GasPriceProvided` | Text | Gas price submitted when the ticket was created | 134 | | `GasPriceAtCreation` | Text | L2 gas price at the time of ticket creation | 135 | | `gasPriceNow` | Text | Current L2 gas price | 136 | | `L2CallValue` | Text | ETH or native callvalue (e.g. `0.0001 ETH ($0.18)`) | 137 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/depositEventFetcher.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Logic for fetching the deposits initiated and message delivered events from the parent chain 3 | */ 4 | 5 | import { Provider } from '@ethersproject/abstract-provider' 6 | import { EventFetcher } from '@arbitrum/sdk' 7 | import { Bridge__factory } from '@arbitrum/sdk/dist/lib/abi/factories/Bridge__factory' 8 | import { L1ERC20Gateway__factory } from '@arbitrum/sdk/dist/lib/abi/factories/L1ERC20Gateway__factory' 9 | import { DepositInitiatedEvent } from '@arbitrum/sdk/dist/lib/abi/L1ERC20Gateway' 10 | 11 | export const getDepositInitiatedEventData = async ( 12 | parentChainGatewayAddress: string, 13 | filter: { 14 | fromBlock: number 15 | toBlock: number 16 | }, 17 | parentChainProvider: Provider 18 | ) => { 19 | const eventFetcher = new EventFetcher(parentChainProvider) 20 | const logs = await eventFetcher.getEvents( 21 | L1ERC20Gateway__factory, 22 | (g: any) => g.filters.DepositInitiated(), 23 | { 24 | ...filter, 25 | address: parentChainGatewayAddress, 26 | } 27 | ) 28 | 29 | return logs 30 | } 31 | 32 | export const getMessageDeliveredEventData = async ( 33 | parentBridgeAddress: string, 34 | filter: { 35 | fromBlock: number 36 | toBlock: number 37 | }, 38 | parentChainProvider: Provider 39 | ) => { 40 | const eventFetcher = new EventFetcher(parentChainProvider) 41 | const logs = await eventFetcher.getEvents( 42 | Bridge__factory, 43 | (g: any) => g.filters.MessageDelivered(), 44 | { ...filter, address: parentBridgeAddress } 45 | ) 46 | 47 | // Filter logs where event.kind is equal to 9 48 | // https://github.com/OffchainLabs/nitro-contracts/blob/38a70a5e14f8b52478eb5db08e7551a82ced14fe/src/libraries/MessageTypes.sol#L9 49 | const filteredLogs = logs.filter(log => log.event.kind === 9) 50 | 51 | return filteredLogs 52 | } 53 | 54 | export const getDepositInitiatedLogs = async ({ 55 | fromBlock, 56 | toBlock, 57 | parentChainProvider, 58 | gatewayAddresses, 59 | }: { 60 | fromBlock: number 61 | toBlock: number 62 | parentChainProvider: Provider 63 | gatewayAddresses: { 64 | parentErc20Gateway: string 65 | parentCustomGateway: string 66 | parentWethGateway: string 67 | } 68 | }) => { 69 | const [ 70 | depositsInitiatedLogsL1Erc20Gateway, 71 | depositsInitiatedLogsL1CustomGateway, 72 | depositsInitiatedLogsL1WethGateway, 73 | ] = await Promise.all( 74 | [ 75 | gatewayAddresses.parentErc20Gateway, 76 | gatewayAddresses.parentCustomGateway, 77 | gatewayAddresses.parentWethGateway, 78 | ].map(gatewayAddress => { 79 | return getDepositInitiatedEventData( 80 | gatewayAddress, 81 | { fromBlock, toBlock }, 82 | parentChainProvider 83 | ) 84 | }) 85 | ) 86 | 87 | return [ 88 | ...depositsInitiatedLogsL1Erc20Gateway, 89 | ...depositsInitiatedLogsL1CustomGateway, 90 | ...depositsInitiatedLogsL1WethGateway, 91 | ] 92 | } 93 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/reportGenerator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Logic for generating the intermediate details for the Retryables 3 | */ 4 | 5 | import { 6 | ParentToChildMessageStatus, 7 | ParentTransactionReceipt, 8 | ParentToChildMessageReader, 9 | } from '@arbitrum/sdk' 10 | import { BigNumber, providers } from 'ethers' 11 | import { TransactionReceipt } from '@ethersproject/abstract-provider' 12 | import { SEVEN_DAYS_IN_SECONDS } from '@arbitrum/sdk/dist/lib/dataEntities/constants' 13 | import { ChildChainTicketReport, ParentChainTicketReport } from './types' 14 | 15 | export const getParentChainRetryableReport = ( 16 | arbParentTxReceipt: ParentTransactionReceipt, 17 | retryableMessage: ParentToChildMessageReader 18 | ): ParentChainTicketReport => { 19 | return { 20 | id: arbParentTxReceipt.transactionHash, 21 | transactionHash: arbParentTxReceipt.transactionHash, 22 | sender: arbParentTxReceipt.from, 23 | retryableTicketID: retryableMessage.retryableCreationId, 24 | } 25 | } 26 | 27 | export const getChildChainRetryableReport = async ({ 28 | childChainTx, 29 | childChainTxReceipt, 30 | retryableMessage, 31 | childChainProvider, 32 | }: { 33 | childChainTx: providers.TransactionResponse 34 | childChainTxReceipt: TransactionReceipt 35 | retryableMessage: ParentToChildMessageReader 36 | childChainProvider: providers.Provider 37 | }): Promise => { 38 | let status = await retryableMessage.status() 39 | 40 | const timestamp = ( 41 | await childChainProvider.getBlock(childChainTxReceipt.blockNumber) 42 | ).timestamp 43 | 44 | const childChainTicketReport = { 45 | id: retryableMessage.retryableCreationId, 46 | retryTxHash: (await retryableMessage.getAutoRedeemAttempt()) 47 | ?.transactionHash, 48 | createdAtTimestamp: String(timestamp * 1000), 49 | createdAtBlockNumber: childChainTxReceipt.blockNumber, 50 | timeoutTimestamp: String(Number(timestamp) + SEVEN_DAYS_IN_SECONDS), 51 | deposit: String(retryableMessage.messageData.l2CallValue), // eth amount 52 | status: ParentToChildMessageStatus[status], 53 | retryTo: retryableMessage.messageData.destAddress, 54 | retryData: retryableMessage.messageData.data, 55 | gasFeeCap: (childChainTx.maxFeePerGas ?? BigNumber.from(0)).toNumber(), 56 | gasLimit: childChainTx.gasLimit.toNumber(), 57 | } 58 | 59 | return childChainTicketReport 60 | } 61 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/retryableChecker.ts: -------------------------------------------------------------------------------- 1 | /* 2 | This file contains the logic for checking retryables in the given block range for the chain 3 | */ 4 | 5 | import { providers } from 'ethers' 6 | import { 7 | ParentTransactionReceipt, 8 | ParentToChildMessageStatus, 9 | } from '@arbitrum/sdk' 10 | import { ChildNetwork } from '../../utils' 11 | import { 12 | getMessageDeliveredEventData, 13 | getDepositInitiatedLogs, 14 | } from './depositEventFetcher' 15 | import { 16 | getParentChainRetryableReport, 17 | getChildChainRetryableReport, 18 | } from './reportGenerator' 19 | import { getExplorerUrlPrefixes } from '../../utils' 20 | import { OnFailedRetryableFound, OnRedeemedRetryableFound } from './types' 21 | import { getTokenDepositData } from './tokenDataFetcher' 22 | import { SEVEN_DAYS_IN_SECONDS } from '@arbitrum/sdk/dist/lib/dataEntities/constants' 23 | 24 | export const checkRetryables = async ( 25 | parentChainProvider: providers.Provider, 26 | childChainProvider: providers.Provider, 27 | childChain: ChildNetwork, 28 | bridgeAddress: string, 29 | fromBlock: number, 30 | toBlock: number, 31 | enableAlerting: boolean, 32 | onFailedRetryableFound?: OnFailedRetryableFound, 33 | onRedeemedRetryableFound?: OnRedeemedRetryableFound 34 | ): Promise => { 35 | let retryablesFound = false 36 | 37 | const messageDeliveredLogs = await getMessageDeliveredEventData( 38 | bridgeAddress, 39 | { fromBlock, toBlock }, 40 | parentChainProvider 41 | ) 42 | 43 | // used for finding the token-details associated with a deposit, if any 44 | const depositsInitiatedLogs = await getDepositInitiatedLogs({ 45 | fromBlock, 46 | toBlock, 47 | parentChainProvider, 48 | gatewayAddresses: { 49 | parentErc20Gateway: childChain.tokenBridge!.parentErc20Gateway, 50 | parentCustomGateway: childChain.tokenBridge!.parentCustomGateway, 51 | parentWethGateway: childChain.tokenBridge!.parentWethGateway, 52 | }, 53 | }) 54 | 55 | const uniqueTxHashes = new Set() 56 | for (let messageDeliveredLog of messageDeliveredLogs) { 57 | const { transactionHash: parentTxHash } = messageDeliveredLog 58 | uniqueTxHashes.add(parentTxHash) 59 | } 60 | 61 | const { PARENT_CHAIN_TX_PREFIX, CHILD_CHAIN_TX_PREFIX } = 62 | getExplorerUrlPrefixes(childChain) 63 | 64 | // for each parent-chain-transaction found, extract the Retryables thus created by it 65 | for (const parentTxHash of uniqueTxHashes) { 66 | const parentTxReceipt = await parentChainProvider.getTransactionReceipt( 67 | parentTxHash 68 | ) 69 | const arbParentTxReceipt = new ParentTransactionReceipt(parentTxReceipt) 70 | const retryables = await arbParentTxReceipt.getParentToChildMessages( 71 | childChainProvider 72 | ) 73 | 74 | if (retryables.length > 0) { 75 | console.log( 76 | `${retryables.length} retryable${ 77 | retryables.length === 1 ? '' : 's' 78 | } found for ${ 79 | childChain.name 80 | } chain. Checking their status:\n\nParentChainTxHash: ${ 81 | PARENT_CHAIN_TX_PREFIX + parentTxHash 82 | }` 83 | ) 84 | console.log('----------------------------------------------------------') 85 | 86 | // for each retryable, extract the detail for it's status / redemption 87 | for (let msgIndex = 0; msgIndex < retryables.length; msgIndex++) { 88 | const retryableMessage = retryables[msgIndex] 89 | const retryableTicketId = retryableMessage.retryableCreationId 90 | let status = await retryableMessage.status() 91 | 92 | // if we find a successful Retryable, call `onRedeemedRetryableFound()` 93 | if (status === ParentToChildMessageStatus.REDEEMED) { 94 | if (enableAlerting && onRedeemedRetryableFound) { 95 | await onRedeemedRetryableFound({ 96 | ChildTx: `${CHILD_CHAIN_TX_PREFIX}${retryableMessage.retryableCreationId}`, 97 | ParentTx: `${PARENT_CHAIN_TX_PREFIX}${parentTxHash}`, 98 | createdAt: Date.now(), // fallback; won't overwrite real one 99 | timeout: Date.now() + SEVEN_DAYS_IN_SECONDS * 1000, 100 | status: 'Executed', 101 | priority: 'Unset', 102 | metadata: { 103 | tokensDeposited: undefined, 104 | gasPriceProvided: '-', 105 | gasPriceAtCreation: undefined, 106 | gasPriceNow: '-', 107 | l2CallValue: '-', 108 | }, 109 | }) 110 | } 111 | } 112 | 113 | // if a Retryable is not in a successful state, extract it's details and call `onFailedRetryableFound()` 114 | if (status !== ParentToChildMessageStatus.REDEEMED) { 115 | const childChainTx = await childChainProvider.getTransaction( 116 | retryableTicketId 117 | ) 118 | const childChainTxReceipt = 119 | await childChainProvider.getTransactionReceipt( 120 | retryableMessage.retryableCreationId 121 | ) 122 | 123 | if (!childChainTxReceipt) { 124 | // if child-chain tx is very recent, the tx receipt might not be found yet 125 | // if not handled, this will result in `undefined` error while trying to extract retryable details 126 | console.log( 127 | `${msgIndex + 1}. ${ 128 | ParentToChildMessageStatus[status] 129 | }:\nChildChainTxHash: ${ 130 | CHILD_CHAIN_TX_PREFIX + retryableTicketId 131 | } (Receipt not found yet)` 132 | ) 133 | continue 134 | } 135 | 136 | const parentChainRetryableReport = getParentChainRetryableReport( 137 | arbParentTxReceipt, 138 | retryableMessage 139 | ) 140 | const childChainRetryableReport = await getChildChainRetryableReport({ 141 | retryableMessage, 142 | childChainTx, 143 | childChainTxReceipt, 144 | childChainProvider, 145 | }) 146 | const tokenDepositData = await getTokenDepositData({ 147 | childChainTx, 148 | retryableMessage, 149 | arbParentTxReceipt, 150 | depositsInitiatedLogs, 151 | parentChainProvider, 152 | }) 153 | 154 | // Call the provided callback if it exists 155 | if (enableAlerting && onFailedRetryableFound) { 156 | await onFailedRetryableFound({ 157 | parentChainRetryableReport, 158 | childChainRetryableReport, 159 | tokenDepositData, 160 | childChain, 161 | }) 162 | } 163 | } 164 | 165 | // format the result message 166 | console.log( 167 | `${msgIndex + 1}. ${ 168 | ParentToChildMessageStatus[status] 169 | }:\nChildChainTxHash: ${CHILD_CHAIN_TX_PREFIX + retryableTicketId}` 170 | ) 171 | console.log( 172 | '----------------------------------------------------------' 173 | ) 174 | } 175 | retryablesFound = true // Set to true if retryables are found 176 | } 177 | } 178 | 179 | return retryablesFound 180 | } 181 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/retryableCheckerMode.ts: -------------------------------------------------------------------------------- 1 | /* 2 | You can check retryables in two modes: 3 | 1. One-off mode: Check retryables for a specific block range and exit 4 | 2. Continuous mode / Watch mode: Check retryables for the latest blocks, and continue checking for new blocks as they are added to the chain 5 | 6 | ``` 7 | 8 | */ 9 | 10 | import { SEVEN_DAYS_IN_SECONDS } from '@arbitrum/sdk/dist/lib/dataEntities/constants' 11 | import { 12 | CheckRetryablesOneOffParams, 13 | CheckRetryablesContinuousParams, 14 | } from './types' 15 | import { ChildNetwork } from '../../utils' 16 | import { checkRetryables } from './retryableChecker' 17 | 18 | export const getParentChainBlockTime = (childChain: ChildNetwork) => { 19 | const parentChainId = childChain.parentChainId 20 | 21 | // for Ethereum / Sepolia / Holesky 22 | if ( 23 | parentChainId === 1 || 24 | parentChainId === 11155111 || 25 | parentChainId === 17000 26 | ) { 27 | return 12 28 | } 29 | 30 | // for Base / Base Sepolia 31 | if (parentChainId === 8453 || parentChainId === 84532) { 32 | return 2 33 | } 34 | 35 | // for arbitrum networks, return the standard block time 36 | return 2 // ARB_MINIMUM_BLOCK_TIME_IN_SECONDS 37 | } 38 | 39 | export const checkRetryablesOneOff = async ({ 40 | parentChainProvider, 41 | childChainProvider, 42 | childChain, 43 | fromBlock, 44 | toBlock, 45 | enableAlerting, 46 | onFailedRetryableFound, 47 | onRedeemedRetryableFound, 48 | }: CheckRetryablesOneOffParams): Promise => { 49 | if (toBlock === 0) { 50 | try { 51 | const currentBlock = await parentChainProvider.getBlockNumber() 52 | if (!currentBlock) { 53 | throw new Error('Failed to retrieve the latest block.') 54 | } 55 | toBlock = currentBlock 56 | 57 | // if no `fromBlock` or `toBlock` is provided, monitor for 14 days worth of blocks only 58 | // only enforce `fromBlock` check if we want to report the ticket to the alerting system 59 | if (fromBlock === 0 && enableAlerting) { 60 | fromBlock = 61 | toBlock - 62 | (2 * SEVEN_DAYS_IN_SECONDS) / getParentChainBlockTime(childChain) 63 | console.log( 64 | `Alerting mode enabled: limiting block-range to last 14 days [${fromBlock} to ${toBlock}]` 65 | ) 66 | } 67 | } catch (error) { 68 | console.error( 69 | `Error getting the latest block: ${(error as Error).message}` 70 | ) 71 | throw error 72 | } 73 | } 74 | 75 | const MAX_BLOCKS_TO_PROCESS = 5000 // event_logs can only be processed in batches of MAX_BLOCKS_TO_PROCESS blocks 76 | 77 | // if the block range provided is >=MAX_BLOCKS_TO_PROCESS, we might get rate limited while fetching logs from the node 78 | // so we break down the range into smaller chunks and process them sequentially 79 | // generate the final ranges' batches to process [ [fromBlock, toBlock], [fromBlock, toBlock], ...] 80 | const ranges = [] 81 | for (let i = fromBlock; i <= toBlock; i += MAX_BLOCKS_TO_PROCESS) { 82 | ranges.push([i, Math.min(i + MAX_BLOCKS_TO_PROCESS - 1, toBlock)]) 83 | } 84 | 85 | let retryablesFound = false 86 | for (const range of ranges) { 87 | retryablesFound = 88 | (await checkRetryables( 89 | parentChainProvider, 90 | childChainProvider, 91 | childChain, 92 | childChain.ethBridge.bridge, 93 | range[0], 94 | range[1], 95 | enableAlerting, 96 | onFailedRetryableFound, 97 | onRedeemedRetryableFound 98 | )) || retryablesFound // the final `retryablesFound` value is the OR of all the `retryablesFound` for ranges 99 | } 100 | 101 | return toBlock 102 | } 103 | 104 | export const checkRetryablesContinuous = async ({ 105 | parentChainProvider, 106 | childChainProvider, 107 | childChain, 108 | fromBlock, 109 | toBlock, 110 | enableAlerting, 111 | continuous, 112 | onFailedRetryableFound, 113 | onRedeemedRetryableFound, 114 | }: CheckRetryablesContinuousParams) => { 115 | const processingDurationInSeconds = 180 116 | let isContinuous = continuous 117 | const startTime = Date.now() 118 | 119 | // Function to process blocks and check for retryables 120 | const processBlocks = async () => { 121 | const lastBlockChecked = await checkRetryablesOneOff({ 122 | parentChainProvider, 123 | childChainProvider, 124 | childChain, 125 | fromBlock, 126 | toBlock, 127 | enableAlerting, 128 | onFailedRetryableFound, 129 | onRedeemedRetryableFound, 130 | }) 131 | console.log('Check completed for block:', lastBlockChecked) 132 | fromBlock = lastBlockChecked + 1 133 | console.log('Continuing from block:', fromBlock) 134 | 135 | toBlock = await parentChainProvider.getBlockNumber() 136 | console.log(`Processed blocks up to ${lastBlockChecked}`) 137 | 138 | return lastBlockChecked 139 | } 140 | 141 | // Continuous loop for checking retryables 142 | while (isContinuous) { 143 | const lastBlockChecked = await processBlocks() 144 | 145 | if (lastBlockChecked >= toBlock) { 146 | // Wait for a short interval before checking again 147 | await new Promise(resolve => setTimeout(resolve, 1000)) 148 | } 149 | 150 | const currentTime = Date.now() 151 | const elapsedTimeInSeconds = Math.floor((currentTime - startTime) / 1000) 152 | 153 | if (elapsedTimeInSeconds >= processingDurationInSeconds) { 154 | isContinuous = false 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/tokenDataFetcher.ts: -------------------------------------------------------------------------------- 1 | import { FetchedEvent } from '@arbitrum/sdk/dist/lib/utils/eventFetcher' 2 | import { TypedEvent } from '@arbitrum/sdk/dist/lib/abi/common' 3 | import { providers } from 'ethers' 4 | import { 5 | ParentTransactionReceipt, 6 | ParentToChildMessageReader, 7 | } from '@arbitrum/sdk' 8 | import { ERC20__factory } from '@arbitrum/sdk/dist/lib/abi/factories/ERC20__factory' 9 | import { TokenDepositData } from './types' 10 | 11 | export const getTokenDepositData = async ({ 12 | childChainTx, 13 | retryableMessage, 14 | arbParentTxReceipt, 15 | depositsInitiatedLogs, 16 | parentChainProvider, 17 | }: { 18 | childChainTx: providers.TransactionResponse 19 | retryableMessage: ParentToChildMessageReader 20 | arbParentTxReceipt: ParentTransactionReceipt 21 | depositsInitiatedLogs: FetchedEvent>[] 22 | parentChainProvider: providers.Provider 23 | }): Promise => { 24 | let parentChainErc20Address: string | undefined, 25 | tokenAmount: string | undefined, 26 | tokenDepositData: TokenDepositData | undefined 27 | 28 | try { 29 | const retryableMessageData = childChainTx.data 30 | const retryableBody = retryableMessageData.split('0xc9f95d32')[1] 31 | const requestId = '0x' + retryableBody.slice(0, 64) 32 | const depositsInitiatedEvent = depositsInitiatedLogs.find( 33 | log => log.topics[3] === requestId 34 | ) 35 | parentChainErc20Address = depositsInitiatedEvent?.event[0] 36 | tokenAmount = depositsInitiatedEvent?.event[4]?.toString() 37 | } catch (e) { 38 | console.log(e) 39 | } 40 | 41 | if (parentChainErc20Address) { 42 | try { 43 | const erc20 = ERC20__factory.connect( 44 | parentChainErc20Address, 45 | parentChainProvider 46 | ) 47 | const [symbol, decimals] = await Promise.all([ 48 | erc20.symbol(), 49 | erc20.decimals(), 50 | ]) 51 | tokenDepositData = { 52 | l2TicketId: retryableMessage.retryableCreationId, 53 | tokenAmount, 54 | sender: arbParentTxReceipt.from, 55 | l1Token: { 56 | symbol, 57 | decimals, 58 | id: parentChainErc20Address, 59 | }, 60 | } 61 | } catch (e) { 62 | console.log('failed to fetch token data', e) 63 | } 64 | } 65 | 66 | return tokenDepositData 67 | } 68 | -------------------------------------------------------------------------------- /packages/retryable-monitor/core/types.ts: -------------------------------------------------------------------------------- 1 | import { ChildNetwork } from '../../utils' 2 | import { providers } from 'ethers' 3 | 4 | // Type for options passed to findRetryables function 5 | export interface FindRetryablesOptions { 6 | fromBlock: number 7 | toBlock: number 8 | continuous: boolean 9 | configPath: string 10 | enableAlerting: boolean 11 | writeToNotion: boolean 12 | } 13 | 14 | export interface CheckRetryablesOneOffParams { 15 | parentChainProvider: providers.Provider 16 | childChainProvider: providers.Provider 17 | childChain: ChildNetwork 18 | fromBlock: number 19 | toBlock: number 20 | enableAlerting: boolean 21 | onFailedRetryableFound: OnFailedRetryableFound 22 | onRedeemedRetryableFound: OnRedeemedRetryableFound 23 | } 24 | 25 | export interface CheckRetryablesContinuousParams 26 | extends CheckRetryablesOneOffParams { 27 | continuous: boolean 28 | } 29 | 30 | export interface ParentChainTicketReport { 31 | id: string 32 | transactionHash: string 33 | sender: string 34 | retryableTicketID: string 35 | } 36 | 37 | export interface ChildChainTicketReport { 38 | id: string 39 | retryTxHash?: string 40 | createdAtTimestamp: string 41 | createdAtBlockNumber: number 42 | timeoutTimestamp: string 43 | deposit: string 44 | status: string 45 | retryTo: string 46 | retryData: string 47 | gasFeeCap: number 48 | gasLimit: number 49 | } 50 | 51 | export interface TokenDepositData { 52 | l2TicketId: string 53 | tokenAmount?: string 54 | sender: string 55 | l1Token: { 56 | symbol: string 57 | decimals: number 58 | id: string 59 | } 60 | } 61 | 62 | export interface OnRetryableFoundParams { 63 | ChildTx: string 64 | ParentTx: string 65 | createdAt: number 66 | timeout?: number 67 | status:string 68 | priority?: 'High' | 'Medium' | 'Low' | 'Unset' 69 | decision?: string 70 | metadata?: { 71 | tokensDeposited?: string 72 | gasPriceProvided: string 73 | gasPriceAtCreation?: string 74 | gasPriceNow: string 75 | l2CallValue: string 76 | createdAt?: number 77 | decision?: string 78 | 79 | } 80 | } 81 | 82 | export interface OnFailedRetryableFoundParams { 83 | parentChainRetryableReport: ParentChainTicketReport 84 | childChainRetryableReport: ChildChainTicketReport 85 | tokenDepositData?: TokenDepositData 86 | childChain: ChildNetwork 87 | } 88 | 89 | export type OnFailedRetryableFound = ( 90 | params: OnFailedRetryableFoundParams 91 | ) => Promise 92 | 93 | export type OnRedeemedRetryableFound = ( 94 | params: OnRetryableFoundParams 95 | ) => Promise 96 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/handleFailedRetryablesFound.ts: -------------------------------------------------------------------------------- 1 | import { BigNumber, ethers, providers } from 'ethers' 2 | import { getExplorerUrlPrefixes } from '../../utils' 3 | import { OnFailedRetryableFoundParams } from '../core/types' 4 | import { reportFailedRetryables } from './reportFailedRetryables' 5 | import { syncRetryableToNotion } from './notion/syncRetryableToNotion' 6 | import { 7 | formatL2Callvalue, 8 | getGasInfo, 9 | getTokenPrice, 10 | } from './slack/slackMessageFormattingUtils' 11 | 12 | export const handleFailedRetryablesFound = async ( 13 | ticket: OnFailedRetryableFoundParams, 14 | writeToNotion: boolean 15 | ) => { 16 | // report the Retryable in Slack 17 | await reportFailedRetryables(ticket) 18 | 19 | // sync the Retryable to Notion 20 | if (writeToNotion) { 21 | const { 22 | tokenDepositData, 23 | childChainRetryableReport, 24 | childChain, 25 | parentChainRetryableReport, 26 | } = ticket 27 | 28 | const childChainProvider = new providers.JsonRpcProvider( 29 | String(childChain.orbitRpcUrl) 30 | ) 31 | 32 | const parentChainProvider = new providers.JsonRpcProvider( 33 | String(childChain.parentRpcUrl) 34 | ) 35 | 36 | const formattedCallValueFull = await formatL2Callvalue( 37 | childChainRetryableReport, 38 | childChain, 39 | parentChainProvider 40 | ) 41 | 42 | const l2CallValueFormatted = formattedCallValueFull 43 | .replace('\n\t *Child chain callvalue:* ', '') 44 | .trim() 45 | 46 | let formattedTokenString: string | undefined = undefined 47 | if (tokenDepositData?.tokenAmount && tokenDepositData?.l1Token) { 48 | const amount = BigNumber.from(tokenDepositData.tokenAmount) 49 | const decimals = tokenDepositData.l1Token.decimals 50 | const symbol = tokenDepositData.l1Token.symbol 51 | const address = tokenDepositData.l1Token.id 52 | 53 | const humanAmount = Number(amount) / 10 ** decimals 54 | const price = (await getTokenPrice(address)) ?? 1 55 | const usdValue = humanAmount * price 56 | 57 | formattedTokenString = `${humanAmount.toFixed( 58 | 6 59 | )} ${symbol} ($${usdValue.toFixed(2)}) (${address})` 60 | } 61 | const { l2GasPrice, l2GasPriceAtCreation } = await getGasInfo( 62 | childChainRetryableReport.createdAtBlockNumber, 63 | childChainRetryableReport.id, 64 | childChainProvider 65 | ) 66 | 67 | const gasPriceProvided = `${ethers.utils.formatUnits( 68 | childChainRetryableReport.gasFeeCap, 69 | 'gwei' 70 | )} gwei` 71 | const gasPriceAtCreation = l2GasPriceAtCreation 72 | ? `${ethers.utils.formatUnits(l2GasPriceAtCreation, 'gwei')} gwei` 73 | : undefined 74 | const gasPriceNow = `${ethers.utils.formatUnits(l2GasPrice, 'gwei')} gwei` 75 | 76 | const { PARENT_CHAIN_TX_PREFIX, CHILD_CHAIN_TX_PREFIX } = 77 | getExplorerUrlPrefixes(childChain) 78 | 79 | await syncRetryableToNotion({ 80 | ChildTx: `${CHILD_CHAIN_TX_PREFIX}${childChainRetryableReport.id}`, 81 | ParentTx: `${PARENT_CHAIN_TX_PREFIX}${parentChainRetryableReport.transactionHash}`, 82 | createdAt: Number(childChainRetryableReport.createdAtTimestamp) * 1000, 83 | timeout: Number(childChainRetryableReport.timeoutTimestamp) * 1000, 84 | status: childChainRetryableReport.status, 85 | priority: 'Unset', 86 | metadata: { 87 | tokensDeposited: formattedTokenString, 88 | gasPriceProvided, 89 | gasPriceAtCreation, 90 | gasPriceNow, 91 | l2CallValue: l2CallValueFormatted, 92 | decision: 'Triage' 93 | }, 94 | }) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/handleRedeemedRetryablesFound.ts: -------------------------------------------------------------------------------- 1 | import { OnRetryableFoundParams } from '../core/types' 2 | import { syncRetryableToNotion } from './notion/syncRetryableToNotion' 3 | 4 | export const handleRedeemedRetryablesFound = async ( 5 | ticket: OnRetryableFoundParams, 6 | writeToNotion: boolean 7 | ) => { 8 | if (writeToNotion) { 9 | await syncRetryableToNotion(ticket) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/notion/alertUntriagedRetraybles.ts: -------------------------------------------------------------------------------- 1 | import { notionClient, databaseId } from './createNotionClient' 2 | import { postSlackMessage } from '../slack/postSlackMessage' 3 | 4 | export const alertUntriagedNotionRetryables = async () => { 5 | const response = await notionClient.databases.query({ 6 | database_id: databaseId, 7 | page_size: 100, 8 | filter: { 9 | and: [ 10 | { 11 | or: [ 12 | { property: 'Decision', select: { equals: 'Triage' } }, 13 | { property: 'Decision', select: { equals: 'Should Redeem' } }, 14 | ], 15 | }, 16 | { 17 | property: 'Status', 18 | select: { does_not_equal: 'Executed' }, 19 | }, 20 | ], 21 | }, 22 | }) 23 | 24 | for (const page of response.results) { 25 | const props = (page as any).properties 26 | const timeoutStr = props?.timeoutTimestamp?.date?.start 27 | const retryableUrl = 28 | props?.ChildTx?.title?.[0]?.text?.content || '(unknown)' 29 | const decision = props?.Decision?.select?.name || '(unknown)' 30 | 31 | await postSlackMessage({ 32 | message: `⚠️ Retryable ticket still pending:\n• Retryable: ${retryableUrl}\n• Decision: ${decision}\n• Timeout: ${timeoutStr}`, 33 | }) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/notion/createNotionClient.ts: -------------------------------------------------------------------------------- 1 | import { Client } from '@notionhq/client' 2 | import dotenv from 'dotenv' 3 | 4 | dotenv.config() 5 | 6 | export const notionClient = new Client({ 7 | auth: process.env.RETRYABLE_MONITORING_NOTION_TOKEN, 8 | }) 9 | export const databaseId = process.env.RETRYABLE_MONITORING_NOTION_DB_ID! 10 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/notion/syncRetryableToNotion.ts: -------------------------------------------------------------------------------- 1 | import { notionClient } from './createNotionClient' 2 | import { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' 3 | import { postSlackMessage } from '../slack/postSlackMessage' 4 | import { OnRetryableFoundParams } from '../../core/types' 5 | 6 | const databaseId = process.env.RETRYABLE_MONITORING_NOTION_DB_ID! 7 | 8 | export async function syncRetryableToNotion( 9 | input: OnRetryableFoundParams 10 | ): Promise<{ id: string; status: string; isNew: boolean } | undefined> { 11 | const { 12 | ChildTx, 13 | ParentTx, 14 | createdAt, 15 | status, 16 | priority = 'Unset', 17 | metadata, 18 | } = input 19 | 20 | try { 21 | const search = await notionClient.databases.query({ 22 | database_id: databaseId, 23 | filter: { 24 | property: 'ChildTx', 25 | rich_text: { 26 | equals: ChildTx, 27 | }, 28 | }, 29 | }) 30 | 31 | const isRetryableFoundInNotion = search.results.length > 0 32 | 33 | const rawCreatedAt = metadata?.createdAt ?? createdAt 34 | const createdAtMs = 35 | rawCreatedAt > 1e14 36 | ? Math.floor(rawCreatedAt / 1000) 37 | : rawCreatedAt > 1e12 38 | ? rawCreatedAt 39 | : rawCreatedAt > 1e10 40 | ? rawCreatedAt 41 | : rawCreatedAt * 1000 42 | 43 | const notionProps: Record = { 44 | ParentTx: { rich_text: [{ text: { content: ParentTx } }] }, 45 | CreatedAt: { date: { start: new Date(createdAtMs).toISOString() } }, 46 | Priority: { select: { name: priority } }, 47 | } 48 | 49 | if (input.timeout) { 50 | notionProps['timeoutTimestamp'] = { 51 | date: { start: new Date(input.timeout).toISOString() }, 52 | } 53 | } 54 | 55 | if (metadata) { 56 | notionProps['GasPriceProvided'] = { 57 | rich_text: [{ text: { content: metadata.gasPriceProvided } }], 58 | } 59 | notionProps['GasPriceAtCreation'] = { 60 | rich_text: [{ text: { content: metadata.gasPriceAtCreation ?? 'N/A' } }], 61 | } 62 | notionProps['GasPriceNow'] = { 63 | rich_text: [{ text: { content: metadata.gasPriceNow } }], 64 | } 65 | notionProps['TotalRetryableDeposit'] = { 66 | rich_text: [{ text: { content: metadata.l2CallValue } }], 67 | } 68 | if (metadata.tokensDeposited) { 69 | notionProps['TokensDeposited'] = { 70 | rich_text: [{ text: { content: metadata.tokensDeposited } }], 71 | } 72 | } 73 | } 74 | 75 | if (isRetryableFoundInNotion) { 76 | const page = search.results[0] 77 | 78 | if (!('properties' in page)) { 79 | const errorMessage = `⚠️ Notion sync failed: page for ${ChildTx} is missing 'properties'. Skipping update.` 80 | console.error(errorMessage) 81 | await postSlackMessage({ message: errorMessage }) 82 | return 83 | } 84 | 85 | const props = (page as PageObjectResponse).properties 86 | const statusProp = props?.Status 87 | const decisionProp = props?.Decision 88 | 89 | const currentStatus = 90 | statusProp?.type === 'select' && statusProp.select 91 | ? statusProp.select.name 92 | : undefined 93 | const currentDecision = 94 | decisionProp?.type === 'select' && decisionProp.select 95 | ? decisionProp.select.name 96 | : undefined 97 | 98 | // ✅ Handle Executed updates 99 | if (status === 'Executed') { 100 | const executedProps: Record = { 101 | Status: { select: { name: 'Executed' } }, 102 | } 103 | 104 | if (input.timeout) { 105 | executedProps['timeoutTimestamp'] = { 106 | date: { start: new Date(input.timeout).toISOString() }, 107 | } 108 | } 109 | 110 | if (!currentDecision && metadata?.decision) { 111 | executedProps['Decision'] = { 112 | select: { name: metadata.decision }, 113 | } 114 | } 115 | 116 | await notionClient.pages.update({ 117 | page_id: page.id, 118 | properties: executedProps, 119 | }) 120 | 121 | return { id: page.id, status: 'Executed', isNew: false } 122 | } 123 | 124 | notionProps['Status'] = { select: { name: status } } 125 | 126 | // Only set Decision if it's missing 127 | if (!currentDecision && metadata?.decision) { 128 | notionProps['Decision'] = { 129 | select: { name: metadata.decision }, 130 | } 131 | } 132 | 133 | await notionClient.pages.update({ 134 | page_id: page.id, 135 | properties: notionProps, 136 | }) 137 | 138 | return { id: page.id, status: currentStatus ?? status, isNew: false } 139 | } 140 | 141 | // If not found and Executed, skip creation 142 | if (!isRetryableFoundInNotion && status === 'Executed') { 143 | return undefined 144 | } 145 | 146 | const created = await notionClient.pages.create({ 147 | parent: { database_id: databaseId }, 148 | properties: { 149 | ChildTx: { title: [{ text: { content: ChildTx } }] }, 150 | Status: { select: { name: status } }, 151 | ...(metadata?.decision ? { Decision: { select: { name: metadata.decision } } } : {}), 152 | ...notionProps, 153 | }, 154 | }) 155 | 156 | return { id: created.id, status, isNew: true } 157 | } catch (err) { 158 | console.error('❌ Failed to sync ticket to Notion:', err) 159 | return undefined 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/reportFailedRetryables.ts: -------------------------------------------------------------------------------- 1 | import { providers } from 'ethers' 2 | import { ChildNetwork } from 'utils' 3 | import { 4 | ChildChainTicketReport, 5 | ParentChainTicketReport, 6 | TokenDepositData, 7 | } from '../core/types' 8 | import { postSlackMessage } from './slack/postSlackMessage' 9 | import { generateFailedRetryableSlackMessage } from './slack/slackMessageGenerator' 10 | 11 | export const reportFailedRetryables = async ({ 12 | parentChainRetryableReport, 13 | childChainRetryableReport, 14 | tokenDepositData, 15 | childChain, 16 | }: { 17 | parentChainRetryableReport: ParentChainTicketReport 18 | childChainRetryableReport: ChildChainTicketReport 19 | tokenDepositData?: TokenDepositData 20 | childChain: ChildNetwork 21 | }) => { 22 | const t = childChainRetryableReport 23 | const now = Math.floor(new Date().getTime() / 1000) // now in s 24 | 25 | // don't report tickets which are not yet scheduled if they have been created in last 2h 26 | const reportingPeriodForNotScheduled = 2 * 60 * 60 // 2 hours in s 27 | if ( 28 | t.status == 'NOT_YET_CREATED' && 29 | now - +t.createdAtTimestamp < reportingPeriodForNotScheduled 30 | ) { 31 | return 32 | } 33 | 34 | // don't report tickets which expired more than 2 days ago 35 | const reportingPeriodForExpired = 2 * 24 * 60 * 60 // 2 days in s 36 | if ( 37 | t.status == 'EXPIRED' && 38 | now - +t.timeoutTimestamp > reportingPeriodForExpired 39 | ) { 40 | return 41 | } 42 | 43 | const childChainProvider = new providers.JsonRpcProvider( 44 | String(childChain.orbitRpcUrl) 45 | ) 46 | 47 | const parentChainProvider = new providers.JsonRpcProvider( 48 | String(childChain.parentRpcUrl) 49 | ) 50 | 51 | try { 52 | const reportStr = await generateFailedRetryableSlackMessage({ 53 | parentChainRetryableReport, 54 | childChainRetryableReport, 55 | tokenDepositData, 56 | childChain, 57 | parentChainProvider, 58 | childChainProvider, 59 | }) 60 | 61 | postSlackMessage({ message: reportStr }) 62 | } catch (e) { 63 | console.log('Could not send slack message', e) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/slack/postSlackMessage.ts: -------------------------------------------------------------------------------- 1 | import { postSlackMessage as commonPostSlackMessage } from '../../../utils/postSlackMessage' 2 | 3 | const slackToken = process.env.RETRYABLE_MONITORING_SLACK_TOKEN 4 | const slackChannel = process.env.RETRYABLE_MONITORING_SLACK_CHANNEL 5 | 6 | export const postSlackMessage = ({ message }: { message: string }) => { 7 | if (!slackToken) throw new Error(`Slack token is required.`) 8 | if (!slackChannel) throw new Error(`Slack channel is required.`) 9 | 10 | if (process.env.NODE_ENV === 'DEV') return 11 | if (process.env.NODE_ENV === 'CI' && message === 'success') return 12 | 13 | commonPostSlackMessage({ 14 | slackToken, 15 | slackChannel, 16 | message, 17 | }) 18 | } 19 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/slack/slackMessageFormattingUtils.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import { ParentToChildMessageStatus } from '@arbitrum/sdk' 3 | import { ERC20__factory } from '@arbitrum/sdk/dist/lib/abi/factories/ERC20__factory' 4 | import { BigNumber, ethers } from 'ethers' 5 | import { ArbGasInfo__factory } from '@arbitrum/sdk/dist/lib/abi/factories/ArbGasInfo__factory' 6 | import { 7 | ARB_GAS_INFO, 8 | ARB_RETRYABLE_TX_ADDRESS, 9 | } from '@arbitrum/sdk/dist/lib/dataEntities/constants' 10 | import { ArbRetryableTx__factory } from '@arbitrum/sdk/dist/lib/abi/factories/ArbRetryableTx__factory' 11 | import { Provider } from '@ethersproject/abstract-provider' 12 | import { 13 | ChildChainTicketReport, 14 | ParentChainTicketReport, 15 | TokenDepositData, 16 | } from '../../core/types' 17 | import { ChildNetwork, getExplorerUrlPrefixes } from '../../../utils' 18 | 19 | /** 20 | * 21 | * 22 | * ALL the `util` functions copied over from the original monitoring repo, edited to fit the L2-L3 types 23 | * https://github.com/OffchainLabs/arb-monitoring/blob/master/lib/failed_retryables.ts 24 | * 25 | * 26 | */ 27 | 28 | let ethPriceCache: number 29 | let tokenPriceCache: { [key: string]: number } = {} 30 | 31 | export const getTimeDifference = (timestampInSeconds: number) => { 32 | const now = new Date().getTime() / 1000 33 | const difference = timestampInSeconds - now 34 | 35 | const days = Math.floor(difference / (24 * 60 * 60)) 36 | const hours = Math.floor((difference % (24 * 60 * 60)) / (60 * 60)) 37 | const minutes = Math.floor((difference % (60 * 60)) / 60) 38 | const seconds = Math.floor(difference % 60) 39 | 40 | if (days > 0) { 41 | return `${days}days : ${hours}h : ${minutes}min : ${seconds}s` 42 | } else if (hours > 0) { 43 | return `${hours}h : ${minutes}min : ${seconds}s` 44 | } else if (minutes > 0) { 45 | return `${minutes}min : ${seconds}s` 46 | } else { 47 | return `${seconds}s` 48 | } 49 | } 50 | 51 | export const formatPrefix = ( 52 | ticket: ChildChainTicketReport, 53 | childChainName: string 54 | ) => { 55 | const now = Math.floor(new Date().getTime() / 1000) // now in s 56 | 57 | let prefix 58 | switch (ticket.status) { 59 | case ParentToChildMessageStatus[ 60 | ParentToChildMessageStatus.FUNDS_DEPOSITED_ON_CHILD 61 | ]: 62 | prefix = `*[${childChainName}] Redeem failed for ticket:*` 63 | break 64 | case ParentToChildMessageStatus[ParentToChildMessageStatus.EXPIRED]: 65 | prefix = `*[${childChainName}] Retryable ticket expired:*` 66 | break 67 | case ParentToChildMessageStatus[ParentToChildMessageStatus.NOT_YET_CREATED]: 68 | prefix = `*[${childChainName}] Retryable ticket hasn't been scheduled:*` 69 | break 70 | default: 71 | prefix = `*[${childChainName}] Found retryable ticket in unrecognized state:*` 72 | } 73 | 74 | // if ticket is about to expire in less than 48h make it a bit dramatic 75 | if (ticket.status == 'RedeemFailed' || ticket.status == 'Created') { 76 | const criticalSoonToExpirePeriod = 2 * 24 * 60 * 60 // 2 days in s 77 | const expiresIn = +ticket.timeoutTimestamp - now 78 | if (expiresIn < criticalSoonToExpirePeriod) { 79 | prefix = `🆘📣 ${prefix} 📣🆘` 80 | } 81 | } 82 | 83 | return prefix 84 | } 85 | 86 | export const formatInitiator = async ( 87 | deposit: TokenDepositData | undefined, 88 | l1Report: ParentChainTicketReport | undefined, 89 | childChain: ChildNetwork 90 | ) => { 91 | const { PARENT_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes(childChain) 92 | 93 | if (deposit !== undefined) { 94 | let msg = '\n\t *Deposit initiated by:* ' 95 | // let text = await getContractName(Chain.ETHEREUM, deposit.sender) 96 | return `${msg}<${PARENT_CHAIN_ADDRESS_PREFIX + deposit.sender}|${ 97 | deposit.sender 98 | }>` 99 | } 100 | 101 | if (l1Report !== undefined) { 102 | let msg = '\n\t *Retryable sender:* ' 103 | // let text = await getContractName(Chain.ETHEREUM, l1Report.sender) 104 | return `${msg}<${PARENT_CHAIN_ADDRESS_PREFIX + l1Report.sender}|${ 105 | l1Report.sender 106 | }>` 107 | } 108 | 109 | return '' 110 | } 111 | 112 | export const formatId = ( 113 | ticket: ChildChainTicketReport, 114 | childChain: ChildNetwork 115 | ) => { 116 | let msg = '\n\t *Child chain ticket creation TX:* ' 117 | 118 | if (ticket.id == null) { 119 | return msg + '-' 120 | } 121 | 122 | const { CHILD_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) 123 | 124 | return `${msg}<${CHILD_CHAIN_TX_PREFIX + ticket.id}|${ticket.id}>` 125 | } 126 | 127 | export const formatL1TX = ( 128 | l1Report: ParentChainTicketReport | undefined, 129 | childChain: ChildNetwork 130 | ) => { 131 | let msg = '\n\t *Parent Chain TX:* ' 132 | 133 | if (l1Report == undefined) { 134 | return msg + '-' 135 | } 136 | 137 | const { PARENT_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) 138 | 139 | return `${msg}<${PARENT_CHAIN_TX_PREFIX + l1Report.transactionHash}|${ 140 | l1Report.transactionHash 141 | }>` 142 | } 143 | 144 | export const formatL2ExecutionTX = ( 145 | ticket: ChildChainTicketReport, 146 | childChain: ChildNetwork 147 | ) => { 148 | let msg = '\n\t *Child chain execution TX:* ' 149 | 150 | if (!ticket.retryTxHash) { 151 | return msg + ': No auto-redeem attempt found' 152 | } 153 | 154 | const { CHILD_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) 155 | 156 | return `${msg}<${CHILD_CHAIN_TX_PREFIX + ticket.retryTxHash}|${ 157 | ticket.retryTxHash 158 | }>` 159 | } 160 | 161 | export const formatL2Callvalue = async ( 162 | ticket: ChildChainTicketReport, 163 | childChain: ChildNetwork, 164 | parentChainProvider: Provider 165 | ) => { 166 | if (childChain.nativeToken) { 167 | const erc20 = ERC20__factory.connect( 168 | childChain.nativeToken, 169 | parentChainProvider 170 | ) 171 | const [symbol, decimals] = await Promise.all([ 172 | erc20.symbol(), 173 | erc20.decimals(), 174 | ]) 175 | 176 | const nativeTokenAmount = ethers.utils.formatUnits(ticket.deposit, decimals) 177 | return `\n\t *Child chain callvalue:* ${nativeTokenAmount} ${symbol} (Gas token: ${symbol})` 178 | } else { 179 | const ethAmount = ethers.utils.formatEther(ticket.deposit) 180 | const depositWorthInUsd = (+ethAmount * (await getEthPrice())).toFixed(2) 181 | return `\n\t *Child chain callvalue:* ${ethAmount} ETH ($${depositWorthInUsd})` 182 | } 183 | } 184 | 185 | export const formatTokenDepositData = async ( 186 | deposit: TokenDepositData | undefined 187 | ) => { 188 | let msg = '\n\t *Tokens deposited:* ' 189 | 190 | if (deposit === undefined) { 191 | return msg + '-' 192 | } 193 | 194 | const amount = deposit.tokenAmount 195 | ? ethers.utils.formatUnits(deposit.tokenAmount, deposit.l1Token.decimals) 196 | : '-' 197 | 198 | const tokenPriceInUSD = await getTokenPrice(deposit.l1Token.id) 199 | if (tokenPriceInUSD !== undefined) { 200 | const depositWorthInUSD = (+amount * tokenPriceInUSD).toFixed(2) 201 | msg = `${msg} ${amount} ${deposit.l1Token.symbol} (\$${depositWorthInUSD}) (${deposit.l1Token.id})` 202 | } else { 203 | msg = `${msg} ${amount} ${deposit.l1Token.symbol} (${deposit.l1Token.id})` 204 | } 205 | 206 | return msg 207 | } 208 | 209 | export const formatDestination = async ( 210 | ticket: ChildChainTicketReport, 211 | childChain: ChildNetwork 212 | ) => { 213 | let msg = `\n\t *Destination:* ` 214 | const { CHILD_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes(childChain) 215 | 216 | return `${msg}<${CHILD_CHAIN_ADDRESS_PREFIX + ticket.retryTo}|${ 217 | ticket.retryTo 218 | }>` 219 | } 220 | 221 | export const formatGasData = async ( 222 | ticket: ChildChainTicketReport, 223 | childChainProvider: Provider 224 | ) => { 225 | const { l2GasPrice, l2GasPriceAtCreation, redeemEstimate } = await getGasInfo( 226 | +ticket.createdAtBlockNumber, 227 | ticket.id, 228 | childChainProvider 229 | ) 230 | 231 | let msg = `\n\t *Gas params:* ` 232 | msg += `\n\t\t gas price provided: ${ethers.utils.formatUnits( 233 | ticket.gasFeeCap, 234 | 'gwei' 235 | )} gwei` 236 | 237 | if (l2GasPriceAtCreation) { 238 | msg += `\n\t\t gas price at ticket creation block: ${ethers.utils.formatUnits( 239 | l2GasPriceAtCreation, 240 | 'gwei' 241 | )} gwei` 242 | } else { 243 | msg += `\n\t\t gas price at ticket creation block: unable to fetch (missing data)` 244 | } 245 | 246 | msg += `\n\t\t gas price now: ${ethers.utils.formatUnits( 247 | l2GasPrice, 248 | 'gwei' 249 | )} gwei` 250 | msg += `\n\t\t gas limit provided: ${ticket.gasLimit}` 251 | 252 | if (redeemEstimate) { 253 | msg += `\n\t\t redeem gas estimate: ${redeemEstimate} ` 254 | } else { 255 | msg += `\n\t\t redeem gas estimate: estimateGas call reverted` 256 | } 257 | 258 | return msg 259 | } 260 | 261 | export const formatCreatedAt = (ticket: ChildChainTicketReport) => { 262 | return `\n\t *Created at:* ${timestampToDate(+ticket.createdAtTimestamp)}` 263 | } 264 | 265 | export const formatExpiration = (ticket: ChildChainTicketReport) => { 266 | let msg = `\n\t *${ 267 | ticket.status == 'Expired' ? `Expired` : `Expires` 268 | } at:* ${timestampToDate(+ticket.timeoutTimestamp)}` 269 | 270 | if (ticket.status == 'RedeemFailed' || ticket.status == 'Created') { 271 | msg = `${msg} (that's ${getTimeDifference( 272 | +ticket.timeoutTimestamp 273 | )} from now)` 274 | } 275 | 276 | return msg 277 | } 278 | 279 | export const getEthPrice = async () => { 280 | if (ethPriceCache !== undefined) { 281 | return ethPriceCache 282 | } 283 | 284 | const url = 285 | 'https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd' 286 | const response = await axios.get(url) 287 | ethPriceCache = +response.data['ethereum'].usd 288 | return ethPriceCache 289 | } 290 | 291 | export const getTokenPrice = async (tokenAddress: string) => { 292 | if (tokenPriceCache[tokenAddress] !== undefined) { 293 | return tokenPriceCache[tokenAddress] 294 | } 295 | 296 | const url = `https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=${tokenAddress}&vs_currencies=usd` 297 | 298 | const response = await axios.get(url) 299 | if (response.data[tokenAddress] == undefined) { 300 | return undefined 301 | } 302 | 303 | tokenPriceCache[tokenAddress] = +response.data[tokenAddress].usd 304 | return tokenPriceCache[tokenAddress] 305 | } 306 | 307 | // Unix timestamp 308 | export const getPastTimestamp = (daysAgoInMs: number) => { 309 | const now = new Date().getTime() 310 | return Math.floor((now - daysAgoInMs) / 1000) 311 | } 312 | 313 | export const timestampToDate = (timestampInSeconds: number) => { 314 | const date = new Date(timestampInSeconds * 1000) 315 | return date.toUTCString() 316 | } 317 | 318 | /** 319 | * Call precompiles to get info about gas price and gas estimation for the TX execution. 320 | * 321 | * @param createdAtBlockNumber 322 | * @param txData 323 | * @returns 324 | */ 325 | export async function getGasInfo( 326 | createdAtBlockNumber: number, 327 | ticketId: string, 328 | childChainProvider: ethers.providers.Provider 329 | ): Promise<{ 330 | l2GasPrice: BigNumber 331 | l2GasPriceAtCreation: BigNumber | undefined 332 | redeemEstimate: BigNumber | undefined 333 | }> { 334 | // connect precompiles 335 | const arbGasInfo = ArbGasInfo__factory.connect( 336 | ARB_GAS_INFO, 337 | childChainProvider 338 | ) 339 | const retryablePrecompile = ArbRetryableTx__factory.connect( 340 | ARB_RETRYABLE_TX_ADDRESS, 341 | childChainProvider 342 | ) 343 | 344 | // get current gas price 345 | const gasComponents = await arbGasInfo.callStatic.getPricesInWei() 346 | const l2GasPrice = gasComponents[5] 347 | 348 | // get gas price when retryable was created 349 | let l2GasPriceAtCreation = undefined 350 | try { 351 | const gasComponentsAtCreation = await arbGasInfo.callStatic.getPricesInWei({ 352 | blockTag: createdAtBlockNumber, 353 | }) 354 | l2GasPriceAtCreation = gasComponentsAtCreation[5] 355 | } catch {} 356 | 357 | // get gas estimation for redeem 358 | let redeemEstimate = undefined 359 | try { 360 | redeemEstimate = await retryablePrecompile.estimateGas.redeem(ticketId) 361 | } catch {} 362 | 363 | return { l2GasPrice, l2GasPriceAtCreation, redeemEstimate } 364 | } 365 | -------------------------------------------------------------------------------- /packages/retryable-monitor/handlers/slack/slackMessageGenerator.ts: -------------------------------------------------------------------------------- 1 | import { providers } from 'ethers' 2 | import { ChildNetwork } from '../../../utils' 3 | import { 4 | ChildChainTicketReport, 5 | ParentChainTicketReport, 6 | TokenDepositData, 7 | } from '../../core/types' 8 | import { 9 | formatPrefix, 10 | formatInitiator, 11 | formatDestination, 12 | formatL1TX, 13 | formatId, 14 | formatL2ExecutionTX, 15 | formatL2Callvalue, 16 | formatTokenDepositData, 17 | formatGasData, 18 | formatCreatedAt, 19 | formatExpiration, 20 | } from './slackMessageFormattingUtils' 21 | 22 | export const generateFailedRetryableSlackMessage = async ({ 23 | parentChainRetryableReport, 24 | childChainRetryableReport, 25 | tokenDepositData, 26 | childChain, 27 | parentChainProvider, 28 | childChainProvider, 29 | }: { 30 | parentChainRetryableReport: ParentChainTicketReport 31 | childChainRetryableReport: ChildChainTicketReport 32 | tokenDepositData?: TokenDepositData 33 | childChain: ChildNetwork 34 | parentChainProvider: providers.Provider 35 | childChainProvider: providers.Provider 36 | }): Promise => { 37 | const t = childChainRetryableReport 38 | const l1Report = parentChainRetryableReport 39 | 40 | // build message to report 41 | return ( 42 | formatPrefix(t, childChain.name) + 43 | (await formatInitiator(tokenDepositData, l1Report, childChain)) + 44 | (await formatDestination(t, childChain)) + 45 | formatL1TX(l1Report, childChain) + 46 | formatId(t, childChain) + 47 | formatL2ExecutionTX(t, childChain) + 48 | (await formatL2Callvalue(t, childChain, parentChainProvider)) + 49 | (await formatTokenDepositData(tokenDepositData)) + 50 | (await formatGasData(t, childChainProvider)) + 51 | formatCreatedAt(t) + 52 | formatExpiration(t) + 53 | '\n=================================================================' 54 | ) 55 | } 56 | -------------------------------------------------------------------------------- /packages/retryable-monitor/index.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import yargs from 'yargs' 3 | import winston from 'winston' 4 | import { providers } from 'ethers' 5 | import { 6 | getArbitrumNetwork, 7 | registerCustomArbitrumNetwork, 8 | } from '@arbitrum/sdk' 9 | import { FindRetryablesOptions } from './core/types' 10 | import { ChildNetwork, DEFAULT_CONFIG_PATH, getConfig } from '../utils' 11 | import { 12 | checkRetryablesOneOff, 13 | checkRetryablesContinuous, 14 | } from './core/retryableCheckerMode' 15 | import { postSlackMessage } from './handlers/slack/postSlackMessage' 16 | import { alertUntriagedNotionRetryables } from './handlers/notion/alertUntriagedRetraybles' 17 | import { handleFailedRetryablesFound } from './handlers/handleFailedRetryablesFound' 18 | import { handleRedeemedRetryablesFound } from './handlers/handleRedeemedRetryablesFound' 19 | 20 | // Path for the log file 21 | const logFilePath = 'logfile.log' 22 | 23 | // Check if the log file exists, if not, create it 24 | try { 25 | fs.accessSync(logFilePath) 26 | } catch (error) { 27 | try { 28 | fs.writeFileSync(logFilePath, '') 29 | console.log(`Log file created: ${logFilePath}`) 30 | } catch (createError) { 31 | console.error(`Error creating log file: ${(createError as Error).message}`) 32 | process.exit(1) 33 | } 34 | } 35 | 36 | // Configure Winston logger 37 | const logger = winston.createLogger({ 38 | format: winston.format.simple(), 39 | transports: [ 40 | new winston.transports.Console(), 41 | new winston.transports.File({ filename: logFilePath }), 42 | ], 43 | }) 44 | 45 | const networkIsRegistered = (networkId: number) => { 46 | try { 47 | getArbitrumNetwork(networkId) 48 | return true 49 | } catch (_) { 50 | return false 51 | } 52 | } 53 | 54 | // Parsing command line arguments using yargs 55 | const options: FindRetryablesOptions = yargs(process.argv.slice(2)) 56 | .options({ 57 | fromBlock: { type: 'number', default: 0 }, 58 | toBlock: { type: 'number', default: 0 }, 59 | continuous: { type: 'boolean', default: false }, 60 | configPath: { type: 'string', default: DEFAULT_CONFIG_PATH }, 61 | enableAlerting: { type: 'boolean', default: false }, 62 | writeToNotion: { type: 'boolean', default: false }, 63 | }) 64 | .strict() 65 | .parseSync() as FindRetryablesOptions 66 | 67 | const config = getConfig({ configPath: options.configPath }) 68 | 69 | // Function to process a child chain and check for retryable transactions 70 | const processChildChain = async ( 71 | parentChainProvider: providers.Provider, 72 | childChainProvider: providers.Provider, 73 | childChain: ChildNetwork, 74 | fromBlock: number, 75 | toBlock: number, 76 | enableAlerting: boolean, 77 | continuous: boolean, 78 | writeToNotion: boolean 79 | ) => { 80 | if (continuous) { 81 | console.log('Activating continuous check for retryables...') 82 | await checkRetryablesContinuous({ 83 | parentChainProvider, 84 | childChainProvider, 85 | childChain, 86 | fromBlock, 87 | toBlock, 88 | enableAlerting, 89 | continuous, 90 | onFailedRetryableFound: async ticket => { 91 | await handleFailedRetryablesFound(ticket, writeToNotion) 92 | }, 93 | onRedeemedRetryableFound: async ticket => { 94 | await handleRedeemedRetryablesFound(ticket, writeToNotion) 95 | }, 96 | }) 97 | 98 | // todo: get closure on this - will it even be called 99 | if (writeToNotion) { 100 | console.log('Activating continuous sweep of Notion database...') 101 | setInterval(async () => { 102 | await alertUntriagedNotionRetryables() 103 | }, 1000 * 60 * 60) // Run every hour 104 | } 105 | } else { 106 | console.log('Activating one-off check for retryables...') 107 | const retryablesFound = await checkRetryablesOneOff({ 108 | parentChainProvider, 109 | childChainProvider, 110 | childChain, 111 | fromBlock, 112 | toBlock, 113 | enableAlerting, 114 | onFailedRetryableFound: async ticket => { 115 | await handleFailedRetryablesFound(ticket, writeToNotion) 116 | }, 117 | onRedeemedRetryableFound: async ticket => { 118 | await handleRedeemedRetryablesFound(ticket, writeToNotion) 119 | }, 120 | }) 121 | 122 | if (retryablesFound === 0) { 123 | console.log('No retryables found in the specified block range.') 124 | } 125 | } 126 | } 127 | 128 | // Function to process multiple child chains concurrently 129 | const processOrbitChainsConcurrently = async () => { 130 | // log the chains being processed for better debugging in github actions 131 | console.log( 132 | '>>>>>> Processing child chains: ', 133 | config.childChains.map((childChain: ChildNetwork) => ({ 134 | name: childChain.name, 135 | chainID: childChain.chainId, 136 | orbitRpcUrl: childChain.orbitRpcUrl, 137 | parentRpcUrl: childChain.parentRpcUrl, 138 | })) 139 | ) 140 | 141 | const promises = config.childChains.map(async (childChain: ChildNetwork) => { 142 | try { 143 | if (!networkIsRegistered(childChain.chainId)) { 144 | registerCustomArbitrumNetwork(childChain) 145 | } 146 | 147 | const parentChainProvider = new providers.JsonRpcProvider( 148 | String(childChain.parentRpcUrl) 149 | ) 150 | const childChainProvider = new providers.JsonRpcProvider( 151 | String(childChain.orbitRpcUrl) 152 | ) 153 | 154 | return await processChildChain( 155 | parentChainProvider, 156 | childChainProvider, 157 | childChain, 158 | options.fromBlock, 159 | options.toBlock, 160 | options.enableAlerting, 161 | options.continuous, 162 | options.writeToNotion 163 | ) 164 | } catch (e) { 165 | const errorStr = `Retryable monitor - Error processing chain [${childChain.name}]: ${e.message}` 166 | if (options.enableAlerting) { 167 | postSlackMessage({ 168 | message: errorStr, 169 | }) 170 | } 171 | console.error(errorStr) 172 | } 173 | }) 174 | 175 | // keep running the script until we get resolution (success or error) for all the chains 176 | await Promise.allSettled(promises) 177 | 178 | // once we process all the chains go through the Notion database once to alert on any `Unresolved` tickets found 179 | if (options.writeToNotion) { 180 | await alertUntriagedNotionRetryables() 181 | } 182 | } 183 | 184 | // Start processing child chains concurrently 185 | processOrbitChainsConcurrently() 186 | -------------------------------------------------------------------------------- /packages/retryable-monitor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "retryable-monitor", 3 | "version": "1.0.0", 4 | "main": "index.ts", 5 | "dependencies": { 6 | "@arbitrum/nitro-contracts": "v1.1.0-beta.1", 7 | "@arbitrum/sdk": "^4.0.0", 8 | "@arbitrum/token-bridge-contracts": "v1.1.0-beta.2", 9 | "@ethersproject/abstract-provider": "^5.5.1", 10 | "@notionhq/client": "^2.3.0", 11 | "axios": "^1.7.2", 12 | "ethers": "^5.5.4", 13 | "graphql": "^16.6.0", 14 | "graphql-request": "^6.1.0", 15 | "utils": "*", 16 | "winston": "^3.3.3" 17 | }, 18 | "scripts": { 19 | "lint": "eslint .", 20 | "build": "rm -rf ./dist && tsc", 21 | "format": "prettier './**/*.{js,json,md,yml,sol,ts}' --write && yarn run lint --fix", 22 | "dev": "yarn build && node ./dist/retryable-monitor/index.js" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/retryable-monitor/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "./dist" 5 | }, 6 | "include": ["./**/*.ts", "./**/*.d.ts", "packages/*.ts"], 7 | "exclude": ["node_modules", "dist/"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/utils/config.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import * as path from 'path' 3 | import dotenv from 'dotenv' 4 | 5 | dotenv.config({ path: '../../.env' }) 6 | 7 | // config.json file at project root 8 | export const DEFAULT_CONFIG_PATH = '../../config.json' 9 | 10 | export const getConfig = (options: { configPath: string }) => { 11 | try { 12 | const configFileContent = fs.readFileSync( 13 | path.join(process.cwd(), options.configPath), 14 | 'utf-8' 15 | ) 16 | 17 | const parsedConfig = JSON.parse(configFileContent) 18 | 19 | return parsedConfig 20 | } catch (error) { 21 | if (error instanceof SyntaxError) { 22 | throw new Error('Invalid JSON in the config file') 23 | } 24 | if (error instanceof Error) { 25 | throw new Error(`Error reading or parsing config file: ${error.message}`) 26 | } 27 | throw new Error( 28 | 'An unknown error occurred while processing the config file' 29 | ) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/utils/getExplorerUrlPrefixes.ts: -------------------------------------------------------------------------------- 1 | import { ChildNetwork } from './types' 2 | 3 | export const getExplorerUrlPrefixes = (childChain: ChildNetwork) => { 4 | const PARENT_CHAIN_TX_PREFIX = `${childChain.parentExplorerUrl}tx/` 5 | const PARENT_CHAIN_ADDRESS_PREFIX = `${childChain.parentExplorerUrl}address/` 6 | const CHILD_CHAIN_TX_PREFIX = `${childChain.explorerUrl}tx/` 7 | const CHILD_CHAIN_ADDRESS_PREFIX = `${childChain.explorerUrl}address/` 8 | 9 | return { 10 | PARENT_CHAIN_TX_PREFIX, 11 | PARENT_CHAIN_ADDRESS_PREFIX, 12 | CHILD_CHAIN_TX_PREFIX, 13 | CHILD_CHAIN_ADDRESS_PREFIX, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /packages/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from './types' 2 | export * from './config' 3 | export { getExplorerUrlPrefixes } from './getExplorerUrlPrefixes' 4 | export { postSlackMessage } from './postSlackMessage' 5 | 6 | export const sleep = (ms: number) => 7 | new Promise(resolve => setTimeout(resolve, ms)) 8 | -------------------------------------------------------------------------------- /packages/utils/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "utils", 3 | "version": "1.0.0", 4 | "main": "index.ts", 5 | "dependencies": { 6 | "@slack/web-api": "^7.0.4" 7 | }, 8 | "scripts": { 9 | "lint": "eslint ." 10 | } 11 | } -------------------------------------------------------------------------------- /packages/utils/postSlackMessage.ts: -------------------------------------------------------------------------------- 1 | import { WebClient } from '@slack/web-api' 2 | import { sanitizeSlackMessage } from './sanitizeSlackMessage' 3 | 4 | export const postSlackMessage = ({ 5 | slackToken, 6 | slackChannel, 7 | message, 8 | }: { 9 | slackToken: string 10 | slackChannel: string 11 | message: string 12 | }) => { 13 | const web = new WebClient(slackToken) 14 | 15 | console.log(`>>> Posting message to Slack -> ${message}`) 16 | 17 | return web.chat.postMessage({ 18 | text: sanitizeSlackMessage(message), 19 | channel: slackChannel, 20 | unfurl_links: false, 21 | }) 22 | } 23 | -------------------------------------------------------------------------------- /packages/utils/sanitizeSlackMessage.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | dotenv.config({ path: '../../.env' }) 3 | 4 | export const sanitizeSlackMessage = (message: string): string => { 5 | const allKeys = Object.keys(process.env) 6 | const sensitiveKeyContent = ['NEXT', 'API', 'KEY', 'MONITOR', 'INFURA', 'RPC'] 7 | 8 | // Filter sensitive keys based on the predefined content list 9 | const sensitiveKeys = allKeys.filter(key => 10 | sensitiveKeyContent.some(content => key.includes(content)) 11 | ) 12 | 13 | // Sanitize the message by replacing occurrences of sensitive keys with *** 14 | let sanitizedMessage = message 15 | sensitiveKeys.forEach(sensitiveKey => { 16 | const value = process.env[sensitiveKey] 17 | 18 | // make sure the value is not undefined or blank 19 | if (typeof value !== 'undefined' && String(value).trim().length > 0) { 20 | const regex = new RegExp(String(value).trim(), 'g') 21 | sanitizedMessage = sanitizedMessage.replace(regex, '***') 22 | } 23 | }) 24 | 25 | return sanitizedMessage 26 | } 27 | -------------------------------------------------------------------------------- /packages/utils/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "./dist" 5 | }, 6 | "include": ["./**/*.ts", "./**/*.d.ts", "packages/*.ts"], 7 | "exclude": ["node_modules", "dist/"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/utils/types.ts: -------------------------------------------------------------------------------- 1 | import { ArbitrumNetwork } from '@arbitrum/sdk' 2 | 3 | // Interface defining additional properties for ChildNetwork 4 | export interface ChildNetwork extends ArbitrumNetwork { 5 | parentRpcUrl: string 6 | orbitRpcUrl: string 7 | explorerUrl: string 8 | parentExplorerUrl: string 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "declaration": true, 6 | "outDir": "./dist", 7 | "strict": true, 8 | "useUnknownInCatchVariables": false, 9 | "noImplicitAny": true, 10 | "noImplicitReturns": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "strictNullChecks": true, 13 | "noImplicitThis": true, 14 | "resolveJsonModule": true, 15 | "esModuleInterop": true, 16 | "experimentalDecorators": true, 17 | "skipLibCheck": true, 18 | "sourceMap": true, 19 | "types": ["node"] 20 | } 21 | } 22 | --------------------------------------------------------------------------------