├── .editorconfig ├── .github └── update.sh ├── .gitignore ├── LICENSE.txt ├── README.md ├── aave ├── brownie-config.yml └── userVaults.q ├── default.nix ├── kfk ├── 0001-Makefile-don-t-download-k.h-from-github-master-branc.patch └── default.nix ├── load-env.sh ├── log4q ├── 0001-log4q-use-logfmt-format.patch └── default.nix ├── main.nix ├── patches ├── 0001-src-stop-downloading-k.h-from-master-branch.patch └── 0002-src-include-k.h-from-source-directory.patch ├── q ├── default.nix ├── q └── q.k ├── qAbi └── default.nix ├── qBigint └── default.nix ├── qUnit └── default.nix ├── qsh ├── src ├── bundler.q ├── core.q ├── env.q ├── kdb_schema.q ├── routine.q └── utils │ ├── assert.q │ └── globals.q └── terraform ├── service.tf ├── terraform.tf └── variables.tf /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # All files 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | indent_size = 2 9 | indent_style = space 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | # Solidity 14 | # https://github.com/sambacha/prettier-config-solidity 15 | [*.sol] 16 | indent_size = 4 17 | indent_style = space 18 | 19 | # q 20 | # kdb+ 21 | [*.q] 22 | indent_style = space 23 | indent_size = 2 24 | end_of_line = lf 25 | charset = utf-8 26 | trim_trailing_whitespace = true 27 | insert_final_newline = true 28 | 29 | # Markdown 30 | [*.{md,adoc,asciidoc}] 31 | charset = utf-8 32 | end_of_line = lf 33 | insert_final_newline = true 34 | trim_trailing_whitespace = false 35 | 36 | # Match nix files, set indent to spaces with width of two 37 | [*.nix] 38 | indent_style = space 39 | indent_size = 2 40 | 41 | # JavaScript, JSON, JSX, JavaScript Modules, TypeScript 42 | # https://github.com/feross/standard 43 | # https://prettier.io 44 | [*.{cjs,js,json,jsx,mjs,ts,tsx,mts,cts}] 45 | indent_size = 2 46 | indent_style = space 47 | 48 | # TOML 49 | # https://github.com/toml-lang/toml/tree/master/examples 50 | [*.toml] 51 | indent_size = 2 52 | indent_style = space 53 | 54 | # YAML 55 | # http://yaml.org/spec/1.2/2009-07-21/spec.html#id2576668 56 | [*.{yaml,yml}] 57 | indent_size = 2 58 | indent_style = space 59 | 60 | # Shell 61 | # https://google.github.io/styleguide/shell.xml#Indentation 62 | [*.{bash,sh,zsh}] 63 | indent_size = 2 64 | indent_style = space 65 | 66 | # confg + cfg 67 | [*.{conf,cfg}] 68 | charset = UTF-8 69 | end_of_line = LF 70 | indent_size = 4 71 | indent_style = tab 72 | insert_final_newline = true 73 | tab_width = 4 74 | trim_trailing_whitespace = true 75 | 76 | # Match diffs, avoid to trim trailing whitespace 77 | [*.{diff,patch}] 78 | trim_trailing_whitespace = false 79 | 80 | # Ignore fixtures and vendored files 81 | [{dist,artifacts,vendor,test/fixtures,tests_config,__snapshot__,}/**] 82 | charset = unset 83 | end_of_line = unset 84 | indent_size = unset 85 | indent_style = unset 86 | insert_final_newline = unset 87 | trim_trailing_spaces = unset 88 | -------------------------------------------------------------------------------- /.github/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | echo "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" > "hash" 5 | 6 | # try (and fail) building, to get SRI hash of cargo deps 7 | (set +e; nix build |& tee out) 8 | 9 | # parse output, get hash, save 10 | awk '/got: / {print $2}' out > "hash" 11 | 12 | function clean() { rm -rf out; } 13 | trap clean EXIT -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # -*- mode: gitignore; -*- 2 | /out 3 | .DS_Store 4 | .secret 5 | .idea/* 6 | 7 | logs 8 | *.log 9 | .\#* 10 | **/.terraform/* 11 | 12 | # .tfstate files 13 | *.tfstate 14 | *.tfstate.* 15 | *.tfvars 16 | *.tfvars.json 17 | *.ssh 18 | *.key -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | SPDXVersion: SPDX-2.0 2 | Creator: Manifold Finance, Inc 3 | PackageName: kdb-q-bot 4 | PackageOriginator: Manifold Finance, Inc. 5 | PackageLicenseDeclared: MPL-2.0 6 | 7 | Mozilla Public License Version 2.0 8 | ================================== 9 | 10 | 1. Definitions 11 | -------------- 12 | 13 | 1.1. "Contributor" 14 | means each individual or legal entity that creates, contributes to 15 | the creation of, or owns Covered Software. 16 | 17 | 1.2. "Contributor Version" 18 | means the combination of the Contributions of others (if any) used 19 | by a Contributor and that particular Contributor's Contribution. 20 | 21 | 1.3. "Contribution" 22 | means Covered Software of a particular Contributor. 23 | 24 | 1.4. "Covered Software" 25 | means Source Code Form to which the initial Contributor has attached 26 | the notice in Exhibit A, the Executable Form of such Source Code 27 | Form, and Modifications of such Source Code Form, in each case 28 | including portions thereof. 29 | 30 | 1.5. "Incompatible With Secondary Licenses" 31 | means 32 | 33 | (a) that the initial Contributor has attached the notice described 34 | in Exhibit B to the Covered Software; or 35 | 36 | (b) that the Covered Software was made available under the terms of 37 | version 1.1 or earlier of the License, but not also under the 38 | terms of a Secondary License. 39 | 40 | 1.6. "Executable Form" 41 | means any form of the work other than Source Code Form. 42 | 43 | 1.7. "Larger Work" 44 | means a work that combines Covered Software with other material, in 45 | a separate file or files, that is not Covered Software. 46 | 47 | 1.8. "License" 48 | means this document. 49 | 50 | 1.9. "Licensable" 51 | means having the right to grant, to the maximum extent possible, 52 | whether at the time of the initial grant or subsequently, any and 53 | all of the rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | means any of the following: 57 | 58 | (a) any file in Source Code Form that results from an addition to, 59 | deletion from, or modification of the contents of Covered 60 | Software; or 61 | 62 | (b) any new file in Source Code Form that contains any Covered 63 | Software. 64 | 65 | 1.11. "Patent Claims" of a Contributor 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the 69 | License, by the making, using, selling, offering for sale, having 70 | made, import, or transfer of either its Contributions or its 71 | Contributor Version. 72 | 73 | 1.12. "Secondary License" 74 | means either the GNU General Public License, Version 2.0, the GNU 75 | Lesser General Public License, Version 2.1, the GNU Affero General 76 | Public License, Version 3.0, or any later versions of those 77 | licenses. 78 | 79 | 1.13. "Source Code Form" 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | means an individual or a legal entity exercising rights under this 84 | License. For legal entities, "You" includes any entity that 85 | controls, is controlled by, or is under common control with You. For 86 | purposes of this definition, "control" means (a) the power, direct 87 | or indirect, to cause the direction or management of such entity, 88 | whether by contract or otherwise, or (b) ownership of more than 89 | fifty percent (50%) of the outstanding shares or beneficial 90 | ownership of such entity. 91 | 92 | 2. License Grants and Conditions 93 | -------------------------------- 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | (a) under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | (b) under Patent Claims of such Contributor to make, use, sell, offer 107 | for sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | (a) for any code that a Contributor has removed from Covered Software; 125 | or 126 | 127 | (b) for infringements caused by: (i) Your and any other third party's 128 | modifications of Covered Software, or (ii) the combination of its 129 | Contributions with other software (except as part of its Contributor 130 | Version); or 131 | 132 | (c) under Patent Claims infringed by Covered Software in the absence of 133 | its Contributions. 134 | 135 | This License does not grant any rights in the trademarks, service marks, 136 | or logos of any Contributor (except as may be necessary to comply with 137 | the notice requirements in Section 3.4). 138 | 139 | 2.4. Subsequent Licenses 140 | 141 | No Contributor makes additional grants as a result of Your choice to 142 | distribute the Covered Software under a subsequent version of this 143 | License (see Section 10.2) or under the terms of a Secondary License (if 144 | permitted under the terms of Section 3.3). 145 | 146 | 2.5. Representation 147 | 148 | Each Contributor represents that the Contributor believes its 149 | Contributions are its original creation(s) or it has sufficient rights 150 | to grant the rights to its Contributions conveyed by this License. 151 | 152 | 2.6. Fair Use 153 | 154 | This License is not intended to limit any rights You have under 155 | applicable copyright doctrines of fair use, fair dealing, or other 156 | equivalents. 157 | 158 | 2.7. Conditions 159 | 160 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 161 | in Section 2.1. 162 | 163 | 3. Responsibilities 164 | ------------------- 165 | 166 | 3.1. Distribution of Source Form 167 | 168 | All distribution of Covered Software in Source Code Form, including any 169 | Modifications that You create or to which You contribute, must be under 170 | the terms of this License. You must inform recipients that the Source 171 | Code Form of the Covered Software is governed by the terms of this 172 | License, and how they can obtain a copy of this License. You may not 173 | attempt to alter or restrict the recipients' rights in the Source Code 174 | Form. 175 | 176 | 3.2. Distribution of Executable Form 177 | 178 | If You distribute Covered Software in Executable Form then: 179 | 180 | (a) such Covered Software must also be made available in Source Code 181 | Form, as described in Section 3.1, and You must inform recipients of 182 | the Executable Form how they can obtain a copy of such Source Code 183 | Form by reasonable means in a timely manner, at a charge no more 184 | than the cost of distribution to the recipient; and 185 | 186 | (b) You may distribute such Executable Form under the terms of this 187 | License, or sublicense it under different terms, provided that the 188 | license for the Executable Form does not attempt to limit or alter 189 | the recipients' rights in the Source Code Form under this License. 190 | 191 | 3.3. Distribution of a Larger Work 192 | 193 | You may create and distribute a Larger Work under terms of Your choice, 194 | provided that You also comply with the requirements of this License for 195 | the Covered Software. If the Larger Work is a combination of Covered 196 | Software with a work governed by one or more Secondary Licenses, and the 197 | Covered Software is not Incompatible With Secondary Licenses, this 198 | License permits You to additionally distribute such Covered Software 199 | under the terms of such Secondary License(s), so that the recipient of 200 | the Larger Work may, at their option, further distribute the Covered 201 | Software under the terms of either this License or such Secondary 202 | License(s). 203 | 204 | 3.4. Notices 205 | 206 | You may not remove or alter the substance of any license notices 207 | (including copyright notices, patent notices, disclaimers of warranty, 208 | or limitations of liability) contained within the Source Code Form of 209 | the Covered Software, except that You may alter any license notices to 210 | the extent required to remedy known factual inaccuracies. 211 | 212 | 3.5. Application of Additional Terms 213 | 214 | You may choose to offer, and to charge a fee for, warranty, support, 215 | indemnity or liability obligations to one or more recipients of Covered 216 | Software. However, You may do so only on Your own behalf, and not on 217 | behalf of any Contributor. You must make it absolutely clear that any 218 | such warranty, support, indemnity, or liability obligation is offered by 219 | You alone, and You hereby agree to indemnify every Contributor for any 220 | liability incurred by such Contributor as a result of warranty, support, 221 | indemnity or liability terms You offer. You may include additional 222 | disclaimers of warranty and limitations of liability specific to any 223 | jurisdiction. 224 | 225 | 4. Inability to Comply Due to Statute or Regulation 226 | --------------------------------------------------- 227 | 228 | If it is impossible for You to comply with any of the terms of this 229 | License with respect to some or all of the Covered Software due to 230 | statute, judicial order, or regulation then You must: (a) comply with 231 | the terms of this License to the maximum extent possible; and (b) 232 | describe the limitations and the code they affect. Such description must 233 | be placed in a text file included with all distributions of the Covered 234 | Software under this License. Except to the extent prohibited by statute 235 | or regulation, such description must be sufficiently detailed for a 236 | recipient of ordinary skill to be able to understand it. 237 | 238 | 5. Termination 239 | -------------- 240 | 241 | 5.1. The rights granted under this License will terminate automatically 242 | if You fail to comply with any of its terms. However, if You become 243 | compliant, then the rights granted under this License from a particular 244 | Contributor are reinstated (a) provisionally, unless and until such 245 | Contributor explicitly and finally terminates Your grants, and (b) on an 246 | ongoing basis, if such Contributor fails to notify You of the 247 | non-compliance by some reasonable means prior to 60 days after You have 248 | come back into compliance. Moreover, Your grants from a particular 249 | Contributor are reinstated on an ongoing basis if such Contributor 250 | notifies You of the non-compliance by some reasonable means, this is the 251 | first time You have received notice of non-compliance with this License 252 | from such Contributor, and You become compliant prior to 30 days after 253 | Your receipt of the notice. 254 | 255 | 5.2. If You initiate litigation against any entity by asserting a patent 256 | infringement claim (excluding declaratory judgment actions, 257 | counter-claims, and cross-claims) alleging that a Contributor Version 258 | directly or indirectly infringes any patent, then the rights granted to 259 | You by any and all Contributors for the Covered Software under Section 260 | 2.1 of this License shall terminate. 261 | 262 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 263 | end user license agreements (excluding distributors and resellers) which 264 | have been validly granted by You or Your distributors under this License 265 | prior to termination shall survive termination. 266 | 267 | ************************************************************************ 268 | * * 269 | * 6. Disclaimer of Warranty * 270 | * ------------------------- * 271 | * * 272 | * Covered Software is provided under this License on an "as is" * 273 | * basis, without warranty of any kind, either expressed, implied, or * 274 | * statutory, including, without limitation, warranties that the * 275 | * Covered Software is free of defects, merchantable, fit for a * 276 | * particular purpose or non-infringing. The entire risk as to the * 277 | * quality and performance of the Covered Software is with You. * 278 | * Should any Covered Software prove defective in any respect, You * 279 | * (not any Contributor) assume the cost of any necessary servicing, * 280 | * repair, or correction. This disclaimer of warranty constitutes an * 281 | * essential part of this License. No use of any Covered Software is * 282 | * authorized under this License except under this disclaimer. * 283 | * * 284 | ************************************************************************ 285 | 286 | ************************************************************************ 287 | * * 288 | * 7. Limitation of Liability * 289 | * -------------------------- * 290 | * * 291 | * Under no circumstances and under no legal theory, whether tort * 292 | * (including negligence), contract, or otherwise, shall any * 293 | * Contributor, or anyone who distributes Covered Software as * 294 | * permitted above, be liable to You for any direct, indirect, * 295 | * special, incidental, or consequential damages of any character * 296 | * including, without limitation, damages for lost profits, loss of * 297 | * goodwill, work stoppage, computer failure or malfunction, or any * 298 | * and all other commercial damages or losses, even if such party * 299 | * shall have been informed of the possibility of such damages. This * 300 | * limitation of liability shall not apply to liability for death or * 301 | * personal injury resulting from such party's negligence to the * 302 | * extent applicable law prohibits such limitation. Some * 303 | * jurisdictions do not allow the exclusion or limitation of * 304 | * incidental or consequential damages, so this exclusion and * 305 | * limitation may not apply to You. * 306 | * * 307 | ************************************************************************ 308 | 309 | 8. Litigation 310 | ------------- 311 | 312 | Any litigation relating to this License may be brought only in the 313 | courts of a jurisdiction where the defendant maintains its principal 314 | place of business and such litigation shall be governed by laws of that 315 | jurisdiction, without reference to its conflict-of-law provisions. 316 | Nothing in this Section shall prevent a party's ability to bring 317 | cross-claims or counter-claims. 318 | 319 | 9. Miscellaneous 320 | ---------------- 321 | 322 | This License represents the complete agreement concerning the subject 323 | matter hereof. If any provision of this License is held to be 324 | unenforceable, such provision shall be reformed only to the extent 325 | necessary to make it enforceable. Any law or regulation which provides 326 | that the language of a contract shall be construed against the drafter 327 | shall not be used to construe this License against a Contributor. 328 | 329 | 10. Versions of the License 330 | --------------------------- 331 | 332 | 10.1. New Versions 333 | 334 | Mozilla Foundation is the license steward. Except as provided in Section 335 | 10.3, no one other than the license steward has the right to modify or 336 | publish new versions of this License. Each version will be given a 337 | distinguishing version number. 338 | 339 | 10.2. Effect of New Versions 340 | 341 | You may distribute the Covered Software under the terms of the version 342 | of the License under which You originally received the Covered Software, 343 | or under the terms of any subsequent version published by the license 344 | steward. 345 | 346 | 10.3. Modified Versions 347 | 348 | If you create software not governed by this License, and you want to 349 | create a new license for such software, you may create and use a 350 | modified version of this License if you rename the license and remove 351 | any references to the name of the license steward (except to note that 352 | such modified license differs from this License). 353 | 354 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 355 | Licenses 356 | 357 | If You choose to distribute Source Code Form that is Incompatible With 358 | Secondary Licenses under the terms of this version of the License, the 359 | notice described in Exhibit B of this License must be attached. 360 | 361 | Exhibit A - Source Code Form License Notice 362 | ------------------------------------------- 363 | 364 | This Source Code Form is subject to the terms of the Mozilla Public 365 | License, v. 2.0. If a copy of the MPL was not distributed with this 366 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 367 | 368 | If it is not possible or desirable to put the notice in a particular 369 | file, then You may include the notice in a location (such as a LICENSE 370 | file in a relevant directory) where a recipient would be likely to look 371 | for such a notice. 372 | 373 | You may add additional accurate notices of copyright ownership. 374 | 375 | Exhibit B - "Incompatible With Secondary Licenses" Notice 376 | --------------------------------------------------------- 377 | 378 | This Source Code Form is "Incompatible With Secondary Licenses", as 379 | defined by the Mozilla Public License, v. 2.0. 380 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # kdb+/q for EVM 3 | 4 | > [What is kdb+/q?, see kx.com](https://kx.com) 5 | 6 | 7 | ## Overview 8 | 9 | kdb+ adapted for usage on Ethereum/EVM Chains 10 | 11 | This code is no longer in usage at Manifold Finance, which is why we have open sourced it. 12 | 13 | 14 | ## Setup 15 | 16 | >**Warning** 17 | > mortals require instructions, you have been warned. 18 | 19 | 20 | 1. [You will need a license for kdb+ to use this, here is the install instructions via code.kx.com](https://code.kx.com/q/learn/install/) 21 | 22 | 2. [Get the 64bit License via the On-Demand Personal exemption](https://kx.com/64bit-on-demand-personal-edition/) 23 | 24 | 25 | ## Required Libraries 26 | 27 | These libraries are *required* 28 | 29 | [https://github.com/manifoldfinance/qBigInt](https://github.com/manifoldfinance/qBigInt) 30 | 31 | [https://github.com/manifoldfinance/qQuarticRoots](https://github.com/manifoldfinance/qQuarticRoots) 32 | 33 | [https://github.com/manifoldfinance/qAbiEncode](https://github.com/manifoldfinance/qAbiEncode) 34 | 35 | 36 | ## Example: Aave 37 | 38 | [see /aave dir](/aave) 39 | 40 | ![](https://d.pr/i/K4XegR.jpeg) 41 | 42 | Will start 4 processes. 43 | 44 | q realtime db for user balances. 45 | subscribe to all lender events to update db balances. 46 | query snapshot of every user balance (will take a long time to complete but liquidations can still occur in the background). 47 | scan for unstable health and try to liquidate (prompt will appear for unlocking your accounts). 48 | logs folder will hold log info. 49 | 50 | To view live user balances and health: `http://localhost:5010`. 51 | 52 | ## Utils 53 | 54 | [Jupyter Notebook for Sushiswap/Uniswap Liquidity and Market Depth](https://gist.github.com/sambacha/a21955e8a8feec7579a607f153476547#file-sushiswap_liquiditypairs-ipynb) 55 | 56 | ## Install 57 | 58 | From https://code.kx.com/q4m3/14_Introduction_to_Kdb+/1481-the-environment-variables, 59 | set `QHOME` to the location of `q.k`. 60 | To pass in a license, invoke q with `QLIC` set to the *directory* in which 61 | `kc.lic` is located. 62 | Other .q (or .k) files are loaded with `\l path/to/file`. 63 | `.so `files are loaded by defining their name/path when defining the function: 64 | 65 | ## Flashbots Overview 66 | 67 | https://docs.flashbots.net/flashbots-auction/overview 68 | 69 | Notes on the current state of flashbots docs with relevance to assumptions of bundle process. 70 | 71 | ### Anatomy of a bundle 72 | 73 | ```ts 74 | const blockNumber = await provider.getBlockNumber() 75 | const minTimestamp = (await provider.getBlock(blockNumber)).timestamp 76 | const maxTimestamp = minTimestamp + 120 77 | const signedBundle = flashbotsProvider.signBundle( 78 | [ 79 | { 80 | signedTransaction: SIGNED_ORACLE_UPDATE_FROM_PENDING_POOL // serialized signed transaction hex 81 | }, 82 | { 83 | signer: wallet, // ethers signer 84 | transaction: transaction // ethers populated transaction object 85 | } 86 | ]) 87 | const bundleReceipt = await flashbotsProvider.sendRawBundle( 88 | signedBundle, // bundle we signed above 89 | targetBlockNumber, // block number at which this bundle is valid 90 | { 91 | minTimestamp, // optional minimum timestamp at which this bundle is valid (inclusive) 92 | maxTimestamp, // optional maximum timestamp at which this bundle is valid (inclusive) 93 | revertingTxHashes: [tx1, tx2] // optional list of transaction hashes allowed to revert. Without specifying here, any revert invalidates the entire bundle. 94 | } 95 | ) 96 | ) 97 | ``` 98 | 99 | ### Miner reward through coinbase.transfer() 100 | 101 | https://docs.flashbots.net/flashbots-auction/searchers/advanced/coinbase-payment 102 | 103 | To include as last action of smart contract 104 | 105 | ```solidity 106 | block.coinbase.transfer(AMOUNT_TO_TRANSFER) 107 | 108 | ``` 109 | 110 | Edge case to deal with sending to a miner contract 111 | ```solidity 112 | block.coinbase.call{value: _ethAmountToCoinbase}(new bytes(0)); 113 | ``` 114 | subject to [reentrancy attacks](https://medium.com/coinmonks/protect-your-solidity-smart-contracts-from-reentrancy-attacks-9972c3af7c21) 115 | 116 | ### Bundle pricing 117 | 118 | https://docs.flashbots.net/flashbots-auction/searchers/advanced/bundle-pricing 119 | 120 | Conflicting bundles received by flashbots are ordered by the following formula: 121 | 122 | ![\bg_white \Large score=\frac{minerBribe + totalGasUsed * priorityFeePerGas - mempoolGasUsed * priorityFeePerGas}{totalGasUsed}](https://latex.codecogs.com/png.latex?\bg_white&space;\Large&space;score=\frac{minerBribe&space;+&space;totalGasUsed&space;*&space;priorityFeePerGas&space;-&space;mempoolGasUsed&space;*&space;priorityFeePerGas}{totalGasUsed}) 123 | 124 | ### Eligibility 125 | 126 | https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v04 127 | 128 | Bundles must have a target `blockNumber` and a `priorityFeePerGas` >= 1 Gwei. 129 | 130 | ### Reverting txs 131 | 132 | https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v04 133 | 134 | 135 | "When constructing a block the node should reject any bundle or megabundle that has a reverting transaction unless its hash is included in the RevertingTxHashes list of the bundle" 136 | 137 | ### Debugging 138 | 139 | https://docs-staging.flashbots.net/flashbots-auction/searchers/advanced/troubleshooting 140 | 141 | 1. Transaction failure (ANY within the bundle) 142 | 2. Incentives (gas price/coinbase transfers) not high enough to offset value of block space 143 | 144 | Simulate bundle: 145 | ```ts 146 | const signedTransactions = await flashbotsProvider.signBundle(transactionBundle) 147 | const simulation = await flashbotsProvider.simulate(signedTransactions, targetBlockNumber, targetBlockNumber + 1) 148 | console.log(JSON.stringify(simulation, null, 2)) 149 | ``` 150 | 151 | 3. Competitors paying more 152 | 153 | Get conflicting bundles for a prior block: 154 | ```ts 155 | const signedTransactions = await flashbotsProvider.signBundle(transactionBundle) 156 | console.log(await flashbotsProvider.getConflictingBundle( 157 | signedTransactions, 158 | 13140328 // blockNumber 159 | )) 160 | ``` 161 | 162 | 4. Bundle received too late to appear in target block 163 | 164 | Get submission time data and compare to block time: 165 | ```ts 166 | console.log( 167 | await flashbotsProvider.getBundleStats("0x123456789abcdef123456789abcdef123456789abcdef123456789abcdef1234", 13509887) 168 | ) 169 | 170 | ``` 171 | 172 | ## License 173 | 174 | This Source Code Form is subject to the terms of the Mozilla Public 175 | License, v. 2.0. If a copy of the MPL was not distributed with this 176 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 177 | -------------------------------------------------------------------------------- /aave/brownie-config.yml: -------------------------------------------------------------------------------- 1 | # automatically fetch contract sources from Etherscan 2 | autofetch_sources: True 3 | 4 | # require OpenZepplin Contracts v3.0.0 5 | dependencies: 6 | - OpenZeppelin/openzeppelin-contracts@4.3.2 7 | 8 | # path remapping to support OpenZepplin imports with NPM-style path 9 | compiler: 10 | solc: 11 | remappings: 12 | - '@openzeppelin=OpenZeppelin/openzeppelin-contracts@4.3.2' 13 | 14 | networks: 15 | default: poly_alchemy 16 | mainnet: 17 | aave_lending_pool_v2: "0x7d2768dE32b0b80b7a3454c06BdAc94A69DDc7A9" 18 | weth: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" 19 | sushi_router: "0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F" 20 | -------------------------------------------------------------------------------- /aave/userVaults.q: -------------------------------------------------------------------------------- 1 | / user reserve table for off-chain updates and calculations 2 | AAVE_POLYGON: ([user:(); reserve:()] debt:`float$(); collateral:`float$()); 3 | 4 | / table to keep user health 5 | USER_HEALTH: ([user:()] health:`float$(); timestamp:`timestamp$()); 6 | 7 | / func to test if a file or object exists 8 | exists: {not () ~ key x}; 9 | 10 | / load data 11 | if[exists `:USER_HEALTH; 12 | load `USER_HEALTH; 13 | ]; 14 | if[exists `:AAVE_POLYGON; 15 | load `AAVE_POLYGON; 16 | ]; 17 | 18 | / hard-coded token dict 19 | AAVE_TOKENS: (!) . flip( 20 | (`AAVE; lower "0xD6DF932A45C0f255f85145f286eA0b292B21C90B" ); 21 | (`DAI; lower "0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063"); 22 | (`USDC; lower "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174"); 23 | (`USDT; lower "0xc2132D05D31c914a87C6611C10748AEb04B58e8F"); 24 | (`WBTC; lower "0x1BFD67037B42Cf73acF2047067bd4F2C47D9BfD6"); 25 | (`WETH; lower "0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619"); 26 | (`WMATIC; lower "0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270")); 27 | 28 | / hard coded liquidation thresholds 29 | AAVE_THRESHOLDS: (!) . flip( 30 | ( `AAVE; 0.65 ); 31 | ( `DAI; 0.8 ); 32 | ( `USDC; 0.85 ); 33 | ( `USDT; 0.0 ); 34 | ( `WBTC; 0.75 ); 35 | ( `WETH; 0.825 ); 36 | ( `WMATIC; 0.65 ) ); 37 | 38 | / hard coded decimals 39 | DECIMALS: (!) . flip( 40 | ( `AAVE; 18 ); 41 | ( `DAI; 18 ); 42 | ( `USDC; 6 ); 43 | ( `USDT; 6 ); 44 | ( `WBTC; 8 ); 45 | ( `WETH; 18 ); 46 | ( `WMATIC; 18 ) ); 47 | 48 | 49 | / cast hex symbol or string to bytes for local storage 50 | castToBytes:{[x] 51 | tp: type x; 52 | $[4h = tp; / bytes 53 | x; 54 | -4h = tp; / byte 55 | enlist x; 56 | 10h = tp; / string 57 | "X"$2 cut 2_x; 58 | -10h = tp; / char 59 | "X"$"0",x; 60 | -11h = tp; / symbol 61 | "X"$2 cut 2_ string x; 62 | '`unknownType 63 | ] 64 | }; 65 | 66 | / insert function for user vault balance data from chain 67 | aavePolygonInsertDebt:{[ iUser; iReserve; iDebt ] 68 | `AAVE_POLYGON upsert (!) . flip( 69 | (`user; castToBytes iUser); 70 | (`reserve; castToBytes iReserve); 71 | (`debt; iDebt) 72 | ); 73 | }; 74 | 75 | / insert function for user vault balance data from chain 76 | aavePolygonInsertCollateral:{[ iUser; iReserve; iCollateral ] 77 | `AAVE_POLYGON upsert (!) . flip( 78 | (`user; castToBytes iUser); 79 | (`reserve; castToBytes iReserve); 80 | (`collateral; iCollateral) 81 | ); 82 | }; 83 | 84 | aavePolygonInsert:{[ iUser; iReserve; iDebt; iCollateral ] 85 | `AAVE_POLYGON upsert (!) . flip( 86 | (`user; castToBytes iUser); 87 | (`reserve; castToBytes iReserve); 88 | (`debt; iDebt); 89 | (`collateral; iCollateral) 90 | ); 91 | }; 92 | 93 | / withdraw function from events 94 | aavePolygonWithdraw:{[iUser; iReserve; amount] 95 | / only update if balances exist 96 | xUser: castToBytes iUser; 97 | xReserve: castToBytes iReserve; 98 | if[0 < count exec user from AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 99 | update collateral: collateral - amount from `AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 100 | ]; 101 | }; 102 | 103 | / deposit function from events 104 | aavePolygonDeposit:{[iUser; iReserve; amount] 105 | / only update if balances exist 106 | xUser: castToBytes iUser; 107 | xReserve: castToBytes iReserve; 108 | if[0 < count exec user from AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 109 | update collateral: collateral + amount from `AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 110 | ]; 111 | }; 112 | 113 | / repay function from events 114 | aavePolygonRepay:{[iUser; iReserve; amount] 115 | / only update if balances exist 116 | xUser: castToBytes iUser; 117 | xReserve: castToBytes iReserve; 118 | if[0 < count exec user from AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 119 | update debt: debt - amount from `AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 120 | ]; 121 | }; 122 | 123 | / borrow function from events 124 | aavePolygonBorrow:{[iUser; iReserve; amount] 125 | / only update if balances exist 126 | xUser: castToBytes iUser; 127 | xReserve: castToBytes iReserve; 128 | xUser: string iUser; 129 | if[0 < count exec user from AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 130 | update debt: debt + amount from `AAVE_POLYGON where user~\:xUser, reserve~\:xReserve; 131 | ]; 132 | }; 133 | 134 | / liquidate function from events 135 | aavePolygonLiquidate:{[iUser; iCollateral; iDebt; debtToCover; liquidatedCollateralAmount] 136 | / only update if balances exist 137 | xUser: castToBytes iUser; 138 | xCollateral: castToBytes iCollateral; 139 | xDebt: castToBytes iDebt; 140 | if[0 < count exec user from AAVE_POLYGON where user~\:xUser, reserve~\:xCollateral; 141 | update collateral: collateral - liquidatedCollateralAmount from `AAVE_POLYGON where user~\:xUser, reserve~\:xCollateral; 142 | update debt: debt - debtToCover from `AAVE_POLYGON where user~\:xUser, reserve~\:xDebt; 143 | ]; 144 | }; 145 | 146 | 147 | / get unhealthy users function 148 | getUnhealthyUsers:{[] 149 | exec user from USER_HEALTH where health < 1.0 150 | }; 151 | 152 | 153 | / get ref prices function for health calculations 154 | getPrices:{[weth] 155 | coingecko: "https://api.coingecko.com/api/v3/coins/markets?ids=aave,ethereum,bitcoin,matic-network,dai,usd-coin,tether,avalanche-2&vs_currency=usd"; 156 | priceData: .j.k .Q.hg coingecko; 157 | tokenPricesUsd: select symbol:`$upper symbol,current_price from priceData; 158 | tokenPricesDict:tokenPricesUsd[`symbol]!(tokenPricesUsd[`current_price]); 159 | tokenPricesDict[`WETH]:tokenPricesDict[`ETH]; 160 | tokenPricesDict[`WBTC]:tokenPricesDict[`BTC]; 161 | tokenPricesDict[`WMATIC]:tokenPricesDict[`MATIC]; 162 | tokenPricesDict[`WAVAX]:tokenPricesDict[`AVAX]; 163 | / return normalised price in matic 164 | (key tokenPricesDict)!(value tokenPricesDict) % tokenPricesDict[weth] 165 | }; 166 | 167 | getSymbolFromReserve:{[iReserve] 168 | AAVE_TOKENS?iReserve 169 | }; 170 | 171 | 172 | / calculate health factor 173 | updateHealth:{[] 174 | pricesDict: getPrices(`MATIC); 175 | / show pricesDict; 176 | users: exec distinct user from AAVE_POLYGON; 177 | healths: {[prices; iUser] 178 | reserves: 0!select from AAVE_POLYGON where user~\:iUser; 179 | totalColl: sum 0.0^{[prices;iReserve] 180 | sym: getSymbolFromReserve["0x", raze string iReserve`reserve]; 181 | (prices[sym]) * `float$iReserve[`collateral] 182 | }[prices] each reserves; 183 | productSumCollLiq: sum 0.0^{[prices;iReserve] 184 | sym: getSymbolFromReserve["0x", raze string iReserve`reserve]; 185 | (prices[sym]) * (`float$iReserve[`collateral]) * AAVE_THRESHOLDS[sym] 186 | }[prices] each reserves; 187 | avLiqThresh: productSumCollLiq % totalColl; 188 | 189 | totalDebt: sum 0.0^{[prices;iReserve] 190 | sym: getSymbolFromReserve["0x", raze string iReserve`reserve]; 191 | (`float$iReserve[`debt]) * prices[sym] 192 | }[prices] each reserves; 193 | health: $[totalDebt > 0.0; 194 | health: ( totalColl * avLiqThresh ) % totalDebt; 195 | 2.0 196 | ]; 197 | health 198 | }[pricesDict] each users; 199 | `USER_HEALTH upsert ( 200 | [user: users] 201 | health: healths; 202 | timestamp: (count users)#.z.p ); 203 | }; 204 | 205 | 206 | / clean dead users 207 | cleanDead:{[] 208 | / `USER_HEALTH set 0.0^USER_HEALTH; 209 | update 0.0^health from `USER_HEALTH; 210 | deadUsers: exec distinct user from USER_HEALTH where health <= 0.0; 211 | delete from `USER_HEALTH where health <= 0.0; 212 | delete from `AAVE_POLYGON where user in deadUsers; 213 | }; 214 | 215 | / repeater function runs on timer 216 | .z.ts:{[] 217 | updateHealth[]; 218 | cleanDead[]; 219 | save `AAVE_POLYGON; 220 | save `USER_HEALTH; 221 | .Q.gc[]; / garbage cleaner 222 | }; 223 | 224 | / timer in ms for repeater function 225 | \t 4000 226 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { stdenv 2 | , autoPatchelfHook 3 | , bash 4 | , coreutils 5 | , libredirect 6 | , lndir 7 | , makeWrapper 8 | , qLibs 9 | , rlwrap 10 | , runCommand 11 | , shellcheck 12 | , util-linux 13 | , writeTextFile 14 | }: 15 | let 16 | # From https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#1481-the-environment-variables, 17 | # set QHOME to the location of `q.k`. 18 | # To pass in a license, invoke q with QLIC set to the *directory* in which 19 | # kc.lic is located. 20 | # Other .q (or .k) files are loaded with `\l path/to/file`. 21 | # .so files are loaded by defining their name/path when defining the function: 22 | # https://code.kx.com/q/interfaces/using-c-functions/ 23 | 24 | version = "4.0-2020.05.04"; 25 | 26 | # Package that contains just the stdlib 27 | q-stdlib = writeTextFile { 28 | name = "q-stdlib-${version}"; 29 | destination = "/q/q.k"; 30 | text = builtins.readFile ./q.k; 31 | }; 32 | 33 | # The Q unwrapped binary 34 | q-bin = stdenv.mkDerivation { 35 | pname = "q-bin"; 36 | inherit version; 37 | dontUnpack = true; 38 | 39 | nativeBuildInputs = [ 40 | autoPatchelfHook 41 | ]; 42 | 43 | doBuild = false; 44 | 45 | installPhase = '' 46 | runHook preInstall 47 | install -Dm755 ${./q} $out 48 | runHook postInstall 49 | ''; 50 | }; 51 | 52 | # This is a wrapper for the Q binary that is extended with a number of 53 | # features. 54 | # 55 | # See the comments in the script for more details. 56 | q-wrapper = writeTextFile { 57 | name = "q-${version}"; 58 | destination = "/bin/q"; 59 | executable = true; 60 | checkPhase = "${shellcheck}/bin/shellcheck $out/bin/q"; 61 | text = '' 62 | #!${bash}/bin/bash 63 | # A wrapper for the Q binary extended with more environment variables. 64 | # 65 | # See the table below in the Globals for more details. 66 | set -euo pipefail 67 | ### Globals ### 68 | # On which CPU the process should be pinned. 69 | : "''${QCPU:=}" 70 | # In which folder the database should be stored. 71 | : "''${QDB:=''${QHOME:-$QHOME/db}}" 72 | # A root folder for the Q binary to load libraries from. 73 | : "''${QHOME:=}" 74 | # Extend QHOME with a list of paths to load libraries from. The paths 75 | # must be absolute. 76 | : "''${QLIB:=}" 77 | # Override in which directory to look for to load the Q license. 78 | : "''${QLIC:=}" 79 | # This is the load order, by order of precedence 80 | QLIB=''${QHOME:+$QHOME:}''${QLIB:+$QLIB:}${qLibs}/q:${q-stdlib}/q 81 | # Temporary work dir with all of QLIB merged into. 82 | qhome=$(${coreutils}/bin/mktemp -d) 83 | ### Functions ### 84 | log() { 85 | echo "[''$$]$*" >&2 86 | } 87 | # Cleanup to execute on process termination. 88 | at_exit() { 89 | local ret=$? pid= 90 | set +e 91 | # Make sure to forward the signal 92 | pid=$(jobs -p) 93 | if [[ -n $pid ]]; then 94 | log "killing $pid" 95 | kill "$pid" 96 | wait "$pid" 97 | fi 98 | if [[ -n $qhome ]]; then 99 | log "cleaning $qhome" 100 | rm -rf "$qhome" 101 | fi 102 | exit "$ret" 103 | } 104 | ### Main ### 105 | # Setup a callback on process termination 106 | trap at_exit EXIT 107 | # Build the symlink tree 108 | IFS=: 109 | for dir in $QLIB; do 110 | ${lndir}/bin/lndir -silent "$dir" "$qhome" 111 | done 112 | IFS=' ' 113 | # Change directory into the Q DB 114 | if [[ -n $QDB ]]; then 115 | cd "$QDB" 116 | else 117 | log "warn: QDB and QHOME are unset. The db will be written to the current directory." 118 | fi 119 | # Point Q to the symlink filesystem 120 | export QHOME=$qhome 121 | cmd=(${q-bin} "$@") 122 | # Wrap if stdin is interactive 123 | # XXX: this breaks libredirect 124 | # if [[ -t 0 ]]; then 125 | # cmd=(${rlwrap}/bin/rlwrap "''${cmd[@]}") 126 | # fi 127 | log "QCPU=$QCPU" 128 | # Bind to one CPU 129 | if [[ -n $QCPU ]]; then 130 | ${util-linux}/bin/taskset -pc "$QCPU" ''$$ 131 | else 132 | ${util-linux}/bin/taskset -p ''$$ 133 | fi 134 | log "QDB=$QDB" 135 | log "QHOME=$QHOME" 136 | log "QLIB=$QLIB" 137 | log "QLIC=$QLIC" 138 | log "\$''${cmd[*]}" 139 | # Run! 140 | NIX_REDIRECTS=/q/=$QHOME/ LD_PRELOAD=${libredirect}/lib/libredirect.so "''${cmd[@]}" 141 | ''; 142 | }; 143 | in 144 | q-wrapper -------------------------------------------------------------------------------- /kfk/0001-Makefile-don-t-download-k.h-from-github-master-branc.patch: -------------------------------------------------------------------------------- 1 | From 1b9d3667a90358c5d008321818c3e219cc7e30a7 Mon Sep 17 00:00:00 2001 2 | From: Sam Bacha 3 | Date: Sun, 6 Jun 2021 08:25:06 +0200 4 | Subject: [PATCH] Makefile: don't download k.h from github master branch during 5 | build 6 | 7 | We provide this after cloning the repo 8 | --- 9 | Makefile | 6 ++---- 10 | 1 file changed, 2 insertions(+), 4 deletions(-) 11 | 12 | diff --git a/Makefile b/Makefile 13 | index 5af64c9..f6a652f 100644 14 | --- a/Makefile 15 | +++ b/Makefile 16 | @@ -28,12 +28,10 @@ endif 17 | QARCH = $(OSFLAG)$(MS) 18 | Q = $(QHOME)/$(QARCH) 19 | 20 | -all: k.h 21 | +all: 22 | $(CC) kfk.c -m$(MS) $(OPTS) $(LDOPTS_DYNAMIC) $(LD_COMMON) -I$(KFK_INCLUDE) $(LNK) -o $(TGT) $(OSXOPTS) 23 | -static: k.h 24 | +static: 25 | $(CC) kfk.c -m$(MS) $(OPTS) $(LDOPTS_STATIC) $(LD_COMMON) -I$(KFK_INCLUDE) $(LNK) -o $(TGT) $(OSXOPTS) 26 | -k.h: 27 | - curl -s -O -L https://github.com/KxSystems/kdb/raw/master/c/c/k.h 28 | install: 29 | install $(TGT) $(Q) 30 | clean: 31 | -- 32 | 2.31.1 33 | 34 | -------------------------------------------------------------------------------- /kfk/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv, fetchFromGitHub, rdkafka, zlib, openssl, ... }: 2 | 3 | let 4 | version = "1.5.0"; 5 | kdbSrc = fetchFromGitHub { 6 | owner = "KxSystems"; 7 | repo = "kdb"; 8 | rev = "39b957030bf6a4608f2508ff29894d7fac32a0c2"; 9 | sha256 = "0r0yfnfn2g89hc7gvk2y1d568lkda0463jw3ha1gg8h83v1vm63f"; 10 | }; 11 | in 12 | stdenv.mkDerivation { 13 | inherit version; 14 | name = "kfk-${version}"; 15 | 16 | src = fetchFromGitHub { 17 | owner = "KxSystems"; 18 | repo = "kafka"; 19 | rev = "v${version}"; 20 | sha256 = "1yi9f8gy3nv5hncd0qh9q2lzk315pavf3bkc5wq48ki0yxllhaqf"; 21 | }; 22 | 23 | # The Makefile downloaded k.h from kdb repo, master branch. 24 | # Patch this out, and copy it into $src after unpack. 25 | patches = [ ./0001-Makefile-don-t-download-k.h-from-github-master-branc.patch ]; 26 | 27 | preConfigure = '' 28 | cp ${kdbSrc}/c/c/k.h . 29 | ''; 30 | 31 | #NIX_CFLAGS_COMPILE = "-I."; 32 | NIX_CFLAGS_LINK = "-L${zlib.out}/lib -L${openssl.out}/lib"; 33 | 34 | makeFlags = [ 35 | "KAFKA_ROOT=${rdkafka}" 36 | ]; 37 | 38 | # override install phase, cause their `make install` is bonkers, and we only 39 | # really care about libkfk.so anyways. 40 | installPhase = '' 41 | install -Dm644 kfk.q $out/q/kfk.q 42 | install -Dm644 libkfk.so $out/q/l64/libkfk.so 43 | strip $out/q/l64/*.so 44 | ''; 45 | 46 | meta = { 47 | description = "A thin wrapper for kdb+ around librdkafka C API for Kafka"; 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /load-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Load the devshell 3 | source ./env/env.bash 4 | 5 | # load Q libraries from these folders 6 | export QLIB=$PWD/modules/domain/src/main:$PWD/modules/kdb/src/main/q:$PWD/modules/kdb/src/test/q 7 | # pick the first CPU by default 8 | export QCPU=0 9 | # move the DB out 10 | export QDB=$PWD/qdb 11 | mkdir -p "$QDB" 12 | # select which Q license to use 13 | export QLIC=$PWD/config/qlicenses/dev -------------------------------------------------------------------------------- /log4q/0001-log4q-use-logfmt-format.patch: -------------------------------------------------------------------------------- 1 | From fff009102056d071c98f839ed8954d0386bb83eb Mon Sep 17 00:00:00 2001 2 | From: Sam Bacha 3 | Date: Thu, 20 Jan 2022 12:08:01 +0100 4 | Subject: [PATCH] log4q: use logfmt format 5 | 6 | --- 7 | log4q.q | 2 +- 8 | 1 file changed, 1 insertion(+), 1 deletion(-) 9 | 10 | diff --git a/log4q.q b/log4q.q 11 | index 87f601a..67be65c 100644 12 | --- a/log4q.q 13 | +++ b/log4q.q 14 | @@ -1,5 +1,5 @@ 15 | \d .log4q 16 | -fm:"%c\t[%p]:H=%h:PID[%i]:%d:%t:%f: %m\r\n"; 17 | +fm:"level=%c file=%f host=%h pid=%i %m\n"; 18 | sev:snk:`SILENT`DEBUG`INFO`WARN`ERROR`FATAL!();a:{$[1 3 | Date: Sun, 6 Jun 2021 12:40:09 +0200 4 | Subject: [PATCH 1/2] src: stop downloading k.h from master branch 5 | 6 | --- 7 | src/CMakeLists.txt | 2 -- 8 | 1 file changed, 2 deletions(-) 9 | 10 | diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt 11 | index 014bcc5..0e1bca7 100644 12 | --- a/src/CMakeLists.txt 13 | +++ b/src/CMakeLists.txt 14 | @@ -61,8 +61,6 @@ PROTOBUF_GENERATE_CPP(PROTO_KDB_SRCS PROTO_KDB_HDRS kdb_type_specifier.proto) 15 | set(PROTO_SRCS "${PROTO_SRCS};${PROTO_KDB_SRCS}") 16 | set(PROTO_HDRS "${PROTO_HDRS};${PROTO_KDB_HDRS}") 17 | 18 | -file(DOWNLOAD "https://github.com/KxSystems/kdb/raw/master/c/c/k.h" "${CMAKE_BINARY_DIR}/k.h" ) 19 | - 20 | if (MSVC) 21 | file(DOWNLOAD "https://github.com/KxSystems/kdb/raw/master/w64/q.lib" "${CMAKE_BINARY_DIR}/q.lib" ) 22 | set(LINK_LIBS "${CMAKE_BINARY_DIR}/q.lib") 23 | -- 24 | 2.31.1 -------------------------------------------------------------------------------- /patches/0002-src-include-k.h-from-source-directory.patch: -------------------------------------------------------------------------------- 1 | From 27755b440af53f2b77d63b765854f7b3865c146c Mon Sep 17 00:00:00 2001 2 | From: Sam Bacha 3 | Date: Sun, 6 Jun 2021 12:40:09 +0200 4 | Subject: [PATCH 2/2] src: include k.h from source directory 5 | 6 | --- 7 | src/KdbTypes.h | 2 +- 8 | src/MapValues.h | 2 +- 9 | src/MessageFormat.cpp | 2 +- 10 | src/MessageFormat.h | 2 +- 11 | src/MessageParser.h | 2 +- 12 | src/ProtobufKdb.cpp | 2 +- 13 | src/ProtobufKdb.h | 2 +- 14 | src/RepeatedValues.h | 2 +- 15 | src/ScalarValues.h | 2 +- 16 | 9 files changed, 9 insertions(+), 9 deletions(-) 17 | 18 | diff --git a/src/KdbTypes.h b/src/KdbTypes.h 19 | index 80865d3..3d720be 100644 20 | --- a/src/KdbTypes.h 21 | +++ b/src/KdbTypes.h 22 | @@ -6,7 +6,7 @@ 23 | 24 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 25 | // after all the protobuf headers to avoid conflicts. 26 | -#include 27 | +#include "k.h" 28 | 29 | 30 | namespace kx { 31 | diff --git a/src/MapValues.h b/src/MapValues.h 32 | index d12c6bb..69788d7 100644 33 | --- a/src/MapValues.h 34 | +++ b/src/MapValues.h 35 | @@ -5,7 +5,7 @@ 36 | 37 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 38 | // after all the protobuf headers to avoid conflicts. 39 | -#include 40 | +#include "k.h" 41 | 42 | 43 | namespace kx { 44 | diff --git a/src/MessageFormat.cpp b/src/MessageFormat.cpp 45 | index 0a38ff3..d43acc0 100644 46 | --- a/src/MessageFormat.cpp 47 | +++ b/src/MessageFormat.cpp 48 | @@ -6,7 +6,7 @@ 49 | 50 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 51 | // after all the protobuf headers to avoid conflicts. 52 | -#include 53 | +#include "k.h" 54 | 55 | 56 | namespace kx { 57 | diff --git a/src/MessageFormat.h b/src/MessageFormat.h 58 | index f38bcd2..a2cfef5 100644 59 | --- a/src/MessageFormat.h 60 | +++ b/src/MessageFormat.h 61 | @@ -6,7 +6,7 @@ 62 | 63 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 64 | // after all the protobuf headers to avoid conflicts. 65 | -#include 66 | +#include "k.h" 67 | 68 | 69 | namespace kx { 70 | diff --git a/src/MessageParser.h b/src/MessageParser.h 71 | index 02d143a..f0fefb0 100644 72 | --- a/src/MessageParser.h 73 | +++ b/src/MessageParser.h 74 | @@ -9,7 +9,7 @@ 75 | 76 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 77 | // after all the protobuf headers to avoid conflicts. 78 | -#include 79 | +#include "k.h" 80 | 81 | 82 | namespace kx { 83 | diff --git a/src/ProtobufKdb.cpp b/src/ProtobufKdb.cpp 84 | index 8d6fdd2..4f484a1 100644 85 | --- a/src/ProtobufKdb.cpp 86 | +++ b/src/ProtobufKdb.cpp 87 | @@ -25,7 +25,7 @@ 88 | // #define R return 89 | // 90 | // in 'k.h'. 91 | -#include 92 | +#include "k.h" 93 | 94 | 95 | inline bool IsKdbString(K str) 96 | diff --git a/src/ProtobufKdb.h b/src/ProtobufKdb.h 97 | index a3777c7..8348a4d 100644 98 | --- a/src/ProtobufKdb.h 99 | +++ b/src/ProtobufKdb.h 100 | @@ -1,7 +1,7 @@ 101 | #ifndef __PROTOBUF_KDB_H__ 102 | #define __PROTOBUF_KDB_H__ 103 | 104 | -#include 105 | +#include "k.h" 106 | 107 | #ifdef _WIN32 108 | #define EXP __declspec(dllexport) 109 | diff --git a/src/RepeatedValues.h b/src/RepeatedValues.h 110 | index a75239a..119b850 100644 111 | --- a/src/RepeatedValues.h 112 | +++ b/src/RepeatedValues.h 113 | @@ -5,7 +5,7 @@ 114 | 115 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 116 | // after all the protobuf headers to avoid conflicts. 117 | -#include 118 | +#include "k.h" 119 | 120 | 121 | namespace kx { 122 | diff --git a/src/ScalarValues.h b/src/ScalarValues.h 123 | index 6ad6be0..5213a80 100644 124 | --- a/src/ScalarValues.h 125 | +++ b/src/ScalarValues.h 126 | @@ -5,7 +5,7 @@ 127 | 128 | // Due to its excessive and non-specific use of #define, 'k.h' must be included 129 | // after all the protobuf headers to avoid conflicts. 130 | -#include 131 | +#include "k.h" 132 | 133 | 134 | namespace kx { 135 | -- 136 | 2.31.1 -------------------------------------------------------------------------------- /q/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv 2 | , autoPatchelfHook 3 | , bash 4 | , coreutils 5 | , libredirect 6 | , lndir 7 | , makeWrapper 8 | , qLibs 9 | , rlwrap 10 | , runCommand 11 | , shellcheck 12 | , util-linux 13 | , writeTextFile 14 | }: 15 | let 16 | # From https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#1481-the-environment-variables, 17 | # set QHOME to the location of `q.k`. 18 | # To pass in a license, invoke q with QLIC set to the *directory* in which 19 | # kc.lic is located. 20 | # Other .q (or .k) files are loaded with `\l path/to/file`. 21 | # .so files are loaded by defining their name/path when defining the function: 22 | # https://code.kx.com/q/interfaces/using-c-functions/ 23 | 24 | version = "4.0-2020.05.04"; 25 | 26 | # Package that contains just the stdlib 27 | q-stdlib = writeTextFile { 28 | name = "q-stdlib-${version}"; 29 | destination = "/q/q.k"; 30 | text = builtins.readFile ./q.k; 31 | }; 32 | 33 | # The Q unwrapped binary 34 | q-bin = stdenv.mkDerivation { 35 | pname = "q-bin"; 36 | inherit version; 37 | dontUnpack = true; 38 | 39 | nativeBuildInputs = [ 40 | autoPatchelfHook 41 | ]; 42 | 43 | doBuild = false; 44 | 45 | installPhase = '' 46 | runHook preInstall 47 | install -Dm755 ${./q} $out 48 | runHook postInstall 49 | ''; 50 | }; 51 | 52 | # This is a wrapper for the Q binary that is extended with a number of 53 | # features. 54 | # 55 | # See the comments in the script for more details. 56 | q-wrapper = writeTextFile { 57 | name = "q-${version}"; 58 | destination = "/bin/q"; 59 | executable = true; 60 | checkPhase = "${shellcheck}/bin/shellcheck $out/bin/q"; 61 | text = '' 62 | #!${bash}/bin/bash 63 | # A wrapper for the Q binary extended with more environment variables. 64 | # 65 | # See the table below in the Globals for more details. 66 | set -euo pipefail 67 | 68 | ### Globals ### 69 | 70 | # On which CPU the process should be pinned. 71 | : "''${QCPU:=}" 72 | # In which folder the database should be stored. 73 | : "''${QDB:=''${QHOME:-$QHOME/db}}" 74 | # A root folder for the Q binary to load libraries from. 75 | : "''${QHOME:=}" 76 | # Extend QHOME with a list of paths to load libraries from. The paths 77 | # must be absolute. 78 | : "''${QLIB:=}" 79 | # Override in which directory to look for to load the Q license. 80 | : "''${QLIC:=}" 81 | 82 | # This is the load order, by order of precedence 83 | QLIB=''${QHOME:+$QHOME:}''${QLIB:+$QLIB:}${qLibs}/q:${q-stdlib}/q 84 | 85 | # Temporary work dir with all of QLIB merged into. 86 | qhome=$(${coreutils}/bin/mktemp -d) 87 | 88 | ### Functions ### 89 | 90 | log() { 91 | echo "[''$$]$*" >&2 92 | } 93 | 94 | # Cleanup to execute on process termination. 95 | at_exit() { 96 | local ret=$? pid= 97 | set +e 98 | # Make sure to forward the signal 99 | pid=$(jobs -p) 100 | if [[ -n $pid ]]; then 101 | log "killing $pid" 102 | kill "$pid" 103 | wait "$pid" 104 | fi 105 | if [[ -n $qhome ]]; then 106 | log "cleaning $qhome" 107 | rm -rf "$qhome" 108 | fi 109 | exit "$ret" 110 | } 111 | 112 | ### Main ### 113 | 114 | # Setup a callback on process termination 115 | trap at_exit EXIT 116 | 117 | # Build the symlink tree 118 | IFS=: 119 | for dir in $QLIB; do 120 | ${lndir}/bin/lndir -silent "$dir" "$qhome" 121 | done 122 | IFS=' ' 123 | 124 | # Change directory into the Q DB 125 | if [[ -n $QDB ]]; then 126 | cd "$QDB" 127 | else 128 | log "warn: QDB and QHOME are unset. The db will be written to the current directory." 129 | fi 130 | 131 | # Point Q to the symlink filesystem 132 | export QHOME=$qhome 133 | cmd=(${q-bin} "$@") 134 | # Wrap if stdin is interactive 135 | # XXX: this breaks libredirect 136 | # if [[ -t 0 ]]; then 137 | # cmd=(${rlwrap}/bin/rlwrap "''${cmd[@]}") 138 | # fi 139 | log "QCPU=$QCPU" 140 | # Bind to one CPU 141 | if [[ -n $QCPU ]]; then 142 | ${util-linux}/bin/taskset -pc "$QCPU" ''$$ 143 | else 144 | ${util-linux}/bin/taskset -p ''$$ 145 | fi 146 | log "QDB=$QDB" 147 | log "QHOME=$QHOME" 148 | log "QLIB=$QLIB" 149 | log "QLIC=$QLIC" 150 | 151 | log "\$''${cmd[*]}" 152 | # Run! 153 | NIX_REDIRECTS=/q/=$QHOME/ LD_PRELOAD=${libredirect}/lib/libredirect.so "''${cmd[@]}" 154 | ''; 155 | }; 156 | in 157 | q-wrapper 158 | -------------------------------------------------------------------------------- /q/q: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sambacha/q-evm/c6d460069a7f9c7a3b452fa77e5654725943238a/q/q -------------------------------------------------------------------------------- /q/q.k: -------------------------------------------------------------------------------- 1 | \d .q 2 | /each: +-*%&|^<>=$ <= >= <> @ ? in within bin div abs log exp sqrt sin cos tan f' f\: f/: 3 | neg:-:;not:~:;null:^:;string:$:;reciprocal:%:;floor:_:;ceiling:-_-:;signum:{(x>0)-x<0} 4 | mod:{x-y*x div y};xbar:{x*y div x:$[16h=abs[@x];"j"$x;x]};xlog:{log[y]%log x};and:&;or:|;each:{x'y};scan:{x\y};over:{x/y};prior:{x':y} 5 | mmu:$;lsq:!;inv:!:;md5:-15!;ltime:%:;gtime:{t+x-%t:x+x-%x}; /xnull:{$[0>@y;(,y)@~x=y;x=y;y 0N;y]} 6 | 7 | /aggr: last sum prd min max avg wsum wavg f/ /beta:{cov[x;y]%var x} 8 | count:#:;first:*:;svar:{(n*var x)%-1+n:(#x)-+/^x};sdev:{sqrt svar x};scov:{(n*cov[x;y])%-1+n:(#x)-+/^x+y};med:{avg x(@x;'`rank;1_x,,x 0N]}; 12 | rank:{$[0h>@x;'`rank;<@x;'`rank;@x;'`rank;>x]} 13 | asc:{$[99h=@x;(!x)[i]!`s#r i:@x;'`rank;`s#x@r:. x;0h>@x;'`rank;x@>x]} 14 | 15 | msum:{$[99h=@y;(!y)!.z.s[x;. y];y-(-x)_(0i*x#y),y:+\y]};mcount:{msum[x;~^y]};mavg:{msum[x;0.0^y]%mcount[x;y]};mdev:{sqrt mavg[x;y*y]-m*m:mavg[x;y:"f"$y]} 16 | xrank:{$[0h>@y;'`rank;_y*x%#y:<@y;'`rank;y(!#y)-x]};rotate:{$[0h>@y;'`rank;98h<@y;'`type;#y;,/|(0;mod[x;#y])_y;y]};ema:{(*y)(1f-x)\x*y} 17 | 18 | /other: ~,#_ !. getenv exit 19 | distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} 20 | set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? 21 | raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} 22 | sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} 23 | 24 | /file&comm 25 | read0:0::;read1:1::;hclose:>:;hdel:~:;hsym:"s"$-1!';hcount:-7!;peach:{x':y};system:."\\", 26 | 27 | /string: like ss 28 | ltrim:{$[~t&77h>t:@x;.z.s'x;^*x;((^x)?0b)_x;x]};rtrim:{$[~t&77h>t:@x;.z.s'x;^last x;(-(|^x)?0b)_x;x]};trim:{ltrim rtrim x} 29 | lower:{$[$[(~@x)&10h~@*x;&/10h=@:'x;0b];_x;~t&77h>t:abs@@x;.z.s'x;19t:abs@@x;.z.s'x;19@z;:[;z];z]]} 32 | 33 | /select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation 34 | /{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) 35 | view:{(2+*x ss"::")_x:*|*|.`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} 36 | cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} 37 | xcols:{(x,f@&~(f:cols y)in x)#y};keys:{$[98h=@x:.Q.v x;0#`;!+!x]};xkey:{(#x)!.[0!y;();xcols x]}; 38 | xcol:{.Q.ft[{+$[99h=@x;@[!y;(!y)?!x;:;. x];x,(#x)_!y]!. y:+y}x]y};xasc:{$[$[#x;~`s=-2!(0!.Q.v y)x;0];.Q.ft[@[;*x;`s#]].Q.ord[<:;x]y;y]};xdesc:{$[#x;.Q.ord[>:;x]y;y]} 39 | fkeys:{(&~^x)#x:.Q.fk'.Q.V x};meta:{([!c].Q.ty't;f:.Q.fk't;a:-2!'t:. c:.Q.V x)} 40 | 41 | / R uj R(union join) R lj K(left(equi/asof)join) trade asof`sym`time!(`IBM;09:31:00.0) 42 | lj:{$[$[99h=@y;(98h=@!y)&98h=@. y;()~y];.Q.ft[,\:[;y];x];'"type"]} /;la:{$[&/j:z>-1;x,'y z;+.[+ff[x]y;(!+y;j);:;.+y z j:&j]]}{la[x;. y](!y)?(!+!y)#x}[;y]]x} /lj:,\:;aj:{lj[y]`s#xkey[x]z};aj0:{lj[y]`s#(x#z)!z}; /;bn:{@[i;&0>i:x bin y;:;#x]} 43 | ljf:{$[`s=-2!y;ajf[!+!y;x;0!y];.Q.ft[{$[&/j:(#y:. y)>i?:(!+i:!y)#x;.Q.fl[x]y i;+.[+x;(f;j);:;.+.Q.fl[((f:!+y)#x:.Q.ff[x]y)j]y i j:&j]]}[;y]]x]} 44 | .Q.ajf0:{[f;g;x;y;z]x,:();z:0!z;d:$[g;x_z;z];g:(:;^)f;f:(,;^)f;$[&/j:-1i:(!y)?(!+!y)#x}[;y]]x} 47 | ijf:{.Q.ft[{.Q.fl[x j]y i j:&(#y:. y)>i?:(!+i:!y)#x}[;y]]x} 48 | pj:{.Q.ft[{x+0i^y(!+!y)#x}[;y]]x};asof:{f:!$[99h=@y;y;+y];(f_x)(f#x)bin y} 49 | uj:{$[()~x;y;()~y;x;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];lj[(?(!x),!y)#x]y]} 50 | ujf:{$[()~x;y;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];ljf[(?(!x),!y)#x]y]} 51 | 52 | /wj[-1000 2000+\:trade`time;`sym`time;trade;(quote;(max;`ask);(min;`bid))] (given `sym`time xasc quote) 53 | ww:{[a;w;f;y;z]f,:();e:1_z;z:*z;y,'n#+(:/'f)!+{[e;d;a;b]e .'d@\:\:a+!b-a}[*:'e;z f:1_'e]/'$[n:#*w;+$[#g;(g#z)?g#y;0]|/:a+$[#g:-1_f;(f#z)bin@[f#y;*|f;:;]@;z[*f]bin]'w;,0 0]} 54 | wj:{[w;f;y;z]ww[0 1;w;f;y;z]};wj1:{[w;f;y;z]ww[1;w-1 0;f;y;z]} 55 | 56 | fby:{$[(#x 1)=#y;@[(#y)#x[0]0#x 1;g;:;x[0]'x[1]g:.=y];'`length]};xgroup:{x,:();a:x#y:0!y;$[#x_:y;+:'x@=a;a!+f!(#f:!+x)#()]};ungroup:{$[#x:0!x;,/+:'x;x]} 57 | ej:{x,:();y[&#:'i],'(x_z)(!0),/i:(=x#z:0!z)x#y:0!y} /{ungroup lj[z]xgroup[x]y} 58 | 59 | /`[:../]t[.{csv|txt}] 60 | save:{$[1=#p:`\:*|`\:x:-1!x;set[x;. *p]; x 0:.h.tx[p 1]@.*p]}' 61 | load:{$[1=#p:`\:*|`\:x:-1!x;set[*p;. x];set[*p].h.xt[p 1]@0:x]}' 62 | rsave:{x:-1!x;.[`/:x,`;();:;.*|`\:x]}' 63 | rload:{x:-1!x;.[*|`\:x;();:;. x]}' 64 | dsave:{.[*x;1_x,y,`;:;@[;*!+a;`p#].Q.en[*x]a:. y];y}/: 65 | 66 | show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" 67 | 68 | parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! 69 | \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") 70 | k:4.0;K:2020.05.04;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc 71 | res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp 72 | addmonths:{("d"$m+y)+x-"d"$m:"m"$x} 73 | Xf:{y 1:0xfe20,("x"$77+@x$()),13#0x00;(`$($y),"#")1:0x};Cf:Xf`char 74 | f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} 75 | fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} 76 | 77 | /DO NOT USE ROUTINES PAST HERE. SUBJECT TO CHANGE 78 | ff:{$[&/(!+y)in f:!+x;x;x,'(f_y)(#x)#0N]} 79 | fl:{$[98h=t:@x;+fl[+x;+y];99h=t;@[x,y;f;:;x[f]fl'y f@:&(f:!x)in!y];t&t<77h;x^y;@[y;i;:;x i:&(0<#:'x)&~#:'y]]} 80 | opt:{[o]x::$[#i:&o like"-[^0-9]*";i[0]#o;o];((`$1_*:)'o)!1_'o:i_o};def:{x,((!y)#x){$[0h>@x;*:;::](@*x)$y}'y} 81 | qt:{$[99h=@x;(98h=@!x)&98h=@. x;98h=@x]}; v:{$[-11h=@x;.$[":"=*t:$x;`$t,"/";x];x]}; 82 | qp:{$[~98h=@x;0;@x:.+x;~":"=*$x;0]}; V:{$[qp x:v x;((,pf)!,. pf),+$[(~x in!d0[])&(x:.+x)in !*. .Q.vt;.Q.vp x;d0[]@x];0h>@*x:+0!x;x@'!x;x]} 83 | ft:{$[{$[99h=@t:v x;98h=@. t;0]}y;[n:#+!y;n!x 0!y];x y]}; 84 | ord:{if[~&/b:{(!#x)=x?x}c:.q.cols z;'"dup ",$c b?0b];if[~&/b:(y,:())in c;'y b?0b];ft[@[;!+t:0!v z;::]]z;ft[@[;!+t;@[;x@+y!t y]]]z};nv:{$["."~*$x;x;`$"..",$x]} 85 | 86 | tx:{$[(76ht;t;11h=t:@x:. nv@!x;t;98h=t;7h;1=#x:+!x;tx@*x;7h]};tt:0|tx'. V@ 87 | fk:{$[(20h>t)|76h@. nv t:!x;`;t]};t:" bg xhijefcspmdznuvts";ty:{$[0h>x:tx x;.q.upper t@-x;t x]} 88 | nct:{$[`~x;,/nct'.q.tables`.;([]n:x;c:.q.cols x;t:tt x)]} 89 | 90 | fu:{[f;x]$[0h>@x;f x;f[u](u:?x)?x]} /uniques 91 | fc:{$[1@x;`${$[(*x)in n;"a",x;x]}x@&(x:$x)in an;ft[{s[i]:`$($s i:&(s in`i,res,!`.q)|~(s?s)=!#s:id'!x:+x),'"1";+s!. x}]x]} 94 | j10:64/:b6?;x10:b6@0x40\: /base64 J from char10 95 | j12:36/:nA?;x12:nA@0x24\: /base36 J from char12(cusip) 96 | btoa:-32!;sha1:-33!;prf0:+`name`file`line`col`text`pos!*-37! 97 | l:{if[$[1>@d:!f:-1!x;1;`.d~*d];:.[$[qt d;*|`\:f;99=@d;`.;'`type];();:;d:. f]];d@:&~d like"*$";p:(d=`par.txt)|d like"[0-9]*"; 98 | ."\\cd ",$x;{.q.set[x]$[0h>@!a:-1!x;. a;a`]}'d@&~(d=`html)|(d like"*#")|p|s:d like"*.?";if[|/p;L d@&p];if[~`.=x;(."\\l ",$:)'d@&s&~p];} 99 | 100 | sw:{.q.sublist[_.5**|."\\C"]x};tab:{" "/:'(x*|/#:''y)$/:y};t0:{$[x>#y;((#*y)#'" -"),y;y]} 101 | s1:{-3!x} /{$[0>t:@x;$x;99h76h;s1'x;t;$x;|/(97ht:?@:'x;s1'x;(1<#?#:'x)|$[1=#t;(*t)in 1 4 10h;0];s1'x;tab[1]@s2'sw'x]} 104 | S:{x[1]{$[x<#y;((x-2)#y),"..";y]}'$[t&77h>t:@z;,s1 z;99h#x)|(@x)&~11=@x} 112 | 113 | / CAN EXIT HERE FOR SMALL Q 114 | / pt(tables) pf(date/month/year/int) pd(dirs) pv(values) pn(count) pt::0#pf::` 115 | vt:(,`)!,()!(); 116 | bv:{g:$[(::)~x;max;min];x:`:.;d:{`/:'x,'d@&(d:!x)like"[0-9]*"}'P:$[`par.txt in!x;-1!'`$0:`/:x,`par.txt;,x]; 117 | t:?,/!:'.Q.vt:{(&#:'x)(=,/. x)}'{({("DMJJ"`date`month`year`int?.Q.pf)$$last@x:`\:x}'x)!!:'x}'d; 118 | d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#`. pf];(i;x i)}[;g]'+:t#/:g''.Q.vt:t#/:.Q.vt;.Q.vt:P!.q.except[. .Q.pf]''.Q.vt; 119 | .Q.vp:t!{(+(,.Q.pf)!,0#. .Q.pf),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}'d;.Q.pt,:{.[x;();:;+.q.except[!+.Q.vp x;.Q.pf]!x];x}'.q.except[t;.Q.pt];} 120 | 121 | pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!{(`u#pd,'pv)!p2[(x;();0b;())]'[pd;pv]}'pt]} 122 | dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;p1[;y;z]]} 123 | 124 | p:{$[~#D;p2[x;`:.]':y;(,/p2[x]'/':P[i](;)'y)@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} 125 | view:{pd::PD x:$[(::)~x;x;&PV in x];u~:?u::..[pf;();:;pv::PV x];x{.[y;();:;+(x . y,`.d)!y]}'pt::!x:d0[];pn::pt!(#pt)#()} 126 | L:{D::();if[x~,`par.txt;if[~#x:,/D::!:'P::`$":",'0:*x;'`empty]];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} 128 | /L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} 129 | 130 | cn:{$[#n:pn x:.+x;n;pn[x]:(#p1 .)':+(x;pd;pv)]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} 131 | ind:{,/i[j]{fp[pf;p]p1[x;pd y;p:pv y]z}[.+x]'(j:&~=':i)_y-n i:(n:+\0,cn x)bin y} 132 | fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} 133 | foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} 134 | 135 | / select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. 136 | a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} 137 | {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) 138 | 139 | / x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? 140 | qd:{$[(#:)~*x;(?:)~*x 1;0]};xy:{`$$*&x~/:y}; x1:{$[qb x;();IN[*x;a0];$[qd x;1_x;,x];,/x1'1_x]} 141 | x0:{$[qb x;x;IN[*x;a1];x0 a2[a1?*x]. 1_x;x0'x]};x2:{$[qb x; x;IN[*x;a0];$[qd x;(#:;(?:;(,/;xy[x 1]y)));[y:xy[x]y;$[(?:)~*x;(?:;(,/;y));(#:)~*x;(sum;y);(*x;y)]]];x2[;y]'x]} 142 | ua:{((`$$!#u)!u;x2[;u:?,/x1'x]'x:x0'x)};q0:{$[~qb x;,/q0'x;-11h=@x;*`\:x;()]};qe:{$[#x;99h=@x;1]} 143 | ps:{[t;c;b;a]if[-11h=@t;t:. t];if[~qe[a]&qe[b]|-1h=@b;'`nyi];d:pv;v:$[q:0>@b;0;~#b;0;-11h=@v:*. b;pf~*`\:v;0] 144 | if[$[~#c;0;@*c;0;-11h=@x:c[0]1;pf~*`\:x;0];d@:&-6!*c;c:1_c] 145 | if[$[#c;0;(g:(. a)~,pf)|(. a)~,(#:;`i)];f:!a;j:dt[d]t;if[q;:+f!,$[g;?d@&0(#.:)'t:.q.tables`.;if[h:@[hopen;h;0];h"\\l .";>h]} 160 | 161 | /loop through text /lc:{+/{+/0xa=1:(x;y*z;y)}[x;m]'!-_-(-7!x)%m:1048576} /line count of big file 162 | fsn:{[f;s;n]>[-7!s]{[f;s;x;n]i:(#r)^1+last@&"\n"=r:1:(s;x;n);f@`\:i#r;x+i}[f;s;;n]/0};fs:fsn[;;131000] 163 | fpn:{{r:.Q.trp[y;h;{>x;'(y;z)}h:hopen":fifo://",1_$x];>h;r}[y 164 | {[f;h;n]b:"x"$();while[#s:1:h;if[~n>#b,:s;v:`\:b,0x0a;if[1<#v;f@-1_v];b:"x"$last v]];if[#b;f@`\:b]}[x;;z]]};fps:fpn[;;131000] 165 | /e.g. p:` sv(d:`:dir;`2007.02.12;`t);@[;`sym;`p#]p{@[x;`;,;`sym xasc .Q.en[d]get y]}/rdb's 166 | dsftg:{[dpt;xom;f;tw;g]d:*dpt;dpt:par . dpt;x:*xom;if[~0m:_a:(m-:o:xom 1)%b:+/tw 10=@*tw;'`length];0N!($dpt),": ",$m&:M;i::0;n::1000000&_1e8%b 168 | do[-_-m%n;@[dpt;`;$[i;,;:];en[d]@+g f!tw 1:(x;o+b*i;b*n&:m-i)];0N!i+:n];dpt};M:0W 169 | 170 | / fill in empty partitions 171 | chk:{if[x~(::);'"missing dir argument"];f:{`/:'x,'d@&(d:!x)like"[0-9]*"};d@:>.:'$last'`\:'d:$[`par.txt in!x;,/f'-1!'`$0:`/:x,`par.txt;f x] 172 | {[e;u;d]u[i]{.[x;(y;`);:;?[z;();0b;()]]}[d]'e i:&~u in!d}[d[(+u in/:t)?\:1b](0#.)'u,'`;u:?,/t:!:'d]'d} 173 | 174 | Ll:-100!;Lp:-101!;Lx:-102!;Lu:-103!;Ls:-104!;fqk:{("<"~*x)|x~*-3#.{}};fql:{x[1;1]~""};btx:{r:,Lp x;while[@x:Lu(0b;x);r,:(~fql v)#,v:Lp x;if[(fqk v[1]1)&".Q.dbg"~*v 1;f:(Lx(x;"k";,"v"))1]];$[~#f;r;@[r;r?f;,;,">>"]]};bt:{1'pl'(2*4<|/#:'v)_v:1_btx Ll`;};sbt:{,/pl'x};trp:{-105!(x;,y;z)} 175 | dr:{0:0};dw:{@[2@;x;::];};pl0:{.[{[f;v;x;y;z]g:{$[x>0;x&#y;x|-#y]#y};m:1;n:0|(b:+\0,1+#:'s:`\:s)bin y:(0|y)+2*"k)"~2#s:x 3;sf:$[#x 1;x[1],$[0>x 2;"";":",($n+x 2)],": ";""];sn:{$[(#x)&@x;((2*".."~2#x)_x),":";()]}@*x;h:($[4<#v;*|v;" "],5$"[",($z),"]") 176 | if[("locked"~x 3)|$[f;0b;fqk x 1];:h,"(",($[#sn;-1_sn;-2_sf]),")\n\n"];h,:sf,sn;h,`/:((n>m)#,""),g[-m+1;(n+1)#s],(,(" \t"(((#h)*n<1)#0),(y-b n)#s[n]="\t"),"^"),g[m](n+1)_s}[x;y];3#1_y;{dw"pl0\n";}]};pl:pl0 0 177 | jl8:{$[$[#x;~x[3]in y[;3];0b];,x;()],y};srr:{@[{if["abort"~*x;:""];"'",$[.z.q;($.z.Z)," ";""],(x[0],"\n"),/pl'jl8[x 3;x 2]};x;"srr\n"]};prr:{dw srr x}; 178 | lu:{f:{$[@a:Lu(x;*y);(a;Lp a);()]}x;r:{$[#x;fql x 1;0b]}f/f y;$[#r;$[.Q.dbg~**r 1;y;r];y]} 179 | DL:0;dbg:{pm:{dw" q"[x],$[1<#d:$."\\d";d;""]," )"[x],(1|DL)#")"};dq:{v:$[#*y;(~"k)"~2#y[1;3];*(.**y)3);1<#*y 1;@[*y 1;0;"q"=];(1b;`)];x v 1;*v}ds:{."\\d ",((~"."=*x)#"."),x:$x};de:@[{DL-:1;x y;{."\\x .z.e",x}'"xy"}ds;;::] 180 | @[{DL+:1;.z.ex:*x 1;.z.ey:x[1]1};x;::];b:$[#*v:Lp y;0xff~(*. bf:**v)bi:v[0;1];0b];if[b;dw "#",($bi),"\n"];w:(y0:y;v:Lp y);if[e:@*x;x[2;;3]+:1+v 3;prr x];d0:."\\d" 181 | y:*w:{$[~fqk x[1;1]1;x;lu[0b]x]}/w;q:dq v:w 1;dw pl v;while[*(~(,"\\")~l:dr[];pm[q]);$[l~"\\ ";q:~q;l in+,".`&";[w:$["&"=*l;;lu["."=*l]](y;v);y:*w;v:w 1;if[y~y0;prr x];dw pl0[1]v;q:dq v] 182 | (~e)&l in+,":>";:*($[b;bc[bf;bi];]0 0N(":"=*l);de d0);"'"=*l;:*((1_l;());de d0);c&~#*r:Lx(y;"kq"q;(c:":"=*l)_l);:*(r;de d0);$[#*r;::;dw $[q;.Q.s;{$[(::)~x;"";`/:,-3!x]}]r 1]]];de d0;("abort";())} 183 | err:{(*x;jl8[x 3;$[@y;btx y;()],x 2])}; BP:(,"")!,();bp:{*|(Ls(x;(),y;(#y)#z);(*. x)y)};bs:{BP[$x],:y!bp[x;y,:();0xff];};bu:{bp[x;y;BP[$x;y,:()]]} 184 | bd:{bu[x;y];@[`BP;$x;_;y]};bc:{bu[x;y];dbg::{[d;f;i;r;u;v]dbg::d;bs[f;i];$[#*u;d[u;v];r]}[dbg;x;y;z];-1} 185 | 186 | \d .h / c 187 | htc:{,/("<";x;y;"")};hta:{,/("<";$x;,/" ",'($!y),'"=",'{$[10h=@x;"\"",x,"\"";$x]}'. y;">")};htac:{,/(hta[x]y;z;"")} 188 | ha:{htac[`a;(,`href)!,x]y};hb:{htac[`a;`target`href!`v,,x]y};pre:{htc[`pre]@`/:x};xmp:{htc[`xmp]@`/:x} 189 | d:" ";cd:{.q.csv 0:$[.Q.qt x;![x;();0b;(!t)[c]!,:'.q.sv[d]@/:'$v c:&(~l=-10h)&0>l:.Q.tx'v:. t:+0!x];x]};td:{"\t"0:x};hc:{"<"/:"<"\:x};xs:.q.ssr/[;"&<>";("&";"<";">")] 190 | xd:{g:{(#*y)#'(,,"<",x),y,,,""};(,""),(,/'+g[`r]@,/(!x)g'{,xs'$[11h=@x;$x;t&77h>t:@x;$x;x]}'x:+0!x),,""} 191 | 192 | ex:("";);iso8601:{$[^x;"";@[$"p"$x;4 7 10;:;"--T"]]} 193 | eb:{htac[`Workbook;(`$"xmlns",/:$``:ss`:o`:x`:html)!("urn:schemas-microsoft-com:office:",/:$(2#`spreadsheet),`office`excel),,"http://www.w3.org/TR/REC-html40"]x} 194 | es:{htac[`Worksheet;(,`ss:Name)!,$x]htc[`Table]@,/(htc[`Row]@,/ec')'(,!+y),+.+y:0!y};ed:{ex eb es[`Sheet1]x};edsn:{ex eb@"\r\n"/:(!x)es'. x} 195 | ec:{htc[`Cell]htac[`Data;(,`ss:Type)!,$`String`Number`String`DateTime`DateTime`String i](xs;$:;xs@$:;iso8601;iso8601 1899.12.31+"n"$;xs@$:)[i:-10 1 10 12 16 20h bin -@x]x} 196 | 197 | \d .j /[]{} Cbg*xhijefcspmdznuvt 198 | e:{"c"$$[x<128;x;0b/:'@[;0;(1b,i#1b),]10b,/:((5 10;0 4 10)i:x>2047h)_0b\:"h"$x]}@256/: 199 | q:"\"";s:{q,x,q};es:{s@,/{$[x in r:"\t\n\r\"\\";"\\","tnr\"\\"r?x;x]}'x};J:(($`0`1)!$`false`true;s;{$[#x;x;"null"]};es;{s@[x;&"."=8#x;:;"-"]};s)1 2 5 11 12 16h bin 200 | /j:{$[10=abs t:@x;es x;(::)~x;"null";99=t;"{",(","/:({$[q~*x;x;s x]}'j'!x),'":",'j'. x),"}";-1*x;"F"$x;"n"=*x;0n;"t"=*x]} 202 | /k:{c x@&~v[x]&x in" \t\n\r"};v:{~0(0 1 0;1 0 2;1 1 1)\x*3>x:" \"\\"?x};u:{?[v x;x;" "]};d:{$[1" 207 | 208 | c0:`024C7E;c1:`958600;logo:"kx.com" /logo:,/(c0;c1;c0){htac[`font;(,`color)!,x]y}'("[kx";"systems";"]") /808000 209 | logo:,/(c0;c1){htac[`font;(,`color)!,x]htc[`b]y}'("kx";".com") 210 | sa:"a{text-decoration:none}",/`link`visited`active{"a:",($x),"{color:",y,"}"}'$(c0;c0;c1) 211 | html:{htc[`html]htc[`head;htc[`style]sa,sb],htc[`body]x};sb:"body{font:10pt verdana;text-align:justify}" 212 | fram:{htc[`html]htc[`head]htc[`title;x],htac[`frameset;(,`cols)!,($116|316&43+(7+"?"~*z)*|/#:'y),",*"]@,/hta[`frame]'((,`src)!,*z;`name`src!`v,,*|z)} 213 | jx:{[j;x]x:val x;$[$[.Q.qt[x];(N:(*."\\C")-4)";hr:{(#x)#"-"};nbr:htc[`nobr];code:{" "/:@[x;1+2*!_.5*#x:"\t"\:x;{htc[`code]nbr hc x}]} 228 | http:{$[#i:ss[x]"http://";(i[0]#x),/{ha[x;x:i#x],(i:&/x?") ")_x}'i_x;x]};text:{`/:{htc[`p]http code x}'x@&0<#:'x} 229 | data:{if[100<#*x:.Q.tab[1]n#'x,\:(n:|/#:'x:"\t"\:'x:1_'x)#,"";0N!#*x;0N!'x];xmp$["- "~2#x 1;@[x;1;hr];x]} 230 | ht:{[t]x:0:`$"src/",(T:t:$t),".txt";if[~"."=**x;T:*x;x:1_x];h:.q.trim(n@:i)_'x i:&0#x;x,1;(y#x),1+x y-:1]}\n),'" ",/:h 232 | x:`/:("Copyright © ",logo;htc[`h5]T),((#x)#(text;data))@'x:(&~=':~(0<#:'x)&" "=*:'x)_x 233 | c:h{hb[x,"#",y;z],br}[a:t,".htm"]'H;c:nbr@`/:$[&/n=:1;c;1_,/(,br),/:(&n)_c] 234 | (`$a)1:fram[T;H]b:"a/",/:("_",a;a);(`$b)1:'html'(($.z.D),htc[`h5;"Contents"],c;x);} 235 | 236 | \d .o /odbc timedec/cs could be 3(23,12) 237 | ex:{$[.Q.qt x:."s)",x;0!x;x]} / char(1+) binary(display2*) -1 longvarchar -4 longvarbinary sql_no_total(-4)SAS/4096 238 | T:`text`bit```tinyint`smallint`int`bigint`real`float`char`varchar``date`date`datetime``time`time`time 239 | T0: -1 -7 0 0 -6 5 4 -5 7 8 1 12 0 9 9 11 0 10 10 10h 240 | B0:4096 1 0 0 1 2 4 8 4 8 1 255 0 6 6 16 0 6 6 6 241 | C0:4096 1 0 0 3 5 10 19 7 15 1 255 0 10 10 23 0 8 8 8 242 | PS: 1 2 0 0 2 2 2 2 2 2 3 3 0 2 2 2 0 2 2 2h /char basic both 243 | t:0 1 4 5 6 7 8 9 10 11 14 15 19;Columns:{$[#x;Cols`$x;,/Cols'.q.tables`.]} 244 | q)TI:`DATA_TYPE xasc([]TYPE_NAME:T t;DATA_TYPE:T0 t;COLUMN_SIZE:C0 t;LITERAL_PREFIX:`;LITERAL_SUFFIX:`;CREATE_PARAMS:`;NULLABLE:1h;CASE_SENSITIVE:1h;SEARCHABLE:PS t;UNSIGNED_ATTRIBUTE:0h;FIXED_PREC_SCALE:0h;AUTO_INCREMENT:0h;LOCAL_TYPE_NAME:`;MINIMUM_SCALE:0h;MAXIMUM_SCALE:0h) 245 | q)TypeInfo:{$[count x;select from TI where DATA_TYPE="H"$x;TI]} /msqry32 uses special e.g. select p.i from p p 246 | q)Special:{([]SCOPE:2h;COLUMN_NAME:(0&0 similar position substring trim {lower|upper} 262 | -------------------------------------------------------------------------------- /qAbi/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv, fetchFromGitHub, gmp, ... }: 2 | stdenv.mkDerivation { 3 | pname = "qAbiEncode"; 4 | version = "0.0.0+unstable"; 5 | 6 | src = fetchFromGitHub { 7 | owner = "manifoldfinance"; 8 | repo = "qAbiEncode"; 9 | rev = "4445a051ec6b7f457dff5b3a18238c064210390a"; 10 | sha256 = "14ib6pi2j27mfm8i9p3144n5v0g4hkrmdggl0l0g7yi7x5ckkhkj"; 11 | }; 12 | 13 | NIX_CFLAGS_COMPILE = "-I${gmp.dev}/include"; 14 | NIX_CFLAGS_LINK = "-L${gmp.out}/lib"; 15 | 16 | installPhase = '' 17 | ls -la 18 | install -Dm644 abi.q $out/q/abi.q 19 | install -Dm644 qKeccak.so $out/q/l64/qKeccak.so 20 | strip $out/q/l64/*.so 21 | ''; 22 | } -------------------------------------------------------------------------------- /qBigint/default.nix: -------------------------------------------------------------------------------- 1 | { stdenv, fetchFromGitHub, gmp, ... }: 2 | stdenv.mkDerivation { 3 | pname = "qAbiEncode"; 4 | version = "0.0.0+unstable"; 5 | 6 | src = fetchFromGitHub { 7 | owner = "manifoldfinance"; 8 | repo = "qAbiEncode"; 9 | rev = "4445a051ec6b7f457dff5b3a18238c064210390a"; 10 | sha256 = "14ib6pi2j27mfm8i9p3144n5v0g4hkrmdggl0l0g7yi7x5ckkhkj"; 11 | }; 12 | 13 | NIX_CFLAGS_COMPILE = "-I${gmp.dev}/include"; 14 | NIX_CFLAGS_LINK = "-L${gmp.out}/lib"; 15 | 16 | installPhase = '' 17 | ls -la 18 | install -Dm644 abi.q $out/q/abi.q 19 | install -Dm644 qKeccak.so $out/q/l64/qKeccak.so 20 | strip $out/q/l64/*.so 21 | ''; 22 | } -------------------------------------------------------------------------------- /qUnit/default.nix: -------------------------------------------------------------------------------- 1 | { lib, runCommand, fetchurl }: 2 | let 3 | src = fetchurl { 4 | url = "https://raw.githubusercontent.com/timeseries/kdb/18822dd8d932d429b8ebacbbde2e3624440d5777/qunit/qunit.q"; 5 | hash = "sha256-UaWtQ8WSVAhQb6mAoKon42iNeTvbAUg9ltABd0RNgrA="; 6 | }; 7 | in 8 | runCommand "qunit" { } '' 9 | mkdir -p $out 10 | install -D ${src} $out/q/qunit.q 11 | '' -------------------------------------------------------------------------------- /qsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # append to ~/.bashrc 3 | # alias to invoke kdb+ q-lang 4 | q() { 5 | ~/q/l64/q "$@" 6 | } 7 | export -f q 8 | -------------------------------------------------------------------------------- /src/bundler.q: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env q 2 | 3 | // Copyright (c) 2022 Manifold Finance, Inc. 4 | // This Source Code Form is subject to the terms of the Mozilla Public 5 | // License, v. 2.0. If a copy of the MPL was not distributed with this 6 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | // 8 | // bundler.q 9 | // receive latest block data and swap requests 10 | // aggregate by block target, market, side 11 | // order by tx amount desc 12 | // calculate net 13 | // interlace buy / sell orders for minimum slippage with largest orders first 14 | // estimate max arb for slippage requirements 15 | // produce bundles 16 | // pick top estimated profit bundle per block and submit to producer 17 | // dissipate un submitted txs back to the pool for re-aggregation 18 | // set timer function for syncing hdb bundle data 19 | 20 | // start with 21 | // q bundler.q -p 5002 -s 2 22 | 23 | // load libraries 24 | \l domain/kdb_schema.q 25 | \l domain/proto.q 26 | \l utils/hex.q 27 | \l utils/assert.q 28 | \l utils/atomize.q 29 | \l utils/os.q 30 | \l utils/str.q 31 | \l log4q.q 32 | \l abi.q 33 | \l quartic.q 34 | 35 | INFO "msg=\"Initializing bundler\" id=init"; 36 | 37 | / load bundle helpers 38 | \l utils/constants.q 39 | \l utils/globals.q 40 | \l utils/state.q 41 | \l utils/simulate.q 42 | \l utils/strategies.q 43 | \l utils/format.q 44 | \l utils/store.q 45 | 46 | // @kind function 47 | // @desc Submits a bundle to kafka 48 | // @param kfkBundle {dict} `serialBundle`serialKey!(serialised Bundle; serialised Key) 49 | submitBundle: {[kfkBundle] 50 | INFO "msg=\"Publishing bundle to kafka\""; 51 | assert[ type kfkBundle; =; 99h; "bundle must be a dictionary" ]; 52 | assert[ all `serialBundle`serialKey in key kfkBundle; =; 1b; "bundle dictionary must have keys serialBundle & serialKey" ]; 53 | HandleProducer (`publishBundle; kfkBundle`serialBundle; kfkBundle`serialKey); // sync req to flush queue 54 | }; 55 | 56 | 57 | // @kind function 58 | // @desc Triggered by new liquidity data arrival 59 | processBlock: { 60 | cleanBundles[]; 61 | cleanSwaps[]; 62 | updatePlatAccState[]; 63 | updateNextBaseFee[]; 64 | 65 | DEBUG ( 66 | "id=processBlock platform_account=%1 account_nonce=%2 ether=%3 next_base_fee=%4"; 67 | (PlAcc; "J"$convertToDec PlAccNonce; convertHexStrToFloat PlAccEth; convertHexStrToFloat NextBaseFee)); 68 | 69 | // select distinct user swap paths 70 | swapGroups: select sum chainId by to, path from swaps; 71 | paths: (key swapGroups)[`path]; 72 | rtrs: (key swapGroups)[`to]; 73 | numPaths: count paths; 74 | if[0 < numPaths; 75 | i: 0; 76 | do[numPaths; 77 | indy: exec first i from (key swapGroups) where path~\:(reverse paths[i]), to~\:rtrs[i]; 78 | if[ -1 < indy; paths: paths _ indy; rtrs: rtrs _ indy; ]; 79 | i: i+1]; / remove reverse paths, as they are aggregated in calcBundle 80 | DEBUG ("id=processBlock num_sandwich_bundles_to_process=%1"; i); 81 | 82 | / 1. sandwich batch / swaps 83 | calcBundle'[rtrs; paths]; / batch and calculate all possible bundles grouped by path 84 | if[DISABLE_SANDWICHES = 0b; 85 | pendingBundles: select from bundleStatus where 0i=statusId.statusId; / Get pending bundles 86 | numBundles: count pendingBundles; / bundles count 87 | if[numBundles > 0; / profitable bundle produced 88 | topBundleStatus: first sortTableHexDesc[pendingBundles; `profit_eth]; / get top bundle by profit in eth 89 | INFO ("id=processBlock num_profitable_sandwich_bundles=%1"; (numBundles)); 90 | INFO ("msg=\"Best sandwich bundle\" id=processBlock est_profit_eth=%1"; (convertHexStrToFloat topBundleStatus`profit_eth)%1e18); 91 | assert[ cmp[ topBundleStatus`profit_eth; 0x0000 ]; >; 0i; "top bundle must be profitable" ]; 92 | kfkBundle: formatBundle[topBundleStatus]; / format bundle to submit to kafka 93 | submitBundle[kfkBundle]; / submit top bundle 94 | saveBundle[topBundleStatus]; / save bundle status to disk 95 | ]; 96 | ]; 97 | 98 | / 2. back-running 99 | backRunArbInfo: calcBackRunDexArbs[]; 100 | backRunProfitable: backRunArbInfo[0]; 101 | if[0i < backRunProfitable; / Is top dex arb profitable? 102 | topBackRunDex: backRunArbInfo[1]; / top dex arb record 103 | kfkBundle: formatBackRunDexBundle[ topBackRunDex]; / format bundle to submit to kafka 104 | submitBundle[ kfkBundle ]; 105 | saveBackRunDexBundle[topBackRunDex]; / save bundle status to disk 106 | ]; 107 | ]; 108 | / 3. simple state dex arbs 109 | dexArbInfo: calcDexArbs[]; 110 | dexProfitable: dexArbInfo[0]; 111 | if[0i < dexProfitable; / Is top dex arb profitable? 112 | topDex: dexArbInfo[1]; / top dex arb record 113 | kfkBundle: formatDexBundle[ topDex]; / format bundle to submit to kafka 114 | submitBundle[ kfkBundle ]; 115 | saveDexBundle[topDex]; / save bundle status to disk 116 | ]; 117 | / 4. state tri arbs 118 | triArbInfo: calcTriArbs[]; 119 | trigProfitable: triArbInfo[0]; 120 | if[0i < trigProfitable; / Is top tri arb profitable? 121 | topTrig: triArbInfo[1]; / top triangular arb record 122 | kfkBundle: formatTriBundle[ topTrig]; / format bundle to submit to kafka 123 | submitBundle[ kfkBundle ]; 124 | saveTriBundle[topTrig]; / save bundle status to disk 125 | ]; 126 | 127 | / cleanup 128 | cleanData[]; 129 | }; 130 | 131 | // @kind function 132 | // @desc Sets latest block number and calls processBlock 133 | // @param blkNum {long} current eth block number 134 | setLatestBlock: {[blkNum] 135 | bNum: `long$blkNum; 136 | INFO ("msg=\"Received new block data\" id=setLatestBlock received_block_number=%1 previous_latest_block=%2"; (bNum; LatestBlock)); 137 | 138 | updateLatestBlockNumber[bNum]; 139 | 140 | // Check WETH existence 141 | if[not isWethIdSet[]; updateWethId[];]; 142 | 143 | // Only process block if WETH is set and we have latest information 144 | if[all (1b = isWethIdSet[]), 1b = shouldProcessBlock[]; 145 | processBlock[]; 146 | ]; 147 | }; 148 | 149 | // @kind function 150 | // @desc Loads selective state from HDB on startup 151 | loadState: { 152 | DEBUG "msg=\"Loading HDB state\" id=loadState"; 153 | 154 | accs: HandleHDB (`getAccounts;`); 155 | if[0 < count accs; 156 | `accounts insert accs; 157 | ]; 158 | 159 | toks: HandleHDB (`getTokens;`); 160 | if[0 < count toks; 161 | `tokens insert toks; 162 | if[not isWethIdSet[]; updateWethId[];]; 163 | ]; 164 | 165 | pairs: HandleHDB (`getPairs;`); 166 | if[0 < count pairs; 167 | `sushiPairs insert pairs; 168 | ]; 169 | 170 | pltAcc: HandleHDB (`getpltAcc;`); 171 | if[0 < count pltAcc; 172 | `platformAccount upsert pltAcc; 173 | ]; 174 | 175 | pltToks: HandleHDB (`getpltToks;`); 176 | if[0 < count pltToks; 177 | `platformToken insert pltToks; 178 | ]; 179 | 180 | }; 181 | 182 | / load HDB state on startup 183 | loadState[]; 184 | -------------------------------------------------------------------------------- /src/core.q: -------------------------------------------------------------------------------- 1 | /each: +-*%&|^<>=$ <= >= <> @ ? in within bin div abs log exp sqrt sin cos tan f' f\: f/: 2 | neg:-:;not:~:;null:^:;string:$:;reciprocal:%:;floor:_:;ceiling:-_-:;signum:{(x>0)-x<0} 3 | mod:{x-y*x div y};xbar:{x*y div x:$[16h=abs[@x];"j"$x;x]};xlog:{log[y]%log x};and:&;or:|;each:{x'y};scan:{x\y};over:{x/y};prior:{x':y} 4 | mmu:$;lsq:!;inv:!:;md5:-15!;ltime:%:;gtime:{t+x-%t:x+x-%x}; /xnull:{$[0>@y;(,y)@~x=y;x=y;y 0N;y]} 5 | 6 | /aggr: last sum prd min max avg wsum wavg f/ /beta:{cov[x;y]%var x} 7 | count:#:;first:*:;svar:{(n*var x)%-1+n:(#x)-+/^x};sdev:{sqrt svar x};scov:{(n*cov[x;y])%-1+n:(#x)-+/^x+y};med:{avg x(@x;'`rank;1_x,,x 0N]}; 11 | rank:{$[0h>@x;'`rank;<@x;'`rank;@x;'`rank;>x]} 12 | asc:{$[99h=@x;(!x)[i]!`s#r i:@x;'`rank;`s#x@r:. x;0h>@x;'`rank;x@>x]} 13 | 14 | msum:{$[99h=@y;(!y)!.z.s[x;. y];y-(-x)_(0i*x#y),y:+\y]};mcount:{msum[x;~^y]};mavg:{msum[x;0.0^y]%mcount[x;y]};mdev:{sqrt mavg[x;y*y]-m*m:mavg[x;y:"f"$y]} 15 | xrank:{$[0h>@y;'`rank;_y*x%#y:<@y;'`rank;y(!#y)-x]};rotate:{$[0h>@y;'`rank;98h<@y;'`type;#y;,/|(0;mod[x;#y])_y;y]};ema:{(*y)(1f-x)\x*y} 16 | 17 | /other: ~,#_ !. getenv exit 18 | distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} 19 | set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? 20 | raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} 21 | sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} 22 | 23 | /file&comm 24 | read0:0::;read1:1::;hclose:>:;hdel:~:;hsym:"s"$-1!';hcount:-7!;peach:{x':y};system:."\\", 25 | 26 | /string: like ss 27 | ltrim:{$[~t&77h>t:@x;.z.s'x;^*x;((^x)?0b)_x;x]};rtrim:{$[~t&77h>t:@x;.z.s'x;^last x;(-(|^x)?0b)_x;x]};trim:{ltrim rtrim x} 28 | lower:{$[$[(~@x)&10h~@*x;&/10h=@:'x;0b];_x;~t&77h>t:abs@@x;.z.s'x;19t:abs@@x;.z.s'x;19@z;:[;z];z]]} 31 | 32 | /select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation 33 | /{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) 34 | view:{(2+*x ss"::")_x:*|*|.`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} 35 | cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} 36 | xcols:{(x,f@&~(f:cols y)in x)#y};keys:{$[98h=@x:.Q.v x;0#`;!+!x]};xkey:{(#x)!.[0!y;();xcols x]}; 37 | xcol:{.Q.ft[{+$[99h=@x;@[!y;(!y)?!x;:;. x];x,(#x)_!y]!. y:+y}x]y};xasc:{$[$[#x;~`s=-2!(0!.Q.v y)x;0];.Q.ft[@[;*x;`s#]].Q.ord[<:;x]y;y]};xdesc:{$[#x;.Q.ord[>:;x]y;y]} 38 | fkeys:{(&~^x)#x:.Q.fk'.Q.V x};meta:{([!c].Q.ty't;f:.Q.fk't;a:-2!'t:. c:.Q.V x)} 39 | 40 | / R uj R(union join) R lj K(left(equi/asof)join) trade asof`sym`time!(`IBM;09:31:00.0) 41 | lj:{$[$[99h=@y;(98h=@!y)&98h=@. y;()~y];.Q.ft[,\:[;y];x];'"type"]} /;la:{$[&/j:z>-1;x,'y z;+.[+ff[x]y;(!+y;j);:;.+y z j:&j]]}{la[x;. y](!y)?(!+!y)#x}[;y]]x} /lj:,\:;aj:{lj[y]`s#xkey[x]z};aj0:{lj[y]`s#(x#z)!z}; /;bn:{@[i;&0>i:x bin y;:;#x]} 42 | ljf:{$[`s=-2!y;ajf[!+!y;x;0!y];.Q.ft[{$[&/j:(#y:. y)>i?:(!+i:!y)#x;.Q.fl[x]y i;+.[+x;(f;j);:;.+.Q.fl[((f:!+y)#x:.Q.ff[x]y)j]y i j:&j]]}[;y]]x]} 43 | .Q.ajf0:{[f;g;x;y;z]x,:();z:0!z;d:$[g;x_z;z];g:(:;^)f;f:(,;^)f;$[&/j:-1i:(!y)?(!+!y)#x}[;y]]x} 46 | ijf:{.Q.ft[{.Q.fl[x j]y i j:&(#y:. y)>i?:(!+i:!y)#x}[;y]]x} 47 | pj:{.Q.ft[{x+0i^y(!+!y)#x}[;y]]x};asof:{f:!$[99h=@y;y;+y];(f_x)(f#x)bin y} 48 | uj:{$[()~x;y;()~y;x;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];lj[(?(!x),!y)#x]y]} 49 | ujf:{$[()~x;y;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];ljf[(?(!x),!y)#x]y]} 50 | 51 | /wj[-1000 2000+\:trade`time;`sym`time;trade;(quote;(max;`ask);(min;`bid))] (given `sym`time xasc quote) 52 | ww:{[a;w;f;y;z]f,:();e:1_z;z:*z;y,'n#+(:/'f)!+{[e;d;a;b]e .'d@\:\:a+!b-a}[*:'e;z f:1_'e]/'$[n:#*w;+$[#g;(g#z)?g#y;0]|/:a+$[#g:-1_f;(f#z)bin@[f#y;*|f;:;]@;z[*f]bin]'w;,0 0]} 53 | wj:{[w;f;y;z]ww[0 1;w;f;y;z]};wj1:{[w;f;y;z]ww[1;w-1 0;f;y;z]} 54 | 55 | fby:{$[(#x 1)=#y;@[(#y)#x[0]0#x 1;g;:;x[0]'x[1]g:.=y];'`length]};xgroup:{x,:();a:x#y:0!y;$[#x_:y;+:'x@=a;a!+f!(#f:!+x)#()]};ungroup:{$[#x:0!x;,/+:'x;x]} 56 | ej:{x,:();y[&#:'i],'(x_z)(!0),/i:(=x#z:0!z)x#y:0!y} /{ungroup lj[z]xgroup[x]y} 57 | 58 | /`[:../]t[.{csv|txt}] 59 | save:{$[1=#p:`\:*|`\:x:-1!x;set[x;. *p]; x 0:.h.tx[p 1]@.*p]}' 60 | load:{$[1=#p:`\:*|`\:x:-1!x;set[*p;. x];set[*p].h.xt[p 1]@0:x]}' 61 | rsave:{x:-1!x;.[`/:x,`;();:;.*|`\:x]}' 62 | rload:{x:-1!x;.[*|`\:x;();:;. x]}' 63 | dsave:{.[*x;1_x,y,`;:;@[;*!+a;`p#].Q.en[*x]a:. y];y}/: 64 | 65 | show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" 66 | 67 | parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! 68 | \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") 69 | k:4.0;K:2020.05.04;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc 70 | res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp 71 | addmonths:{("d"$m+y)+x-"d"$m:"m"$x} 72 | Xf:{y 1:0xfe20,("x"$77+@x$()),13#0x00;(`$($y),"#")1:0x};Cf:Xf`char 73 | f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} 74 | fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} 75 | 76 | /DO NOT USE ROUTINES PAST HERE. SUBJECT TO CHANGE 77 | ff:{$[&/(!+y)in f:!+x;x;x,'(f_y)(#x)#0N]} 78 | fl:{$[98h=t:@x;+fl[+x;+y];99h=t;@[x,y;f;:;x[f]fl'y f@:&(f:!x)in!y];t&t<77h;x^y;@[y;i;:;x i:&(0<#:'x)&~#:'y]]} 79 | opt:{[o]x::$[#i:&o like"-[^0-9]*";i[0]#o;o];((`$1_*:)'o)!1_'o:i_o};def:{x,((!y)#x){$[0h>@x;*:;::](@*x)$y}'y} 80 | qt:{$[99h=@x;(98h=@!x)&98h=@. x;98h=@x]}; v:{$[-11h=@x;.$[":"=*t:$x;`$t,"/";x];x]}; 81 | qp:{$[~98h=@x;0;@x:.+x;~":"=*$x;0]}; V:{$[qp x:v x;((,pf)!,. pf),+$[(~x in!d0[])&(x:.+x)in !*. .Q.vt;.Q.vp x;d0[]@x];0h>@*x:+0!x;x@'!x;x]} 82 | ft:{$[{$[99h=@t:v x;98h=@. t;0]}y;[n:#+!y;n!x 0!y];x y]}; 83 | ord:{if[~&/b:{(!#x)=x?x}c:.q.cols z;'"dup ",$c b?0b];if[~&/b:(y,:())in c;'y b?0b];ft[@[;!+t:0!v z;::]]z;ft[@[;!+t;@[;x@+y!t y]]]z};nv:{$["."~*$x;x;`$"..",$x]} 84 | 85 | tx:{$[(76ht;t;11h=t:@x:. nv@!x;t;98h=t;7h;1=#x:+!x;tx@*x;7h]};tt:0|tx'. V@ 86 | fk:{$[(20h>t)|76h@. nv t:!x;`;t]};t:" bg xhijefcspmdznuvts";ty:{$[0h>x:tx x;.q.upper t@-x;t x]} 87 | nct:{$[`~x;,/nct'.q.tables`.;([]n:x;c:.q.cols x;t:tt x)]} 88 | 89 | fu:{[f;x]$[0h>@x;f x;f[u](u:?x)?x]} /uniques 90 | fc:{$[1@x;`${$[(*x)in n;"a",x;x]}x@&(x:$x)in an;ft[{s[i]:`$($s i:&(s in`i,res,!`.q)|~(s?s)=!#s:id'!x:+x),'"1";+s!. x}]x]} 93 | j10:64/:b6?;x10:b6@0x40\: /base64 J from char10 94 | j12:36/:nA?;x12:nA@0x24\: /base36 J from char12(cusip) 95 | btoa:-32!;sha1:-33!;prf0:+`name`file`line`col`text`pos!*-37! 96 | l:{if[$[1>@d:!f:-1!x;1;`.d~*d];:.[$[qt d;*|`\:f;99=@d;`.;'`type];();:;d:. f]];d@:&~d like"*$";p:(d=`par.txt)|d like"[0-9]*"; 97 | ."\\cd ",$x;{.q.set[x]$[0h>@!a:-1!x;. a;a`]}'d@&~(d=`html)|(d like"*#")|p|s:d like"*.?";if[|/p;L d@&p];if[~`.=x;(."\\l ",$:)'d@&s&~p];} 98 | 99 | sw:{.q.sublist[_.5**|."\\C"]x};tab:{" "/:'(x*|/#:''y)$/:y};t0:{$[x>#y;((#*y)#'" -"),y;y]} 100 | s1:{-3!x} /{$[0>t:@x;$x;99h76h;s1'x;t;$x;|/(97ht:?@:'x;s1'x;(1<#?#:'x)|$[1=#t;(*t)in 1 4 10h;0];s1'x;tab[1]@s2'sw'x]} 103 | S:{x[1]{$[x<#y;((x-2)#y),"..";y]}'$[t&77h>t:@z;,s1 z;99h#x)|(@x)&~11=@x} 111 | 112 | / CAN EXIT HERE FOR SMALL Q 113 | / pt(tables) pf(date/month/year/int) pd(dirs) pv(values) pn(count) pt::0#pf::` 114 | vt:(,`)!,()!(); 115 | bv:{g:$[(::)~x;max;min];x:`:.;d:{`/:'x,'d@&(d:!x)like"[0-9]*"}'P:$[`par.txt in!x;-1!'`$0:`/:x,`par.txt;,x]; 116 | t:?,/!:'.Q.vt:{(&#:'x)(=,/. x)}'{({("DMJJ"`date`month`year`int?.Q.pf)$$last@x:`\:x}'x)!!:'x}'d; 117 | d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#`. pf];(i;x i)}[;g]'+:t#/:g''.Q.vt:t#/:.Q.vt;.Q.vt:P!.q.except[. .Q.pf]''.Q.vt; 118 | .Q.vp:t!{(+(,.Q.pf)!,0#. .Q.pf),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}'d;.Q.pt,:{.[x;();:;+.q.except[!+.Q.vp x;.Q.pf]!x];x}'.q.except[t;.Q.pt];} 119 | 120 | pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!{(`u#pd,'pv)!p2[(x;();0b;())]'[pd;pv]}'pt]} 121 | dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;p1[;y;z]]} 122 | 123 | p:{$[~#D;p2[x;`:.]':y;(,/p2[x]'/':P[i](;)'y)@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} 124 | view:{pd::PD x:$[(::)~x;x;&PV in x];u~:?u::..[pf;();:;pv::PV x];x{.[y;();:;+(x . y,`.d)!y]}'pt::!x:d0[];pn::pt!(#pt)#()} 125 | L:{D::();if[x~,`par.txt;if[~#x:,/D::!:'P::`$":",'0:*x;'`empty]];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} 127 | /L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} 128 | 129 | cn:{$[#n:pn x:.+x;n;pn[x]:(#p1 .)':+(x;pd;pv)]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} 130 | ind:{,/i[j]{fp[pf;p]p1[x;pd y;p:pv y]z}[.+x]'(j:&~=':i)_y-n i:(n:+\0,cn x)bin y} 131 | fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} 132 | foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} 133 | 134 | / select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. 135 | a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} 136 | {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) 137 | 138 | / x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? 139 | qd:{$[(#:)~*x;(?:)~*x 1;0]};xy:{`$$*&x~/:y}; x1:{$[qb x;();IN[*x;a0];$[qd x;1_x;,x];,/x1'1_x]} 140 | x0:{$[qb x;x;IN[*x;a1];x0 a2[a1?*x]. 1_x;x0'x]};x2:{$[qb x; x;IN[*x;a0];$[qd x;(#:;(?:;(,/;xy[x 1]y)));[y:xy[x]y;$[(?:)~*x;(?:;(,/;y));(#:)~*x;(sum;y);(*x;y)]]];x2[;y]'x]} 141 | ua:{((`$$!#u)!u;x2[;u:?,/x1'x]'x:x0'x)};q0:{$[~qb x;,/q0'x;-11h=@x;*`\:x;()]};qe:{$[#x;99h=@x;1]} 142 | ps:{[t;c;b;a]if[-11h=@t;t:. t];if[~qe[a]&qe[b]|-1h=@b;'`nyi];d:pv;v:$[q:0>@b;0;~#b;0;-11h=@v:*. b;pf~*`\:v;0] 143 | if[$[~#c;0;@*c;0;-11h=@x:c[0]1;pf~*`\:x;0];d@:&-6!*c;c:1_c] 144 | if[$[#c;0;(g:(. a)~,pf)|(. a)~,(#:;`i)];f:!a;j:dt[d]t;if[q;:+f!,$[g;?d@&0(#.:)'t:.q.tables`.;if[h:@[hopen;h;0];h"\\l .";>h]} 159 | 160 | /loop through text /lc:{+/{+/0xa=1:(x;y*z;y)}[x;m]'!-_-(-7!x)%m:1048576} /line count of big file 161 | fsn:{[f;s;n]>[-7!s]{[f;s;x;n]i:(#r)^1+last@&"\n"=r:1:(s;x;n);f@`\:i#r;x+i}[f;s;;n]/0};fs:fsn[;;131000] 162 | fpn:{{r:.Q.trp[y;h;{>x;'(y;z)}h:hopen":fifo://",1_$x];>h;r}[y 163 | {[f;h;n]b:"x"$();while[#s:1:h;if[~n>#b,:s;v:`\:b,0x0a;if[1<#v;f@-1_v];b:"x"$last v]];if[#b;f@`\:b]}[x;;z]]};fps:fpn[;;131000] 164 | /e.g. p:` sv(d:`:dir;`2007.02.12;`t);@[;`sym;`p#]p{@[x;`;,;`sym xasc .Q.en[d]get y]}/rdb's 165 | dsftg:{[dpt;xom;f;tw;g]d:*dpt;dpt:par . dpt;x:*xom;if[~0m:_a:(m-:o:xom 1)%b:+/tw 10=@*tw;'`length];0N!($dpt),": ",$m&:M;i::0;n::1000000&_1e8%b 167 | do[-_-m%n;@[dpt;`;$[i;,;:];en[d]@+g f!tw 1:(x;o+b*i;b*n&:m-i)];0N!i+:n];dpt};M:0W 168 | 169 | / fill in empty partitions 170 | chk:{if[x~(::);'"missing dir argument"];f:{`/:'x,'d@&(d:!x)like"[0-9]*"};d@:>.:'$last'`\:'d:$[`par.txt in!x;,/f'-1!'`$0:`/:x,`par.txt;f x] 171 | {[e;u;d]u[i]{.[x;(y;`);:;?[z;();0b;()]]}[d]'e i:&~u in!d}[d[(+u in/:t)?\:1b](0#.)'u,'`;u:?,/t:!:'d]'d} 172 | 173 | Ll:-100!;Lp:-101!;Lx:-102!;Lu:-103!;Ls:-104!;fqk:{("<"~*x)|x~*-3#.{}};fql:{x[1;1]~""};btx:{r:,Lp x;while[@x:Lu(0b;x);r,:(~fql v)#,v:Lp x;if[(fqk v[1]1)&".Q.dbg"~*v 1;f:(Lx(x;"k";,"v"))1]];$[~#f;r;@[r;r?f;,;,">>"]]};bt:{1'pl'(2*4<|/#:'v)_v:1_btx Ll`;};sbt:{,/pl'x};trp:{-105!(x;,y;z)} 174 | dr:{0:0};dw:{@[2@;x;::];};pl0:{.[{[f;v;x;y;z]g:{$[x>0;x&#y;x|-#y]#y};m:1;n:0|(b:+\0,1+#:'s:`\:s)bin y:(0|y)+2*"k)"~2#s:x 3;sf:$[#x 1;x[1],$[0>x 2;"";":",($n+x 2)],": ";""];sn:{$[(#x)&@x;((2*".."~2#x)_x),":";()]}@*x;h:($[4<#v;*|v;" "],5$"[",($z),"]") 175 | if[("locked"~x 3)|$[f;0b;fqk x 1];:h,"(",($[#sn;-1_sn;-2_sf]),")\n\n"];h,:sf,sn;h,`/:((n>m)#,""),g[-m+1;(n+1)#s],(,(" \t"(((#h)*n<1)#0),(y-b n)#s[n]="\t"),"^"),g[m](n+1)_s}[x;y];3#1_y;{dw"pl0\n";}]};pl:pl0 0 176 | jl8:{$[$[#x;~x[3]in y[;3];0b];,x;()],y};srr:{@[{if["abort"~*x;:""];"'",$[.z.q;($.z.Z)," ";""],(x[0],"\n"),/pl'jl8[x 3;x 2]};x;"srr\n"]};prr:{dw srr x}; 177 | lu:{f:{$[@a:Lu(x;*y);(a;Lp a);()]}x;r:{$[#x;fql x 1;0b]}f/f y;$[#r;$[.Q.dbg~**r 1;y;r];y]} 178 | DL:0;dbg:{pm:{dw" q"[x],$[1<#d:$."\\d";d;""]," )"[x],(1|DL)#")"};dq:{v:$[#*y;(~"k)"~2#y[1;3];*(.**y)3);1<#*y 1;@[*y 1;0;"q"=];(1b;`)];x v 1;*v}ds:{."\\d ",((~"."=*x)#"."),x:$x};de:@[{DL-:1;x y;{."\\x .z.e",x}'"xy"}ds;;::] 179 | @[{DL+:1;.z.ex:*x 1;.z.ey:x[1]1};x;::];b:$[#*v:Lp y;0xff~(*. bf:**v)bi:v[0;1];0b];if[b;dw "#",($bi),"\n"];w:(y0:y;v:Lp y);if[e:@*x;x[2;;3]+:1+v 3;prr x];d0:."\\d" 180 | y:*w:{$[~fqk x[1;1]1;x;lu[0b]x]}/w;q:dq v:w 1;dw pl v;while[*(~(,"\\")~l:dr[];pm[q]);$[l~"\\ ";q:~q;l in+,".`&";[w:$["&"=*l;;lu["."=*l]](y;v);y:*w;v:w 1;if[y~y0;prr x];dw pl0[1]v;q:dq v] 181 | (~e)&l in+,":>";:*($[b;bc[bf;bi];]0 0N(":"=*l);de d0);"'"=*l;:*((1_l;());de d0);c&~#*r:Lx(y;"kq"q;(c:":"=*l)_l);:*(r;de d0);$[#*r;::;dw $[q;.Q.s;{$[(::)~x;"";`/:,-3!x]}]r 1]]];de d0;("abort";())} 182 | err:{(*x;jl8[x 3;$[@y;btx y;()],x 2])}; BP:(,"")!,();bp:{*|(Ls(x;(),y;(#y)#z);(*. x)y)};bs:{BP[$x],:y!bp[x;y,:();0xff];};bu:{bp[x;y;BP[$x;y,:()]]} 183 | bd:{bu[x;y];@[`BP;$x;_;y]};bc:{bu[x;y];dbg::{[d;f;i;r;u;v]dbg::d;bs[f;i];$[#*u;d[u;v];r]}[dbg;x;y;z];-1} 184 | 185 | \d .h / c 186 | htc:{,/("<";x;y;"")};hta:{,/("<";$x;,/" ",'($!y),'"=",'{$[10h=@x;"\"",x,"\"";$x]}'. y;">")};htac:{,/(hta[x]y;z;"")} 187 | ha:{htac[`a;(,`href)!,x]y};hb:{htac[`a;`target`href!`v,,x]y};pre:{htc[`pre]@`/:x};xmp:{htc[`xmp]@`/:x} 188 | d:" ";cd:{.q.csv 0:$[.Q.qt x;![x;();0b;(!t)[c]!,:'.q.sv[d]@/:'$v c:&(~l=-10h)&0>l:.Q.tx'v:. t:+0!x];x]};td:{"\t"0:x};hc:{"<"/:"<"\:x};xs:.q.ssr/[;"&<>";("&";"<";">")] 189 | xd:{g:{(#*y)#'(,,"<",x),y,,,""};(,""),(,/'+g[`r]@,/(!x)g'{,xs'$[11h=@x;$x;t&77h>t:@x;$x;x]}'x:+0!x),,""} 190 | 191 | ex:("";);iso8601:{$[^x;"";@[$"p"$x;4 7 10;:;"--T"]]} 192 | eb:{htac[`Workbook;(`$"xmlns",/:$``:ss`:o`:x`:html)!("urn:schemas-microsoft-com:office:",/:$(2#`spreadsheet),`office`excel),,"http://www.w3.org/TR/REC-html40"]x} 193 | es:{htac[`Worksheet;(,`ss:Name)!,$x]htc[`Table]@,/(htc[`Row]@,/ec')'(,!+y),+.+y:0!y};ed:{ex eb es[`Sheet1]x};edsn:{ex eb@"\r\n"/:(!x)es'. x} 194 | ec:{htc[`Cell]htac[`Data;(,`ss:Type)!,$`String`Number`String`DateTime`DateTime`String i](xs;$:;xs@$:;iso8601;iso8601 1899.12.31+"n"$;xs@$:)[i:-10 1 10 12 16 20h bin -@x]x} 195 | 196 | \d .j /[]{} Cbg*xhijefcspmdznuvt 197 | e:{"c"$$[x<128;x;0b/:'@[;0;(1b,i#1b),]10b,/:((5 10;0 4 10)i:x>2047h)_0b\:"h"$x]}@256/: 198 | q:"\"";s:{q,x,q};es:{s@,/{$[x in r:"\t\n\r\"\\";"\\","tnr\"\\"r?x;x]}'x};J:(($`0`1)!$`false`true;s;{$[#x;x;"null"]};es;{s@[x;&"."=8#x;:;"-"]};s)1 2 5 11 12 16h bin 199 | /j:{$[10=abs t:@x;es x;(::)~x;"null";99=t;"{",(","/:({$[q~*x;x;s x]}'j'!x),'":",'j'. x),"}";-1*x;"F"$x;"n"=*x;0n;"t"=*x]} 201 | /k:{c x@&~v[x]&x in" \t\n\r"};v:{~0(0 1 0;1 0 2;1 1 1)\x*3>x:" \"\\"?x};u:{?[v x;x;" "]};d:{$[1" 206 | 207 | c0:`024C7E;c1:`958600;logo:"kx.com" /logo:,/(c0;c1;c0){htac[`font;(,`color)!,x]y}'("[kx";"systems";"]") /808000 208 | logo:,/(c0;c1){htac[`font;(,`color)!,x]htc[`b]y}'("kx";".com") 209 | sa:"a{text-decoration:none}",/`link`visited`active{"a:",($x),"{color:",y,"}"}'$(c0;c0;c1) 210 | html:{htc[`html]htc[`head;htc[`style]sa,sb],htc[`body]x};sb:"body{font:10pt verdana;text-align:justify}" 211 | fram:{htc[`html]htc[`head]htc[`title;x],htac[`frameset;(,`cols)!,($116|316&43+(7+"?"~*z)*|/#:'y),",*"]@,/hta[`frame]'((,`src)!,*z;`name`src!`v,,*|z)} 212 | jx:{[j;x]x:val x;$[$[.Q.qt[x];(N:(*."\\C")-4)";hr:{(#x)#"-"};nbr:htc[`nobr];code:{" "/:@[x;1+2*!_.5*#x:"\t"\:x;{htc[`code]nbr hc x}]} 227 | http:{$[#i:ss[x]"http://";(i[0]#x),/{ha[x;x:i#x],(i:&/x?") ")_x}'i_x;x]};text:{`/:{htc[`p]http code x}'x@&0<#:'x} 228 | data:{if[100<#*x:.Q.tab[1]n#'x,\:(n:|/#:'x:"\t"\:'x:1_'x)#,"";0N!#*x;0N!'x];xmp$["- "~2#x 1;@[x;1;hr];x]} 229 | ht:{[t]x:0:`$"src/",(T:t:$t),".txt";if[~"."=**x;T:*x;x:1_x];h:.q.trim(n@:i)_'x i:&0#x;x,1;(y#x),1+x y-:1]}\n),'" ",/:h 231 | x:`/:("Copyright © ",logo;htc[`h5]T),((#x)#(text;data))@'x:(&~=':~(0<#:'x)&" "=*:'x)_x 232 | c:h{hb[x,"#",y;z],br}[a:t,".htm"]'H;c:nbr@`/:$[&/n=:1;c;1_,/(,br),/:(&n)_c] 233 | (`$a)1:fram[T;H]b:"a/",/:("_",a;a);(`$b)1:'html'(($.z.D),htc[`h5;"Contents"],c;x);} 234 | 235 | \d .o /odbc timedec/cs could be 3(23,12) 236 | ex:{$[.Q.qt x:."s)",x;0!x;x]} / char(1+) binary(display2*) -1 longvarchar -4 longvarbinary sql_no_total(-4)SAS/4096 237 | T:`text`bit```tinyint`smallint`int`bigint`real`float`char`varchar``date`date`datetime``time`time`time 238 | T0: -1 -7 0 0 -6 5 4 -5 7 8 1 12 0 9 9 11 0 10 10 10h 239 | B0:4096 1 0 0 1 2 4 8 4 8 1 255 0 6 6 16 0 6 6 6 240 | C0:4096 1 0 0 3 5 10 19 7 15 1 255 0 10 10 23 0 8 8 8 241 | PS: 1 2 0 0 2 2 2 2 2 2 3 3 0 2 2 2 0 2 2 2h /char basic both 242 | t:0 1 4 5 6 7 8 9 10 11 14 15 19;Columns:{$[#x;Cols`$x;,/Cols'.q.tables`.]} 243 | q)TI:`DATA_TYPE xasc([]TYPE_NAME:T t;DATA_TYPE:T0 t;COLUMN_SIZE:C0 t;LITERAL_PREFIX:`;LITERAL_SUFFIX:`;CREATE_PARAMS:`;NULLABLE:1h;CASE_SENSITIVE:1h;SEARCHABLE:PS t;UNSIGNED_ATTRIBUTE:0h;FIXED_PREC_SCALE:0h;AUTO_INCREMENT:0h;LOCAL_TYPE_NAME:`;MINIMUM_SCALE:0h;MAXIMUM_SCALE:0h) 244 | q)TypeInfo:{$[count x;select from TI where DATA_TYPE="H"$x;TI]} /msqry32 uses special e.g. select p.i from p p 245 | q)Special:{([]SCOPE:2h;COLUMN_NAME:(0&0 similar position substring trim {lower|upper} -------------------------------------------------------------------------------- /src/env.q: -------------------------------------------------------------------------------- 1 | /other: ~,#_ !. getenv exit 2 | distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} 3 | set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? 4 | raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} 5 | sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} 6 | -------------------------------------------------------------------------------- /src/kdb_schema.q: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Manifold Finance, Inc. 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | tokens: ([tokenId: `int$()] 7 | token_symbol:`symbol$(); 8 | token_address: (); 9 | decimals:`int$()); 10 | 11 | accounts: ([accountId: `int$()] 12 | address: (); 13 | is_externally_owned: `boolean$()); 14 | 15 | v2Routers: ([routerId: 0 1i] 16 | routerName: `SUSHISWAP`UNISWAP; 17 | routerAddress: (0xd9e1ce17f2641f24ae83637ab66a2cca9c378b9f; 0x7a250d5630b4cf539739df2c5dacb4c659f2488d) 18 | ); 19 | 20 | sushiPairs: ([] 21 | id: `long$(); 22 | contract_address: (); 23 | base_token: `tokens$(); 24 | quote_token: `tokens$(); 25 | isActive: `boolean$() 26 | ); 27 | 28 | sushiLiquidity: ([] 29 | block_number: `long$(); 30 | base_token: `tokens$(); 31 | quote_token: `tokens$(); 32 | base_reserve: (); // bigInt byte array 33 | quote_reserve: (); // bigInt byte array 34 | block_timestamp: `long$() 35 | ); 36 | 37 | uniPairs: ([] 38 | id: `long$(); 39 | contract_address: (); 40 | base_token: `tokens$(); 41 | quote_token: `tokens$(); 42 | isActive: `boolean$() 43 | ); 44 | 45 | uniLiquidity: ([] 46 | block_number: `long$(); 47 | base_token: `tokens$(); 48 | quote_token: `tokens$(); 49 | base_reserve: (); // bigInt byte array 50 | quote_reserve: (); // bigInt byte array 51 | block_timestamp: `long$() 52 | ); 53 | 54 | 55 | platformAccount: ([address:()] 56 | isActive:`boolean$() 57 | ); 58 | 59 | platformToken: ([] 60 | address: (); 61 | isActive: `boolean$() 62 | ); 63 | 64 | accountBalance: ([] 65 | block_number: `long$(); 66 | accountId: `accounts$(); 67 | nonce:(); 68 | ether: () // bigInt byte array 69 | ); 70 | 71 | tokenBalance: ([] 72 | block_number: `long$(); 73 | accountId: `accounts$(); 74 | tokenId: `tokens$(); 75 | balance: () // bigInt byte array 76 | ); 77 | 78 | swapType: ([swaptypeId: 0 1 2 3 4 5 6 7 8 9 10i]; 79 | name: ( 80 | `swap_exact_tokens_for_tokens; 81 | `swap_tokens_for_exact_tokens; 82 | `swap_exact_eth_for_tokens; 83 | `swap_tokens_for_exact_ETH; 84 | `swap_exact_tokens_for_ETH; 85 | `swap_eth_for_exact_tokens; 86 | `swap_exact_tokens_for_tokens_supporting_fee_on_transfer_tokens; 87 | `swap_exact_eth_for_tokens_supporting_fee_on_transfer_tokens; 88 | `swap_exact_tokens_for_eth_supporting_fee_on_transfer_tokens; 89 | `swap_exact_tokens_for_tokens_with_flashloan; 90 | `swap_exact_tokens_for_tokens_with_flashloan_multi); 91 | getAmountsIn: 01010100000b); 92 | 93 | // virtual swap pool 94 | swaps:([] 95 | signer: (); // 20 byte array address 96 | tx_from: (); // 20 byte array address 97 | to: (); // 20 byte array address 98 | tx_hash: (); // 32 byte array 99 | signed_tx: (); 100 | nonce: (); 101 | tx_value: (); 102 | chainId: `int$(); 103 | gasPrice: (); 104 | maxPriorityFeePerGas: (); 105 | maxFeePerGas: (); 106 | gasLimit: `long$(); 107 | input: (); 108 | v: (); 109 | r: (); 110 | s: (); 111 | swaptypeId: `swapType$(); 112 | path: (); // tokenId list 113 | amountIn: (); // bigint hex 114 | amountOut: (); // bigint hex 115 | swap_to: (); // 20 byte address output to swap result 116 | deadline: `int$()); // deadline in seconds 117 | 118 | errors: ([errId: 0 1 2 3 4i]; errDesc: `passedDeadline`slippageTooLow`pathNotInLiquidity`bribeTooSmall`cancelledSwap); 119 | 120 | // unbundled swaps e.g. 121 | // passed deadline buffer (10 mins) 122 | // slippage requirements too low 123 | swapsError: ([] 124 | signer: (); // 20 byte array address 125 | tx_from: (); // 20 byte array address 126 | to: (); // 20 byte array address, maybe superfluous with swap_to 127 | tx_hash: (); // 32 byte array 128 | signed_tx: (); 129 | nonce: (); 130 | tx_value: (); 131 | chainId: `int$(); 132 | gasPrice: (); 133 | maxPriorityFeePerGas: (); 134 | maxFeePerGas: (); 135 | gasLimit: `long$(); 136 | input: (); 137 | v: (); 138 | r: (); 139 | s: (); 140 | swaptypeId: `swapType$(); 141 | path: (); // tokenId list 142 | amountIn: (); // bigint hex 143 | amountOut: (); // bigint hex 144 | swap_to: (); // 20 byte address output to swap result 145 | deadline: `int$(); // deadline in seconds 146 | errId: `errors$(); 147 | timestamp: `timestamp$()); 148 | 149 | bundles: ([] 150 | bundleid: `long$(); 151 | signer: (); // 20 byte array address 152 | tx_from: (); // 20 byte array address 153 | to: (); // 20 byte array address 154 | tx_hash: (); // 32 byte array 155 | signed_tx: (); 156 | nonce: (); 157 | tx_value: (); 158 | chainId: `int$(); 159 | gasPrice: (); 160 | maxPriorityFeePerGas: (); 161 | maxFeePerGas: (); 162 | gasLimit: `long$(); 163 | input: (); 164 | v: (); 165 | r: (); 166 | s: (); 167 | swaptypeId: `swapType$(); 168 | path: (); // tokenId list 169 | amountIn: (); // bigint hex 170 | amountOut: (); // bigint hex 171 | swap_to: (); // 20 byte address output to swap result 172 | deadline: `int$()); // deadline in seconds 173 | 174 | bundlesSubmitted: ([] 175 | subbundleid: `long$(); 176 | signer: (); // 20 byte array address 177 | tx_from: (); // 20 byte array address 178 | to: (); // 20 byte array address 179 | tx_hash: (); // 32 byte array 180 | signed_tx: (); 181 | nonce: (); 182 | tx_value: (); 183 | chainId: `int$(); 184 | gasPrice: (); 185 | maxPriorityFeePerGas: (); 186 | maxFeePerGas: (); 187 | gasLimit: `long$(); 188 | input: (); 189 | v: (); 190 | r: (); 191 | s: (); 192 | swaptypeId: `swapType$(); 193 | path: (); // tokenId list 194 | amountIn: (); // bigint hex 195 | amountOut: (); // bigint hex 196 | swap_to: (); // 20 byte address output to swap result 197 | deadline: `int$()); // deadline in seconds 198 | 199 | status: ([statusId: 0 1 2 3 4i]; status: `PENDING`SUBMITTED`ERROR`WON`DISTRIBUTED); 200 | 201 | // bundle status schema 202 | bundleStatus: ([] 203 | bundleid: `long$(); // unique bundle id 204 | statusId: `status$(); // PENDING / SUBMITTED / ERROR / WON / DISTRIBUTED 205 | profit_eth: (); 206 | bribe: (); 207 | profit: (); 208 | profit_token: `tokens$(); 209 | block_number: `long$(); 210 | timestamp: `timestamp$()); 211 | 212 | blockHeader: ([] 213 | block_number: `long$(); 214 | block_hash: (); 215 | block_timestamp: `long$(); 216 | base_fee: () 217 | ); 218 | 219 | triangularArbs: ([ 220 | path0: `int$(); 221 | path1: `int$(); 222 | path2: `int$(); 223 | path3: `int$(); 224 | router0: `int$(); 225 | router1: `int$(); 226 | router2: `int$()] 227 | amountIn: (); 228 | amountOut: (); 229 | profit: (); 230 | profit_eth: ()); 231 | 232 | dexArbs: ([ 233 | path0: `int$(); 234 | path1: `int$(); 235 | path2: `int$(); 236 | router0: `int$(); 237 | router1: `int$()] 238 | amountIn: (); 239 | amountOut: (); 240 | profit: (); 241 | profit_eth: ()); 242 | -------------------------------------------------------------------------------- /src/routine.q: -------------------------------------------------------------------------------- 1 | /each: +-*%&|^<>=$ <= >= <> @ ? in within bin div abs log exp sqrt sin cos tan f' f\: f/: 2 | neg:-:;not:~:;null:^:;string:$:;reciprocal:%:;floor:_:;ceiling:-_-:;signum:{(x>0)-x<0} 3 | mod:{x-y*x div y};xbar:{x*y div x:$[16h=abs[@x];"j"$x;x]};xlog:{log[y]%log x};and:&;or:|;each:{x'y};scan:{x\y};over:{x/y};prior:{x':y} 4 | mmu:$;lsq:!;inv:!:;md5:-15!;ltime:%:;gtime:{t+x-%t:x+x-%x}; /xnull:{$[0>@y;(,y)@~x=y;x=y;y 0N;y]} 5 | 6 | /aggr: last sum prd min max avg wsum wavg f/ /beta:{cov[x;y]%var x} 7 | count:#:;first:*:;svar:{(n*var x)%-1+n:(#x)-+/^x};sdev:{sqrt svar x};scov:{(n*cov[x;y])%-1+n:(#x)-+/^x+y};med:{avg x(@x;'`rank;1_x,,x 0N]}; 11 | rank:{$[0h>@x;'`rank;<@x;'`rank;@x;'`rank;>x]} 12 | asc:{$[99h=@x;(!x)[i]!`s#r i:@x;'`rank;`s#x@r:. x;0h>@x;'`rank;x@>x]} 13 | 14 | msum:{$[99h=@y;(!y)!.z.s[x;. y];y-(-x)_(0i*x#y),y:+\y]};mcount:{msum[x;~^y]};mavg:{msum[x;0.0^y]%mcount[x;y]};mdev:{sqrt mavg[x;y*y]-m*m:mavg[x;y:"f"$y]} 15 | xrank:{$[0h>@y;'`rank;_y*x%#y:<@y;'`rank;y(!#y)-x]};rotate:{$[0h>@y;'`rank;98h<@y;'`type;#y;,/|(0;mod[x;#y])_y;y]};ema:{(*y)(1f-x)\x*y} 16 | 17 | /other: ~,#_ !. getenv exit 18 | distinct:?:;group:=:;where:&:;flip:+:;type:@:;key:!:;til:{$[0>@x;!x;'`type]};value:get:.:;attr:-2!;cut:{$[0h>@x;x*!-_-(#y)%x;x]_y} 19 | set:{$[@x;.[x;();:;y];-19!((,y),x)]};upsert:.[;();,;] / :: ,: files? 20 | raze:,/;union:?,;inter:{x@&x in y};except:{x@&~x in y};cross:{n:#m:&(#x)##y;$[99h=@x;((!x)[m],'n#!y)!(. x)[m],'n#. y;((),x)[m],'n#y]} /extant:{x@&~^x} 21 | sv:{x/:y};vs:{x\:y};sublist:{$[99h=@y;sublist[x;!y]!sublist[x;. y];~0h>@x;$[.Q.qp y;.Q.ind[y];y]i+!"j"$0|x[1]&(#y)-i:*x;abs[x]<#y;x#y;y]} 22 | 23 | /file&comm 24 | read0:0::;read1:1::;hclose:>:;hdel:~:;hsym:"s"$-1!';hcount:-7!;peach:{x':y};system:."\\", 25 | 26 | /string: like ss 27 | ltrim:{$[~t&77h>t:@x;.z.s'x;^*x;((^x)?0b)_x;x]};rtrim:{$[~t&77h>t:@x;.z.s'x;^last x;(-(|^x)?0b)_x;x]};trim:{ltrim rtrim x} 28 | lower:{$[$[(~@x)&10h~@*x;&/10h=@:'x;0b];_x;~t&77h>t:abs@@x;.z.s'x;19t:abs@@x;.z.s'x;19@z;:[;z];z]]} 31 | 32 | /select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation 33 | /{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) 34 | view:{(2+*x ss"::")_x:*|*|.`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} 35 | cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} 36 | xcols:{(x,f@&~(f:cols y)in x)#y};keys:{$[98h=@x:.Q.v x;0#`;!+!x]};xkey:{(#x)!.[0!y;();xcols x]}; 37 | xcol:{.Q.ft[{+$[99h=@x;@[!y;(!y)?!x;:;. x];x,(#x)_!y]!. y:+y}x]y};xasc:{$[$[#x;~`s=-2!(0!.Q.v y)x;0];.Q.ft[@[;*x;`s#]].Q.ord[<:;x]y;y]};xdesc:{$[#x;.Q.ord[>:;x]y;y]} 38 | fkeys:{(&~^x)#x:.Q.fk'.Q.V x};meta:{([!c].Q.ty't;f:.Q.fk't;a:-2!'t:. c:.Q.V x)} 39 | 40 | / R uj R(union join) R lj K(left(equi/asof)join) trade asof`sym`time!(`IBM;09:31:00.0) 41 | lj:{$[$[99h=@y;(98h=@!y)&98h=@. y;()~y];.Q.ft[,\:[;y];x];'"type"]} /;la:{$[&/j:z>-1;x,'y z;+.[+ff[x]y;(!+y;j);:;.+y z j:&j]]}{la[x;. y](!y)?(!+!y)#x}[;y]]x} /lj:,\:;aj:{lj[y]`s#xkey[x]z};aj0:{lj[y]`s#(x#z)!z}; /;bn:{@[i;&0>i:x bin y;:;#x]} 42 | ljf:{$[`s=-2!y;ajf[!+!y;x;0!y];.Q.ft[{$[&/j:(#y:. y)>i?:(!+i:!y)#x;.Q.fl[x]y i;+.[+x;(f;j);:;.+.Q.fl[((f:!+y)#x:.Q.ff[x]y)j]y i j:&j]]}[;y]]x]} 43 | .Q.ajf0:{[f;g;x;y;z]x,:();z:0!z;d:$[g;x_z;z];g:(:;^)f;f:(,;^)f;$[&/j:-1i:(!y)?(!+!y)#x}[;y]]x} 46 | ijf:{.Q.ft[{.Q.fl[x j]y i j:&(#y:. y)>i?:(!+i:!y)#x}[;y]]x} 47 | pj:{.Q.ft[{x+0i^y(!+!y)#x}[;y]]x};asof:{f:!$[99h=@y;y;+y];(f_x)(f#x)bin y} 48 | uj:{$[()~x;y;()~y;x;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];lj[(?(!x),!y)#x]y]} 49 | ujf:{$[()~x;y;98h=@x;x,(!+x:.Q.ff[x;y])#.Q.ff[y;x];ljf[(?(!x),!y)#x]y]} 50 | 51 | /wj[-1000 2000+\:trade`time;`sym`time;trade;(quote;(max;`ask);(min;`bid))] (given `sym`time xasc quote) 52 | ww:{[a;w;f;y;z]f,:();e:1_z;z:*z;y,'n#+(:/'f)!+{[e;d;a;b]e .'d@\:\:a+!b-a}[*:'e;z f:1_'e]/'$[n:#*w;+$[#g;(g#z)?g#y;0]|/:a+$[#g:-1_f;(f#z)bin@[f#y;*|f;:;]@;z[*f]bin]'w;,0 0]} 53 | wj:{[w;f;y;z]ww[0 1;w;f;y;z]};wj1:{[w;f;y;z]ww[1;w-1 0;f;y;z]} 54 | 55 | fby:{$[(#x 1)=#y;@[(#y)#x[0]0#x 1;g;:;x[0]'x[1]g:.=y];'`length]};xgroup:{x,:();a:x#y:0!y;$[#x_:y;+:'x@=a;a!+f!(#f:!+x)#()]};ungroup:{$[#x:0!x;,/+:'x;x]} 56 | ej:{x,:();y[&#:'i],'(x_z)(!0),/i:(=x#z:0!z)x#y:0!y} /{ungroup lj[z]xgroup[x]y} 57 | 58 | /`[:../]t[.{csv|txt}] 59 | save:{$[1=#p:`\:*|`\:x:-1!x;set[x;. *p]; x 0:.h.tx[p 1]@.*p]}' 60 | load:{$[1=#p:`\:*|`\:x:-1!x;set[*p;. x];set[*p].h.xt[p 1]@0:x]}' 61 | rsave:{x:-1!x;.[`/:x,`;();:;.*|`\:x]}' 62 | rload:{x:-1!x;.[*|`\:x;();:;. x]}' 63 | dsave:{.[*x;1_x,y,`;:;@[;*!+a;`p#].Q.en[*x]a:. y];y}/: 64 | 65 | show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" 66 | 67 | parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! 68 | \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") 69 | k:4.0;K:2020.05.04;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc 70 | res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp 71 | addmonths:{("d"$m+y)+x-"d"$m:"m"$x} 72 | Xf:{y 1:0xfe20,("x"$77+@x$()),13#0x00;(`$($y),"#")1:0x};Cf:Xf`char 73 | f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} 74 | fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} -------------------------------------------------------------------------------- /src/utils/assert.q: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Manifold Finance, Inc. 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | // 6 | // assert.q 7 | // Assertions checks that verifies invariant variables (taken from qunit) 8 | 9 | // Assert that the relation between expected and actual value holds 10 | // @param actual An object representing the actual result value 11 | // @param relation Function to check actual with expected 12 | // @param expected An object representing the expected value 13 | // @param msg Description of this test or related message 14 | // @return actual object 15 | assert: {[actual; relation; expected; msg] 16 | relationPassed: .[relation; (actual; expected); 0b]; 17 | if[not relationPassed; 'msg]; 18 | actual 19 | }; 20 | 21 | // assert that actual is true 22 | // @param msg Description of this test or related message 23 | // @return actual object 24 | assertTrue: {[actual; msg] 25 | assert[actual; =; 1b; msg] 26 | }; 27 | 28 | // assert that actual is false 29 | // @param msg Description of this test or related message 30 | // @return actual object 31 | assertFalse: {[actual; msg] 32 | assert[actual; =; 0b; msg] 33 | }; 34 | 35 | // assert that actual is empty i.e. count is zero. 36 | // @param msg Description of this test or related message 37 | // @return actual object 38 | assertEmpty:{[actual; msg] 39 | assert[count actual; =; 0; msg] 40 | }; 41 | 42 | // assert that actual is NOT empty i.e. count is greater than zero. 43 | // @param msg Description of this test or related message 44 | // @return actual object 45 | assertNotEmpty: {[actual; msg] 46 | assert[count actual; >; 0; msg] 47 | }; 48 | -------------------------------------------------------------------------------- /src/utils/globals.q: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Manifold Finance, Inc. 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | // 6 | // --------------------------------------------------------------------------------------------------------------------- 7 | // Globals 8 | // --------------------------------------------------------------------------------------------------------------------- 9 | 10 | // @kind data 11 | // @desc Proto schema fields for platform transaction 12 | PltTxFields: `sender`transaction; 13 | 14 | // @kind data 15 | // @desc Proto schema fields for swap type 16 | SwpTxFields: ( 17 | `erc20_approval; 18 | `swap_exact_tokens_for_tokens; 19 | `swap_tokens_for_exact_tokens; 20 | `swap_exact_eth_for_tokens; 21 | `swap_tokens_for_exact_ETH; 22 | `swap_exact_tokens_for_ETH; 23 | `swap_eth_for_exact_tokens; 24 | `swap_exact_tokens_for_tokens_supporting_fee_on_transfer_tokens; 25 | `swap_exact_eth_for_tokens_supporting_fee_on_transfer_tokens; 26 | `swap_exact_tokens_for_eth_supporting_fee_on_transfer_tokens; 27 | `swap_exact_tokens_for_tokens_with_flashloan; 28 | `swap_exact_tokens_for_tokens_with_flashloan_multi); 29 | 30 | // @kind data 31 | // @desc Index ref for Multi flashloan proto schema field 32 | MultiFlashLoan: 10i; 33 | 34 | // @kind data 35 | // @desc Index ref for Single flashloan proto schema field 36 | SingleFlashLoan: 9i; 37 | 38 | // @kind data 39 | // @desc Index ref for swap exact eth for tokens proto schema field 40 | EthSwap: 2i; 41 | 42 | // @kind data 43 | // @desc Index refs for swaps that need approval txs 44 | ApproveSwapList: 0 1 3 4 6 8i; 45 | 46 | // @kind data 47 | // @desc Proto schema fields for platform transaction combined with swap type 48 | PltTxFields: PltTxFields,SwpTxFields; 49 | 50 | // @kind data 51 | // @desc platform account address in use for signing 52 | PlAcc: 0x0000; 53 | 54 | // @kind data 55 | // @desc platform account nonce 56 | PlAccNonce: 0x0000; 57 | 58 | // @kind data 59 | // @desc ether balance of active platform account 60 | PlAccEth: 0x0000; 61 | 62 | // @kind data 63 | // @desc Next base fee for a block 64 | NextBaseFee: "0x0000"; 65 | 66 | 67 | // @kind data 68 | // @desc Last base fee for a block 69 | LastBaseFee: 0x0000; 70 | 71 | // @kind data 72 | // @desc Updated value of latest block number 73 | LatestBlock: 0j; 74 | 75 | // @kind data 76 | // @desc Updated value of liquidity count for dex paths 77 | DexLiquidityCount: 0j; 78 | 79 | // @kind data 80 | // @desc Updated value of liquidity count for tri paths 81 | TriLiquidityCount: 0j; 82 | 83 | // @kind data 84 | // @desc Value of WETH token Id 85 | WethId: (); 86 | 87 | // @kind data 88 | // @desc Aave related addresses 89 | AaveAssetAddresses: ( 90 | 0xdac17f958d2ee523a2206206994597c13d831ec7; 91 | 0x2260fac5e5542a773aa44fbcfedf7c193bc2c599; 92 | 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2; 93 | 0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e; 94 | 0xe41d2489571d322189246dafa5ebde1f4699f498; 95 | 0x1f9840a85d5af5bf1d1762f925bdaddc4201f984; 96 | 0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9; 97 | 0x0d8775f648430679a709e98d2b0cb6250d2887ef; 98 | 0x4fabb145d64652a948d72533023f6e7a623c7c53; 99 | 0x6b175474e89094c44da98b954eedeac495271d0f; 100 | 0xf629cbd94d3791c9250152bd8dfbdf380e2a3b9c; 101 | 0xdd974d5c2e2928dea5f71b9825b8b646686bd200; 102 | 0x514910771af9ca656af840dff83e8264ecf986ca; 103 | 0x0f5d2fb29fb7d3cfee444a200298f468908cc942; 104 | 0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2; 105 | 0x408e41876cccdc0f92210600ef50372656052a38; 106 | 0xc011a73ee8576fb46f5e1c5751ca3b9fe0af2a6f; 107 | 0x57ab1ec28d129707052df4df418d58a2d46d5f51; 108 | 0x0000000000085d4780b73119b644ae5ecd22b376; 109 | 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48; 110 | 0xd533a949740bb3306d119cc777fa900ba034cd52; 111 | 0x056fd409e1d7a124bd7017459dfea2f387b6d5cd; 112 | 0xba100000625a3754423978a60c9317c58a424e3d; 113 | 0x8798249c2e607446efb7ad49ec89dd1865ff4272; 114 | 0xd5147bc8e386d91cc5dbe72099dac6c9b99276f5); 115 | 116 | // @kind data 117 | // @desc List to capture and cache reserves for which we have liquidity data 118 | ReserveList: (); 119 | 120 | // @kind data 121 | // @desc Table to save possibly triangular arb paths 122 | TrigPaths: 0#enlist `path0`path1`path2`path3!1 2 3 4i; 123 | 124 | // @kind data 125 | // @desc Table to save possibly dex arb paths 126 | DexPaths: 0#enlist `path0`path1`path2!1 2 3i; 127 | 128 | // @kind data 129 | // @desc Value of HDB handle 130 | HandleHDB: hopen `:unix://5003; 131 | 132 | // @kind data 133 | // @desc Value of producer handle 134 | HandleProducer: hopen `:unix://5004; 135 | 136 | // @kind data 137 | // @desc Table to simulate liquidity impacts through bundle txs 138 | SimulatedReserves: ([] 139 | tracking_number: `int$(); 140 | routerId: `int$(); 141 | base_token: `tokens$(); 142 | quote_token: `tokens$(); 143 | base_reserve: (); // bigInt byte array 144 | quote_reserve: () 145 | ); 146 | -------------------------------------------------------------------------------- /terraform/service.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_service" "service" { 2 | metadata { 3 | name = var.name 4 | namespace = var.k8s_namespace 5 | labels = merge(var.k8s_labels, { 6 | name = var.name 7 | }) 8 | } 9 | 10 | spec { 11 | selector = { 12 | name = var.name 13 | } 14 | 15 | port { 16 | port = 80 17 | target_port = "http" 18 | } 19 | 20 | type = "ClusterIP" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /terraform/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | kubernetes = { 4 | source = "hashicorp/kubernetes" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" {} 2 | 3 | variable "image_repository" {} 4 | variable "image_tag" {} 5 | variable "image_pull_secrets" { 6 | type = list(string) 7 | description = "List of secrets containing image pull credentials" 8 | } 9 | 10 | variable "k8s_namespace" {} 11 | 12 | variable "k8s_labels" { 13 | type = map(string) 14 | } 15 | 16 | variable "k8s_node_selector" { 17 | type = map(string) 18 | default = {} 19 | } 20 | 21 | variable "k8s_storage_class_name" { 22 | type = string 23 | } 24 | 25 | variable "kafka_bootstrap_servers" { 26 | type = string 27 | } 28 | 29 | variable "kafka_consumer_group_id" { 30 | type = string 31 | } 32 | 33 | variable "resource_memory" { 34 | default = "1024Mi" 35 | } 36 | 37 | variable "q_license_path" { 38 | description = "Path to the Q license" 39 | } 40 | 41 | variable "q_license_hostname" { 42 | description = "Hostname that the Q license matches against" 43 | } 44 | 45 | variable "q_log_level" { 46 | type = string 47 | description = "Level to log [silent,debug,info]" 48 | default = "info" 49 | } 50 | 51 | variable "bundler_use_flash_loans" { 52 | type = bool 53 | description = "Enable flash loans for triangular arbs" 54 | default = false 55 | } 56 | 57 | variable "dual_block_submission" { 58 | type = bool 59 | description = "Enable submission of bundle for target block and target block + 1" 60 | default = true 61 | } 62 | 63 | variable "bundler_disable_sandwiches" { 64 | type = bool 65 | description = "Disable sandwich bundles" 66 | default = true 67 | } 68 | 69 | variable "enabled" { 70 | type = bool 71 | description = "Whether or not to run an instance of the engine" 72 | default = false 73 | } 74 | --------------------------------------------------------------------------------