├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app ├── __init__.py ├── main.py ├── requirements.txt ├── run.sh └── static │ ├── index.html │ └── lemma-term.js ├── build.sh ├── images ├── build.gif ├── demo.gif ├── demo2.gif ├── demo3.gif ├── e1.png ├── e2.png ├── e3.png ├── e4.png ├── e5.gif └── lemma.png ├── lemma ├── __init__.py ├── __main__.py ├── input_adapter.py ├── lambda_worker.py ├── logo.py ├── output_adapter.py ├── pipeline.py └── settings.py ├── setup.py ├── templates ├── samconfig.toml ├── template_arm64.yaml └── template_x86.yaml └── tools ├── bin └── download ├── config ├── .gau.toml ├── dirsearch.ini └── test.txt ├── demo ├── ffuf ├── gau ├── install_tools.sh ├── nuclei ├── runner └── smuggler /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .aws-sam 3 | app/tools/* 4 | __pycache__ 5 | lemmacli.egg-info 6 | /samconfig.toml 7 | /template.yaml 8 | /*.log 9 | app/tool_requirements.txt 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Python image from the Docker Hub 2 | FROM --platform=linux/amd64 python:3.12-slim 3 | 4 | # Set environment variables to avoid interactive prompts during package installation 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | ENV GO111MODULE=on 7 | 8 | # Install dependencies 9 | RUN apt-get update && \ 10 | apt-get install -y \ 11 | curl \ 12 | unzip \ 13 | wget \ 14 | git \ 15 | && rm -rf /var/lib/apt/lists/* 16 | 17 | # Install AWS CLI v2 18 | RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ 19 | unzip awscliv2.zip && \ 20 | ./aws/install && \ 21 | rm -rf awscliv2.zip aws 22 | 23 | # Install AWS SAM CLI 24 | RUN curl -sSL https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip -o sam-cli.zip && \ 25 | unzip sam-cli.zip -d sam-installation && \ 26 | ./sam-installation/install && \ 27 | rm -rf sam-cli.zip sam-installation 28 | 29 | # Install the latest version of Go 30 | RUN wget "https://go.dev/dl/go1.22.5.linux-amd64.tar.gz" -O go.tar.gz && \ 31 | tar -C /usr/local -xzf go.tar.gz && \ 32 | rm go.tar.gz 33 | 34 | # Set Go environment variables 35 | ENV PATH="/usr/local/go/bin:${PATH}" 36 | 37 | # Verify installations 38 | RUN aws --version && \ 39 | sam --version && \ 40 | go version && \ 41 | python --version 42 | 43 | WORKDIR /lambda 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # THIS IS A COPY OF THE ORIGINAL REPO WHICH WAS TAKEN DOWN, ALL CREDITS TO DEFPARAM https://x.com/defparam 2 | 3 | --- 4 | 5 |

6 | lemma 7 |
8 |

9 | 10 |

11 | 12 | 13 | 14 |

15 | 16 |

17 | Demo • 18 | Features • 19 | Installation • 20 | Web Client • 21 | Terminal Client • 22 | FAQ • 23 | Examples 24 |

25 | 26 | ### Disclaimer 27 | The author of this project is not responsible for any damage or data loss incurred as a result of using this software. Use this software at your own risk. While efforts have been made to ensure the accuracy and reliability of the software, it is provided "as is" without warranty of any kind. By using this software, you agree to assume all risks associated with its use. Opinions are that of the author and not that of AWS. Review the [AWS pentesting policy](https://aws.amazon.com/security/penetration-testing/) prior to executing any security tools on AWS Lambda. 28 | 29 | 30 | # Lemma 31 | Lemma is a Python-based AWS Lambda package and client designed to execute packaged command-line tools in a scalable, remote environment on AWS Lambda. Lemma takes advantage of the new [Response Streaming](https://aws.amazon.com/blogs/compute/introducing-aws-lambda-response-streaming/) feature on AWS Lambda to stream real-time stdout back to the user as the tool is running. The Lemma project comprises three main components: 32 | 33 | 1) Lemma Lambda Function Package: This package bundles a collection of command-line Linux tools provided by the user, making them accessible via AWS Lambda. It allows users to execute these tools remotely and scale their executions across multiple lambda instances. 34 | 35 | 2) Web-CLI: This component provides a web-based terminal interface built with [xterm.js](https://xtermjs.org/), [AWS Lambda Web Adapter](https://github.com/awslabs/aws-lambda-web-adapter) and [FastAPI](https://fastapi.tiangolo.com/), accessible via the Lambda URL. This web UI allows users to execute their command-line tools packaged in the Lambda entirely within their web browser. 36 | 37 | 3) Terminal-CLI: A python-based command-line interface tool in charge invoking the Lemma Lambda function. This tool facilitates the remote execution of the Lambda-hosted tools from a local environment. It pipes stdin and stdout between local and remote tools, providing the ability to execute and scale cli-based workflows onto lambda and back using pipes. 38 | 39 | While the intented use case for Lemma is to run verbose security security tooling on AWS lambda, Lemma can be used for any type of command-line tool you wish to run remotely. 40 | 41 | # Demo 42 | 43 | Web-CLI: 44 |

45 | 46 |
47 |

48 | Terminal-CLI: 49 |

50 | 51 |
52 |

53 | 54 | # Features 55 | - Supports both a Web-CLI and a Terminal-CLI 56 | - Quick and easy build script 57 | - Support for adding your own custom tools 58 | - Support for x86_64 and ARM64 lambda types 59 | - Support for choosing memory, region and timeout 60 | - Flexible terminal piping support 61 | 62 | # Installation 63 | ## Requirements for Lemma Lambda 64 | 1) An AWS account 65 | 2) AWS access credentials with permissions to execute cloudformation templates 66 | 3) Docker, python3 with pip 67 | 68 | ## Lambda Build and Deploy Steps 69 | Steps to build and deploy on a fresh Ubuntu 22 instance 70 | 71 | 1) `sudo apt update` 72 | 2) `sudo apt install docker.io python3 python3-pip` 73 | 3) `git clone https://github.com/defparam/lemma` 74 | 4) `cd lemma` 75 | 5) `export AWS_ACCESS_KEY_ID=` 76 | 6) `export AWS_SECRET_ACCESS_KEY=` 77 | 7) `./build.sh` 78 | 8) Fill out all the questions 79 | 9) Copy the lambda URL with the key 80 | 81 | Web-CLI: 82 | 1) Open chrome and simply browse to your lambda URL w/key 83 | 84 | Terminal-CLI: 85 | 1) While in the lemma directory: `pip3 install .` (The Terminal-CLI is also available on pypi: `pip install lemmacli`) 86 | 2) Invoke: `lemma` 87 | 3) When asked about the lambda URL, paste it into the prompt. This URL will be saved at `~/.lemma/lemma.ini` 88 | 89 | Build Walkthrough: 90 |

91 | 92 |
93 |

94 | 95 | ## Lemma Web Client 96 | 97 | Lemma's web client is packaged inside the Lemma function itself for easy access. It simply is just 1 html file and 1 javascript file (Also importing xterm.js from CDN). To access it simply just copy and paste your lemma lambda url/key into your chrome web browser and hit enter. For usage details just type the `help` command. 98 | 99 | ## Lemma Terminal Client 100 | 101 | ### Usage 102 | ``` 103 | positional arguments: 104 | remote_command lemma -- remote_command 105 | 106 | options: 107 | -h, --help show this help message and exit 108 | -w WORKERS, --workers WORKERS 109 | Number of concurrent Lambda service workers 110 | -l, --lambda-url Prompt user to enter a new lambda url 111 | -i INVOCATIONS, --invocations INVOCATIONS 112 | The number of invocations of the remote command 113 | -p, --per-stdin Invoke the remote command for each line of stdin (-i is ignored) 114 | -d DIV_STDIN, --div-stdin DIV_STDIN 115 | Divide stdin into DIV_STDIN parts at a newline boundary and invoke on each (-i is ignored) 116 | -o, --omit-stdin Omit stdin to the remote command stdin 117 | -e, --no-stderr prevent stderr from being streamed into response 118 | -b, --line-buffered Stream only line chunks to stdout 119 | -v, --verbose Enable verbose remote output 120 | -t, --tools List available tools 121 | ``` 122 | 123 | | Remote Command Macro | Description 124 | |-------------------------|---------------------------------------------------- 125 | | `%INDEX%` | You can place this macro on a remote command to insert the current invocation count into the command (starts at 0) 126 | | `%STDIN%` | You can place this macro on a remote command to insert any data that may exist on lemma client's stdin. (Warning: new line characters aren't permitted except in -p mode) 127 | 128 | ## FAQ 129 | 130 | Q: Why did you make this? Aren't there other frameworks? 131 | 132 | A: I recently read about a new lambda feature [Response Streaming](https://aws.amazon.com/blogs/compute/introducing-aws-lambda-response-streaming/) that was only a year old and thought about how wonderful it would be in linux to pipe lambda streams together with security tooling because prior to that all responses back from lambda were buffered. Futhermore, I saw lambda's web adapter and though it would be a super neat feature to have lambda present a web page of a terminal to invoke these streaming commands. 133 | 134 | Q: Does this work on MacOS or Windows? 135 | 136 | A: In theory yes, but at this point i've only tested linux. 137 | 138 | Q: Do you support other cloud providers? 139 | 140 | A: No, mainly because I'm not sure if other cloud providers even support response streaming with their FaaS product and secondly I don't have the time to research it and make this tool generic. 141 | 142 | Q: How do I package my own tools? 143 | 144 | A: If you have a normal bash script, simply move it into the `./tools` directory, make it executable and re-build your lambda, its that easy. If your tool installation requires more advanced setup then place those steps into `./tools/install_tools.sh` and re-build your lambda. NOTE: inside a lambda the only writable directory is `/tmp`, so if your tool needs a mutable settings area create a wrapper script to manage it at `/tmp` 145 | 146 | Q: Why do you support both arm64 and x86_64? 147 | 148 | A: If you end up running A LOT of executions to the point where you care about your AWS bill you may want to use arm64 architecture since it is generally billed cheaper than x86_64. Also billing rates are slightly different depending on the region and memory requirements as well. 149 | 150 | Q: Where do I get my `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` ? 151 | 152 | A: You can generate it when you log into your AWS account in IAM. I won't go into how to do this since there are plenty of resources you can google. 153 | 154 | Q: How come I can only run 10 parallel lambdas at a time? 155 | 156 | A: This is a quota set for all new AWS accounts. To increase it to 100-1000 you have to place a quota increase request into AWS through your account. 157 | 158 | Q: What's the deal with the `key` parameter on the Lambda URLs? 159 | 160 | A: So this lambda application basically provides an RCE API. These lambda URLs are publically accessible. There is a IAM AUTH mode where you can sign your requests with SigV4 but I haven't implemented it yet. As a band-aid solution I added a poor-man's randomly generated API key. When you access it with the key it sets it as a cookie and redirects you back to the root page. If the key is not correct the lambda will return 404. In general I recommend only keeping your lambda url public at times of use then disable/delete it. 161 | 162 | Q: Does lambda support streaming data into stdin of a function? 163 | 164 | A: No, not at this time. When the client invokes a function all stdin is known and transmitted on invoke. Only stdout of a lambda function supports streaming. 165 | 166 | Q: I have a tool/workload that requires more than 15 minutes, how can i increase the lambda timeout? 167 | 168 | A: You can't. Lambda is strict on the timeout being max 15 minutes. The way to solve this problem is to break your workflow down to partial executions that execute under 15 minutes. 169 | 170 | Q: On the lemma python client what's the deal with these `-i`, `-p` and `-d` modes? 171 | 172 | A: These modes are all mutually exclusive and if none of them are specified it is assumed the mode is `-i 1`. The `-i` mode flag allows you to explicitly specify the number of lambda executions of the remote command. The `-p` mode flag tells the client that for all stdin data presented to the client, perform a lambda execution for each "line" of stdin and place that line as stdin into the lambda function unless the user wants it withheld via the `-o` flag. The `-d` mode flag tells the client to consume all stdin at once, split that stdin in parts of `DIV_STDIN` (at a line boundary) and invoke the function for each part. The `-d` mode will not invoke lambda functions until all stdin data is recieved. The `-p` mode will start invoking lambda functions the moment a line is recieve on stdin. 173 | 174 | Q: What are remote command macros? 175 | 176 | A: Remote command macros are a way for the client to insert data into your remote command. There are currently only 2 macros supported: `%INDEX%` and `%STDIN%`. `%INDEX%` allows you do put the invocation index into an argument of the remote command. This is useful for cases where you need to provide some differentiator between function executions. `%STDIN%` allows you to place what is it stdin onto the remote command. This is useful especially in `-p` mode where you may want to take a line from stdin and apply it as an argument of the remote command. 177 | 178 | Q: What's the reason for `-o`, `-b` and `-e` ? 179 | 180 | A: `-o` is used when you want to use stdin data but you do not want to send any stdin data to the remote function stdin. This is useful for cases when you want to use `-p` mode and the `%STDIN%` macro so that each line of stdin is only used as a remote command argument. `-b` buffered line mode forces every thread worker to write responses to stdout in line chunks. This is used to prevent data from 2 or more different thread workers from writing to the same line. This is default off because non-buffered byte streaming is important for animated cli tools. Lastly, `-e` tells the remote function to only respond back with stdout, by default both stderr and stdout are sent back in the response stream. 181 | 182 | Q: My tool runs super slow, what gives? 183 | 184 | A: It's likely that the memory configuration you set for your lambda is too small for the tool you are trying to run. I ran into this issue with `dirsearch` which led me to ultimately remove it from the tools since it wouldn't reliably run without >1GB of memory and ffuf runs beautifully with 256MB of memory. I'm a big proponent of running go-based tools on lambda since it appears to consume much less memory than python. 185 | 186 | Q: You are missing tool X, Are you accepting PRs? 187 | 188 | A: Yes I know, I haven't spent time curating a good collection of default tools I've been mostly focused on lemma itself. I'm happy to review PRs for tool additions into `./tools/install_tools.sh` if they make sense. 189 | 190 | Q: Is this expensive? 191 | 192 | A: It really depends on your usage patterns, but if you are casually invoking tools interactively it is amazingly cheap or most likely free. However, use a pricing calculator to get an idea before doing any heavy workloads. There is no lambda executing while you are idling in the Web-CLI. You invoke a lambda each for loading the root html, lemma-term.js and tools.json but after that the only time lambda is invoked is when you run a tool or refresh/open the page again. 193 | 194 | Q: I'm scared, what is `./build.sh` doing with my creds? 195 | 196 | A: `./build.sh` is running AWS SAM (Serverless Application Model) on a single template, this creates and deploys a cloudformation template that essentially just synthesizes 1 lambda function and a couple of IAM permissions. It is very light weight and you can review the templates used in the `./templates` directory. Also you can review `./build.log` and `./deploy.log` to get an idea of what is going on. In general feel free to review all the code to get a sense of the project. The largest risk is with `./tools/install_tools.sh` blindly pulling in 3rd party software, so feel free to modify that if you are concerned with supply chain risks. 197 | 198 | Q: My deploy is failing and the deploy.log says `'MemorySize' value failed to satisfy constraint: Member must have value less than or equal to 3008`. How come? 199 | 200 | A: AWS Lambda can technically support memory sizes from 128 to 10240 but the quota limits you to 3008 max. Place a quota increase request into AWS if you need the expanded memory support. 201 | 202 | Q: I have a deployed lemma lambda and I'm trying to delete it but `./build.sh` is failing, what do I do? 203 | 204 | A: If the delete fails you can always log onto your AWS account in browser, go to the region you deployed into and head over to: `CloudFormation -> Stacks`, click into the `lemma` stack and click `delete` 205 | 206 | ## Examples 207 | 208 | **Example 1:** A 1 worker example showing 10 invocations using STDIN and INDEX macros: 209 | 210 | `echo cat | lemma -i 10 -w 1 -b -- demo Invoke - %STDIN% %INDEX%` 211 | 212 | 213 | **Example 2:** A 10 worker example showing 10 invocations using STDIN and INDEX macros: 214 | 215 | `echo cat | lemma -i 10 -w 10 -b -- demo Invoke - %STDIN% %INDEX%` 216 | 217 | 218 | **Example 3:** A 5 worker example with an invocation per stdin line where the line is used as an argument 219 | 220 | `printf "aaa\nbbb\nccc\nddd\neee" | lemma -p -b -w 5 -- demo test:%STDIN%` 221 | 222 | 223 | **Example 4:** 2 workers running Subfinder pulling domains for hackerone.com and bugcrowd.com whose output goes directly to 10 workers actively checking if https:443 is open on all domains found from subfinder 224 | 225 | `printf "hackerone.com\nbugcrowd.com" | lemma -p -w 2 -b -e -- subfinder | lemma -d 10 -w 10 -b -e -- httpx -p https:443` 226 | 227 | 228 | **Example 5:** Using TamperMonkey scripts to add context menus that forward commands to lemma Web-CLI 229 | 230 | Here are some example TamperMonkey scripts: 231 | 232 | ```js 233 | // ==UserScript== 234 | // @name lemma: ffuf current host 235 | // @namespace http://tampermonkey.net/ 236 | // @description Context menu to execute UserScript 237 | // @version 0.1 238 | // @author author 239 | // @include * 240 | // @grant GM_openInTab 241 | // @run-at context-menu 242 | // ==/UserScript== 243 | 244 | 245 | (function() { 246 | 'use strict'; 247 | 248 | let lemmaurl = "https://votk3e7gxhxxxylrv2hd4ng7fa0hmarz.lambda-url.us-east-1.on.aws/?cmd="; 249 | let ffufcmd = "run ffuf -w ./tools/wordlists/common.txt -u " + window.location.origin + "/FUZZ -mc 200"; 250 | lemmaurl = lemmaurl + encodeURIComponent(ffufcmd); 251 | GM_openInTab(lemmaurl, { active: true }); 252 | 253 | })(); 254 | ``` 255 | 256 | ```js 257 | // ==UserScript== 258 | // @name lemma: smuggler current host 259 | // @namespace http://tampermonkey.net/ 260 | // @description Context menu to execute UserScript 261 | // @version 0.1 262 | // @author author 263 | // @include * 264 | // @grant GM_openInTab 265 | // @run-at context-menu 266 | // ==/UserScript== 267 | 268 | 269 | (function() { 270 | 'use strict'; 271 | 272 | let lemmaurl = "https://votk3e7gxhxxxylrv2hd4ng7fa0hmarz.lambda-url.us-east-1.on.aws/?cmd="; 273 | let ffufcmd = "run smuggler -u " + window.location.origin; 274 | lemmaurl = lemmaurl + encodeURIComponent(ffufcmd); 275 | GM_openInTab(lemmaurl, { active: true }); 276 | 277 | })(); 278 | ``` 279 | 280 | TamperMonkey/WebCLI Demo: 281 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/app/__init__.py -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, Query, Request 2 | from fastapi.responses import StreamingResponse, FileResponse, Response 3 | from fastapi.middleware.cors import CORSMiddleware 4 | from fastapi.staticfiles import StaticFiles 5 | import asyncio 6 | import os 7 | import json 8 | import urllib.parse 9 | import traceback 10 | import requests 11 | import time 12 | import shlex 13 | 14 | 15 | app = FastAPI() 16 | 17 | # Allow all origins, all methods, and all headers 18 | app.add_middleware( 19 | CORSMiddleware, 20 | allow_origins=["*"], # You can specify a list of allowed origins here 21 | allow_credentials=True, 22 | allow_methods=["*"], # Allow all HTTP methods 23 | allow_headers=["*"], # Allow all headers 24 | expose_headers=["x-lemma-timeout"], # Expose the custom header 25 | ) 26 | 27 | # get a list of every file (not directory) with +x set in the tools directory 28 | tools = [f for f in os.listdir("tools") if os.path.isfile(os.path.join("tools", f)) and os.access(os.path.join("tools", f), os.X_OK)] 29 | 30 | # write it to a json file at the root of the static directory 31 | with open("/tmp/tools.json", "w") as f: 32 | json.dump(tools, f) 33 | 34 | def access_allowed(request): 35 | key = os.getenv("LEMMA_API_KEY") 36 | ckey = request.cookies.get("LEMMA_API_KEY") 37 | if ckey and ckey == key: 38 | return None 39 | # check if key is in the header field "x-lemma-api-key" 40 | header_key = request.headers.get("x-lemma-api-key") 41 | if header_key and header_key == key: 42 | return None 43 | 44 | if key and request.query_params.get("key") == key: 45 | # return redirect to '/' with a cookie set 46 | r = Response(status_code=302, headers={"Location": "/"}) 47 | # set the cookie as secure and httponly 48 | r.set_cookie("LEMMA_API_KEY", key, secure=True, httponly=True) 49 | return r 50 | return Response(status_code=404) 51 | 52 | @app.exception_handler(404) 53 | async def custom_404_handler(request, exc): 54 | # Customize the response here 55 | return Response(status_code=404) 56 | 57 | @app.exception_handler(405) 58 | async def custom_405_handler(request, exc): 59 | # Customize the response here 60 | return Response(status_code=404) 61 | 62 | # Mount the tools.json file to /static/tools.json 63 | @app.get("/tools.json") 64 | async def get_tools(request: Request): 65 | response = access_allowed(request) 66 | if response is not None: 67 | return response 68 | return FileResponse("/tmp/tools.json", media_type="application/json", headers={"Cache-Control": "no-store"}) 69 | 70 | @app.get("/static/lemma-term.js") 71 | async def read_js(request: Request): 72 | response = access_allowed(request) 73 | if response is not None: 74 | return response 75 | return FileResponse("static/lemma-term.js", media_type="application/javascript", headers={"Cache-Control": "no-store"}) 76 | 77 | @app.get("/") 78 | async def read_root(request: Request): 79 | response = access_allowed(request) 80 | if response is not None: 81 | return response 82 | return FileResponse("static/index.html", media_type="text/html", headers={"Cache-Control": "no-store"}) 83 | 84 | async def execute(command, stdinput=None, verbose=False, no_stderr=False): 85 | global g_runningprocess 86 | global g_timeout 87 | global g_req_context 88 | global g_lam_context 89 | 90 | timeout = int(os.getenv("LEMMA_TIMEOUT", 60)) - 5 # subtract 5 seconds to allow for cleanup 91 | time_start = time.time() 92 | 93 | if g_req_context is not None: 94 | # we are running on AWS Lambda 95 | if verbose: 96 | r = json.loads(g_req_context) 97 | yield bytes(f"\x1b[32mLambda Request ID: \u001b[38;2;145;231;255m{r['requestId']}\x1b[0m\n", "utf-8") 98 | url = "http://checkip.amazonaws.com/" 99 | pubipv4 = requests.get(url).text.strip() 100 | yield bytes(f"\x1b[32mLambda Public IPv4: \u001b[38;2;145;231;255m{pubipv4}\x1b[0m\n", "utf-8") 101 | 102 | try: 103 | if verbose: 104 | yield bytes(f"\x1b[32mLambda Command: \u001b[38;2;145;231;255m", "utf-8") + bytes(str(shlex.split(command)), "utf-8") + b"\x1b[0m\n\n" 105 | 106 | process = await asyncio.create_subprocess_exec( 107 | *shlex.split(command), 108 | stdin=asyncio.subprocess.PIPE if stdinput else None, 109 | stdout=asyncio.subprocess.PIPE, 110 | stderr=asyncio.subprocess.PIPE if no_stderr else asyncio.subprocess.STDOUT 111 | ) 112 | except FileNotFoundError: 113 | if verbose: 114 | yield b"\n\x1b[31mRemote Error:\x1b[0m command not found\n" 115 | yield b"\r\n" 116 | return 117 | except: 118 | # yield back the traceback if the command failed to execute 119 | if verbose: 120 | yield traceback.format_exc().encode() 121 | yield b"\r\n" 122 | return 123 | 124 | # If input_data is provided, write it to the process's stdin 125 | if stdinput: 126 | process.stdin.write(stdinput) 127 | await process.stdin.drain() 128 | process.stdin.close() 129 | 130 | # Read and yield stdout data 131 | while True: 132 | try: 133 | data = await asyncio.wait_for(process.stdout.read(4096), timeout=1) 134 | except asyncio.exceptions.TimeoutError: 135 | if (time.time() - time_start) > timeout: 136 | process.kill() 137 | if verbose: 138 | yield b"\n\x1b[31mRemote Error:\x1b[0m lambda function timed out (Lemma Timeout: %d seconds)\n"%(timeout) 139 | yield b"\r\n" 140 | return 141 | continue 142 | if data: 143 | yield data 144 | else: 145 | break 146 | 147 | await process.wait() 148 | if verbose: 149 | yield b"\n\x1b[32mRemote Command Finished \x1b[38;2;145;231;255m- Elapsed Time: " + str(round(time.time() - time_start)).encode() + b" seconds\x1b[0m\n" 150 | 151 | @app.post("/runtool") 152 | async def tool( 153 | request: Request, 154 | cmd = Query(""), 155 | verbose = Query("false"), 156 | no_stderr = Query("false") 157 | ): 158 | response = access_allowed(request) 159 | if response is not None: 160 | return response 161 | 162 | verbose = True if verbose.lower() == "true" else False 163 | no_stderr = True if no_stderr.lower() == "true" else False 164 | 165 | global g_req_context 166 | global g_lam_context 167 | g_req_context = request.headers.get('x-amzn-request-context') 168 | g_lam_context = request.headers.get('x-amzn-lambda-context') 169 | 170 | stdinput = await request.body() 171 | cmd = urllib.parse.unquote(cmd).strip() 172 | 173 | # check if the command is in the tools directory 174 | if cmd.split()[0] not in tools: 175 | return Response(status_code=200, content="\x1b[31mError:\x1b[0m Command not found\n".encode()) 176 | 177 | cmd = "./tools/" + cmd 178 | 179 | headers = { 180 | "X-Lemma-Timeout": os.getenv("LEMMA_TIMEOUT", "60") 181 | } 182 | 183 | return StreamingResponse(execute(cmd, stdinput, verbose, no_stderr), media_type="text/html", headers=headers) -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.6.0 2 | anyio==4.2.0 3 | click==8.1.7 4 | exceptiongroup==1.2.0 5 | fastapi==0.109.2 6 | h11==0.14.0 7 | idna==3.7 8 | pydantic==2.6.1 9 | pydantic_core==2.16.2 10 | sniffio==1.3.0 11 | starlette==0.36.3 12 | typing_extensions==4.9.0 13 | uvicorn==0.27.0.post1 14 | setuptools 15 | requests 16 | -r tool_requirements.txt -------------------------------------------------------------------------------- /app/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PATH=$PATH:$LAMBDA_TASK_ROOT/bin \ 4 | PYTHONPATH=$PYTHONPATH:/opt/python:$LAMBDA_RUNTIME_DIR \ 5 | exec python -m uvicorn --port=$PORT main:app 6 | -------------------------------------------------------------------------------- /app/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Lemma Web Terminal 8 | 36 | 37 | 38 |
39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /app/static/lemma-term.js: -------------------------------------------------------------------------------- 1 | 2 | let disableInput = false; 3 | let prompt = '\x1b[33mlemma$\x1b[0m '; 4 | let commandBuffer = ''; 5 | let toollist = []; 6 | let lineBuffer = []; 7 | let history = []; 8 | let historyIndex = -1; 9 | let offset = 0; 10 | let lambdaurl = ""; 11 | let lemmaauth = ""; 12 | 13 | function stripAnsiCodes(str) { 14 | // Regular expression to match ANSI escape codes 15 | const ansiRegex = /\x1b\[[0-9;]*m/g; 16 | return str.replace(ansiRegex, ''); 17 | } 18 | 19 | async function load_tools() 20 | { 21 | try { 22 | const response = await fetch(lambdaurl + '/tools.json', { 23 | headers: { 24 | "x-lemma-api-key": lemmaauth 25 | } 26 | }); 27 | if (!response.ok) { 28 | throw new Error('Network response was not ok'); 29 | } 30 | const tools = await response.json(); 31 | return tools; 32 | } 33 | catch (error) { 34 | console.error('Error fetching tools:', error); 35 | } 36 | return []; 37 | } 38 | 39 | function populate_tools() { 40 | load_tools().then((tools) => { 41 | toollist = []; 42 | tools.forEach(tool => { 43 | toollist.push(tool); 44 | }); 45 | }); 46 | } 47 | 48 | function list_tools(terminal) { 49 | load_tools().then((tools) => { 50 | terminal.write('Available Remote Tools:\r\n'); 51 | toollist = []; 52 | tools.forEach(tool => { 53 | toollist.push(tool); 54 | terminal.write(` \u001b[38;2;145;231;255m${tool}\u001b[0m\r\n`); 55 | }); 56 | terminal.write("\r\n"); 57 | terminal.write("Run using \x1b[32mrun \u001b[38;2;145;231;255m \x1b[0m or \x1b[32mfork \u001b[38;2;145;231;255m \x1b[0m or simply \u001b[38;2;145;231;255m \x1b[0m\r\n"); 58 | terminal.write(prompt); 59 | }); 60 | } 61 | 62 | function typeNextChar(terminal, text, delay) { 63 | return new Promise((resolve) => { 64 | let index = 0; 65 | 66 | function step() { 67 | if (index < text.length) { 68 | terminal.write(text[index]); 69 | index++; 70 | setTimeout(step, delay); 71 | } else { 72 | terminal.write('\r\n\r\n'); 73 | list_tools(terminal); 74 | disableInput = false; 75 | resolve(); 76 | } 77 | } 78 | 79 | step(); 80 | }); 81 | } 82 | 83 | async function printWithDelay(terminal, text, delay) { 84 | await typeNextChar(terminal, text, delay); 85 | } 86 | 87 | async function intro(terminal) 88 | { 89 | disableInput = true; 90 | const sstr = "\u001b\u005b\u0033\u0033\u006d\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u2584\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2588\u2584\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2588\u2584\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2584\u0020\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2580\u2580\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2588\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2588\u2588\u2588\u2584\u2584\u2584\u2584\u2584\u0020\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u2588\u2588\u2584\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2580\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u2584\u2588\u2580\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u2588\u2588\u2584\u0020\u0020\u2584\u2588\u2580\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u2580\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u2588\u2588\u2588\u2588\u2580\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u2584\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u2580\u2588\u2588\u0020\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u2584\u2584\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u2584\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2588\u0020\u000d\u000a\u0020\u0020\u0020\u0020\u0020\u2580\u2588\u2588\u2588\u2588\u2588\u2588\u2580\u2580\u2580\u0020\u0020\u0020\u0020\u2580\u2588\u2588\u2588\u2588\u2588\u2580\u2580\u2580\u0020\u0020\u0020\u2590\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u0020\u2588\u2588\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u258c\u0020\u0020\u0020\u2588\u2588\u2580\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u0020\u2588\u2588\u2580\u001b\u005b\u0030\u006d\u000d\u000a" 91 | terminal.write(sstr+ "\r\n\r\n"); 92 | terminal.write(' '); 93 | const phrase = 'Response Streaming CLI Tools on AWS Lambda'; 94 | const delay = 25; // Delay in milliseconds 95 | 96 | printWithDelay(terminal, phrase, delay); 97 | } 98 | 99 | async function execute_remote_tool(terminal, args) { 100 | const abortController = new AbortController(); 101 | 102 | try { 103 | const url = new URL('/runtool', lambdaurl); 104 | url.searchParams.set('cmd', encodeURIComponent(args)); 105 | url.searchParams.set('verbose', "true"); 106 | 107 | const response = await fetch(url.toString(), { 108 | method: 'POST', 109 | signal: abortController.signal, 110 | headers: { 111 | "x-lemma-api-key": lemmaauth 112 | } 113 | }); 114 | 115 | if (!response.body) { 116 | throw new Error('ReadableStream not supported.'); 117 | } 118 | 119 | // Get the timeout from the header and set the timeout 120 | const timeoutHeader = response.headers.get('x-lemma-timeout'); 121 | const timeout = parseInt(timeoutHeader, 10) * 1000; // Convert to milliseconds 122 | 123 | // Set a timeout to abort the request 124 | const timeoutId = setTimeout(() => { 125 | abortController.abort(); 126 | terminal.write('\r\n\u001b[31mError: Stream timeout exceeded.\x1b[0m\r\n'); 127 | }, timeout); 128 | 129 | const reader = response.body.getReader(); 130 | const decoder = new TextDecoder('utf-8'); 131 | 132 | while (true) { 133 | const { done, value } = await reader.read(); 134 | if (done) { 135 | clearTimeout(timeoutId); // Clear the timeout if done 136 | break; 137 | } 138 | let chunk = decoder.decode(value, { stream: true }); 139 | // replace any \n with \r\n 140 | chunk = chunk.replace(/\n/g, '\r\n'); 141 | terminal.write(chunk); 142 | } 143 | 144 | terminal.write('\r\n\u001b[38;2;145;231;255mRemote tool execution complete.\x1b[0m\r\n'); 145 | disableInput = false; 146 | terminal.write(prompt); 147 | } catch (error) { 148 | if (error.name === 'AbortError') { 149 | terminal.write('\r\n\u001b[31mError: Remote execution failed due to timeout.\x1b[0m\r\n'); 150 | } else { 151 | terminal.write(`\r\nError: ${error.message}\r\n`); 152 | } 153 | disableInput = false; 154 | terminal.write(prompt); 155 | } 156 | } 157 | 158 | document.addEventListener('DOMContentLoaded', () => { 159 | 160 | 161 | let truerows = 45; 162 | let truecols = 150; 163 | 164 | if (terminalContainer.clientWidth <= 1024) { 165 | truecols = 100; 166 | } 167 | 168 | 169 | const terminal = new Terminal({ 170 | rows: truerows, 171 | cols: truecols 172 | }); 173 | const fitAddon = new FitAddon.FitAddon(); 174 | terminal.loadAddon(fitAddon); 175 | function fitTerminal() { 176 | const containerWidth = terminalContainer.clientWidth; 177 | const containerHeight = terminalContainer.clientHeight; 178 | const cols = truecols; 179 | const rows = truerows; 180 | const cellWidth = containerWidth / cols; 181 | const cellHeight = containerHeight / rows; 182 | // Set the font size based on the smallest dimension to maintain aspect ratio 183 | const fontSize = Math.min(cellWidth, cellHeight) * 1.6; 184 | console.log(cols) 185 | // 186 | // // Apply the calculated font size to the terminal 187 | // // Apply the calculated font size to the terminal 188 | terminal.options.fontSize = fontSize; 189 | 190 | fitAddon.fit(); 191 | console.log("fitting terminal") 192 | } 193 | 194 | terminal.open(document.getElementById('terminalContainer')); 195 | fitTerminal(); 196 | terminal.focus(); 197 | 198 | // Adjust terminal size when window is resized 199 | window.addEventListener('resize', fitTerminal); 200 | 201 | // first lets get the LAMBDA_URL cookie using document.cookie 202 | const cookies = document.cookie.split(';'); 203 | 204 | cookies.forEach(cookie => { 205 | if (cookie.includes('LEMMA_URL')) { 206 | lambdaurl = cookie.split('=')[1]; 207 | } 208 | }); 209 | 210 | cookies.forEach(cookie => { 211 | if (cookie.includes('LEMMA_OVERRIDE_API_KEY')) { 212 | lemmaauth = cookie.split('=')[1]; 213 | } 214 | }); 215 | 216 | if (lambdaurl === "") { 217 | // get the host name of the page 218 | lambdaurl = window.location.origin 219 | } 220 | 221 | 222 | // check if the command query has been set 223 | const urlParams = new URLSearchParams(window.location.search); 224 | const command = urlParams.get('cmd'); 225 | if (command) { 226 | populate_tools(); 227 | terminal.write(prompt+`${command}\r\n`); 228 | executeCommand(command); 229 | } 230 | else 231 | { 232 | intro(terminal); 233 | } 234 | 235 | async function simpleShell(term, data) { 236 | let CurX = term.buffer.active.cursorX; 237 | let CurY = term.buffer.active.cursorY; 238 | let MaxX = term.cols; 239 | let MaxY = term.rows; 240 | 241 | if (disableInput === true) { 242 | return; 243 | } 244 | // string splitting is needed to also handle multichar input (eg. from copy) 245 | for (let i = 0; i < data.length; ++i) { 246 | const c = data[i]; 247 | if (c === '\r') { // was pressed case 248 | offset = 0; 249 | term.write('\r\n'); 250 | if (lineBuffer.length) { 251 | // we have something in line buffer, normally a shell does its REPL logic here 252 | // for simplicity - just join characters and exec... 253 | const command = lineBuffer.join(''); 254 | lineBuffer.length = 0; 255 | history.push(command); 256 | historyIndex = history.length; 257 | executeCommand(command); 258 | } 259 | else { 260 | term.write(prompt); 261 | } 262 | } else if (c === '\x7F') { // was pressed case 263 | if (lineBuffer.length) { 264 | if (offset === 0) { 265 | if (CurX === 0) { 266 | // go to the previous line end 267 | term.write('\x1b[1A'); // control code: move up one line 268 | term.write('\x1b[' + MaxX + 'C'); // control code: move to the end of the line 269 | } 270 | lineBuffer.pop(); 271 | term.write('\b \b'); 272 | } 273 | } 274 | } else if (['\x1b[5', '\x1b[6'].includes(data.slice(i, i + 3))) { 275 | // not implemented 276 | i += 3; 277 | } else if (['\x1b[F', '\x1b[H'].includes(data.slice(i, i + 3))) { 278 | 279 | if (data.slice(i, i + 3) === '\x1b[H') { // Home key 280 | // not implemented 281 | } 282 | else if (data.slice(i, i + 3) === '\x1b[F') { // End key 283 | // not implemented 284 | } 285 | i += 3; 286 | } else if (['\x1b[A', '\x1b[B', '\x1b[C', '\x1b[D'].includes(data.slice(i, i + 3))) { // keys pressed 287 | if (data.slice(i, i + 3) === '\x1b[A') { // up arrow 288 | if (historyIndex > 0) { 289 | historyIndex--; 290 | updateCommandBuffer(history[historyIndex]); 291 | } 292 | } else if (data.slice(i, i + 3) === '\x1b[B') { // down arrow 293 | if (historyIndex < history.length - 1) { 294 | historyIndex++; 295 | updateCommandBuffer(history[historyIndex]); 296 | 297 | } else { 298 | historyIndex = history.length; 299 | updateCommandBuffer(''); 300 | } 301 | } 302 | else if (data.slice(i, i + 3) === '\x1b[C') { // right arrow 303 | // not implemented 304 | } 305 | else if (data.slice(i, i + 3) === '\x1b[D') { // left arrow 306 | // not implemented 307 | } 308 | i += 2; 309 | } else { // push everything else into the line buffer and echo back to user 310 | // if we are at the end of the line, 311 | // move up a row and to the beginning of the line 312 | if (CurX === MaxX - 1) { 313 | term.write('\r\n'); 314 | } 315 | lineBuffer.push(c); 316 | term.write(c); 317 | 318 | } 319 | } 320 | } 321 | 322 | terminal.onData(data => simpleShell(terminal, data)); 323 | 324 | function executeCommandSingle(command) { 325 | 326 | // Empty function for now 327 | //terminal.write(`Executing command: ${command}\r\n`); 328 | // split command and get first token 329 | const command0 = command.split(' ')[0]; 330 | const command1 = command.split(' ')[1]; 331 | 332 | if (command0 === 'help') { 333 | terminal.write('Available Local Commands:\r\n'); 334 | terminal.write(' \x1b[32mhelp -\x1b[0m Show this help message\r\n'); 335 | terminal.write(' \x1b[32mclear -\x1b[0m Clear the terminal\r\n'); 336 | terminal.write(' \x1b[32mtools -\x1b[0m Show a list of remote tools\r\n'); 337 | terminal.write(' \x1b[32msize -\x1b[0m Show or Set terminal size (i.e size, size 45x100)\r\n'); 338 | terminal.write(' \x1b[32mrun -\x1b[0m Run a remote tool in the current terminal\r\n'); 339 | terminal.write(' \x1b[32mfork -\x1b[0m Run a remote tool in a new terminal\r\n'); 340 | terminal.write(' \x1b[32mset-url -\x1b[0m Set the lambda URL\r\n'); 341 | terminal.write(prompt); 342 | } else if (command0 === 'clear') { 343 | terminal.clear(); 344 | terminal.write(prompt); 345 | } else if (command0 === 'tools') { 346 | list_tools(terminal); 347 | } else if (command0 === 'reset') { 348 | truerows = 45; 349 | truecols = 150; 350 | 351 | if (terminalContainer.clientWidth <= 1024) { 352 | truecols = 100; 353 | } 354 | toollist = []; 355 | terminal.clear(); 356 | terminal.resize(truerows, truecols); 357 | fitTerminal() 358 | intro(terminal); 359 | } else if (command0 === 'size') { 360 | 361 | if (command1 === undefined) { 362 | terminal.write("Terminal size: " + truerows + "x" + truecols + "\r\n"); 363 | terminal.write(prompt); 364 | return 365 | } 366 | 367 | const r = command1.split('x')[0]; 368 | const c = command1.split('x')[1]; 369 | 370 | if (r === undefined || c === undefined) { 371 | terminal.write(prompt); 372 | return 373 | } 374 | 375 | // resize the terminal based on r and c 376 | terminal.resize(r, c); 377 | truerows = r; 378 | truecols = c; 379 | fitTerminal() 380 | terminal.write(prompt); 381 | 382 | } else if (command0 === 'fork') { 383 | const args = ("run " + command.split(' ').slice(1).join(' ')); 384 | const url = new URL(window.location.href); 385 | const finalurl = url.origin + url.pathname + "?cmd=" + encodeURIComponent(args); 386 | window.open(finalurl, '_blank'); 387 | terminal.write(prompt); 388 | 389 | } else if (command0 === 'set-url') { 390 | // lets take the LAMBDA_URL and set it as a cookie called LAMBDA_URL 391 | const url = command.split(' ').slice(1).join(' ').trim(); 392 | 393 | if ((url === "") || (url === undefined)) { 394 | document.cookie = "LEMMA_URL=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/"; 395 | document.cookie = "LEMMA_OVERRIDE_API_KEY=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/"; 396 | lambdaurl = window.location.origin; 397 | lemmaauth = ""; 398 | 399 | terminal.write(`\x1b[32mLambda URL reset\x1b[0m\r\n`); 400 | terminal.write(prompt); 401 | return 402 | } 403 | 404 | let parsedUrl; 405 | try { 406 | parsedUrl = new URL(url); 407 | } catch (_) { 408 | terminal.write(`\x1b[31mInvalid Lambda URL:\x1b[0m ${url}\r\n`); 409 | terminal.write(prompt); 410 | return 411 | } 412 | 413 | const hostname = parsedUrl.hostname; 414 | const keyValue = parsedUrl.searchParams.get('key'); 415 | 416 | if (keyValue !== null) { 417 | document.cookie = "LEMMA_URL=https://" + hostname + "; path=/" 418 | document.cookie = "LEMMA_OVERRIDE_API_KEY=" + keyValue + "; path=/" 419 | lambdaurl = "https://" + hostname; 420 | lemmaauth = keyValue; 421 | terminal.write(`\x1b[32mLambda URL set to:\x1b[0m ${url}\r\n`); 422 | terminal.write(prompt); 423 | } 424 | else { 425 | terminal.write(`\x1b[31mInvalid Lambda URL:\x1b[0m ${url}\r\n`); 426 | terminal.write(prompt); 427 | } 428 | 429 | } else if (command0 === 'run') { 430 | const args = command.split(' ').slice(1).join(' '); 431 | disableInput = true; 432 | execute_remote_tool(terminal, args); 433 | } else { 434 | 435 | // check if the command is a tool 436 | if (toollist.includes(command0)) 437 | { 438 | disableInput = true; 439 | execute_remote_tool(terminal, command); 440 | } 441 | else 442 | { 443 | 444 | terminal.write(`\x1b[31mCommand not found:\x1b[0m ${command0}\r\n`); 445 | terminal.write(prompt); 446 | } 447 | } 448 | 449 | } 450 | 451 | function executeCommand(command) { 452 | if (command.includes(';')) { 453 | const commands = command.split(';'); 454 | commands.forEach((cmd) => { 455 | // Execute each command in the list and trim any leading/trailing whitespace 456 | executeCommandSingle(cmd.trim()); 457 | }); 458 | } else { 459 | executeCommandSingle(command); 460 | } 461 | } 462 | 463 | function updateCommandBuffer(command) { 464 | // Clear current line 465 | 466 | terminal.write('\r'+prompt + ' '.repeat(lineBuffer.length) + '\r'+prompt); 467 | // push every character in the command to the lineBuffer 468 | lineBuffer = command.split(''); 469 | 470 | // Convert the command string into an array of characters 471 | let commandArray = command.split(''); 472 | let promptlen = stripAnsiCodes(prompt).length 473 | let i = promptlen; 474 | while (i < commandArray.length) { 475 | if (i % terminal.cols === 0 && i !== 0 ) { 476 | commandArray.splice(i-promptlen-1, 0, '\r\n'); 477 | } 478 | i++; 479 | } 480 | terminal.write(commandArray.join('')); 481 | } 482 | 483 | // Apply custom CSS to make the scrollbar invisible 484 | const terminalElement = document.querySelector('#terminal .xterm-viewport'); 485 | if (terminalElement) { 486 | terminalElement.style.overflowY = 'hidden'; 487 | } 488 | }); 489 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | logo() { 4 | echo -e "\x1b[33m" 5 | echo -e " ██ ▄▄▄▄▄▄▄▄▄▄▄▄▄ ▐██ ▄█▄ ██ ▄█▄ ▄▄ " 6 | echo -e " ██▌ ▀▀ ▐██ ███ ███▌ ███ ███▌ ████ " 7 | echo -e " ▐██ ███ ███▌ ████ ███▌ ████ ▄█ ██ " 8 | echo -e " ██▌ ▄███▄▄▄▄▄ ▐████ █████ ▐████▄ █████ █▀ ██▌ " 9 | echo -e " ▐██ ▐██ ██ ▐██ ▄█▀ ███ ██ ██▄ ▄█▀ ███ ██████████ " 10 | echo -e " ███ ███ ███ ████▀ ██▌ ▐██ ████▀ ██▌ ▄██ ▀██ " 11 | echo -e " ██▌ ▄▄ ██▌ ██ ██▌ ██ ██▌ ▄██ ███ " 12 | echo -e " ▀██████▀▀▀ ▀█████▀▀▀ ▐██ ██▌ ██ ██▌ ██▀ ██▀\x1b[0m " 13 | echo -e "" 14 | echo -e " Response Streaming CLI Tools on AWS Lambda " 15 | echo -e "" 16 | echo -e "Lemma build and deploy script" 17 | echo -e "" 18 | } 19 | 20 | function choose_aws_region() { 21 | local choice 22 | while true; do 23 | read -p "Choose an AWS region to deploy to [default: us-east-1]: " choice 24 | choice=${choice:-us-east-1} 25 | 26 | if [[ "$choice" =~ ^[a-zA-Z0-9-]+$ ]]; then 27 | # return the choice 28 | aws_region=$choice 29 | break 30 | else 31 | echo "Invalid choice. Please enter a valid AWS region." 32 | fi 33 | done 34 | } 35 | 36 | function choose_architecture() { 37 | local choice 38 | while true; do 39 | read -p "Choose architecture (arm64 or x86_64) [default: arm64]: " choice 40 | choice=${choice:-arm64} 41 | 42 | if [[ "$choice" == "arm64" || "$choice" == "x86_64" ]]; then 43 | # return the choice 44 | arch=$choice 45 | break 46 | else 47 | echo "Invalid choice. Please enter 'arm64' or 'x86_64'." 48 | fi 49 | done 50 | } 51 | 52 | function lambda_timeout() { 53 | local choice 54 | while true; do 55 | read -p "Choose lambda timeout (1-900 seconds) [default: 300]: " choice 56 | choice=${choice:-300} 57 | 58 | if [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le 900 ]; then 59 | # return the choice 60 | lambda_timeout=$choice 61 | break 62 | else 63 | echo "Invalid choice. Please enter a number between 1 and 900." 64 | fi 65 | done 66 | } 67 | 68 | function lambda_memory() { 69 | local choice 70 | while true; do 71 | read -p "Choose lambda memory limit (multiples of 64, 128-10240 MB) [default: 1024]: " choice 72 | choice=${choice:-1024} 73 | 74 | if [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 128 ] && [ "$choice" -le 10240 ] && [ "$(($choice % 64))" -eq 0 ]; then 75 | # return the choice 76 | lambda_memory=$choice 77 | break 78 | else 79 | echo "Invalid choice. Please enter a number between 128 and 10240, a multiple of 64 only." 80 | fi 81 | done 82 | } 83 | 84 | function install_tools() { 85 | # ask a Y/N question to the user if they want to install tools, default is Y 86 | local choice 87 | while true; do 88 | read -p "Do you want to install tools into the lambda package? [Y/n]: " choice 89 | choice=${choice:-Y} 90 | 91 | if [[ "$choice" == "Y" || "$choice" == "y" ]]; then 92 | # run the install tools script 93 | echo "Installing tools..." 94 | ./tools/install_tools.sh $arch 95 | echo -e "Tools installed\n" 96 | break 97 | elif [[ "$choice" == "N" || "$choice" == "n" ]]; then 98 | break 99 | else 100 | echo "Invalid choice. Please enter 'Y' or 'N'." 101 | fi 102 | done 103 | } 104 | 105 | function remove_template_ask() { 106 | # ask a Y/N question to the user if they want to install tools, default is Y 107 | local choice 108 | while true; do 109 | read -p "template.yaml exists, create a new template? [y/N]: " choice 110 | choice=${choice:-N} 111 | 112 | if [[ "$choice" == "Y" || "$choice" == "y" ]]; then 113 | rm -f template.yaml 114 | echo -e "Template removed\n" 115 | break 116 | elif [[ "$choice" == "N" || "$choice" == "n" ]]; then 117 | echo -e "" 118 | break 119 | else 120 | echo "Invalid choice. Please enter 'Y' or 'N'." 121 | fi 122 | done 123 | } 124 | 125 | logo 126 | 127 | if [ -f /.dockerenv ]; then 128 | echo "Lemma Build and Deploy..." 129 | else 130 | echo "Docker Build and Run..." 131 | docker build -t lemma . 132 | 133 | if [ "$1" == "delete" ]; then 134 | docker run -it --rm -v ~/.aws:/root/.aws -v .:/lambda \ 135 | -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \ 136 | lemma /lambda/build.sh delete 137 | exit 0 138 | fi 139 | 140 | # forward AWS credentials to the container in both .aws and environment variables 141 | docker run -it --rm -v ~/.aws:/root/.aws -v .:/lambda \ 142 | -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \ 143 | lemma /lambda/build.sh 144 | exit 0 145 | fi 146 | 147 | # check if arg1 is 'delete' 148 | if [ "$1" == "delete" ]; then 149 | echo "Removing Lemma Lambda from AWS... (Please wait, this may take a while)" 150 | sam delete --no-prompts > delete.log 2>&1 151 | # check if it fails 152 | if [ $? -ne 0 ]; then 153 | echo -e "\x1b[31mRemoval failed. Check delete.log for more information.\x1b[0m" 154 | exit 1 155 | fi 156 | echo -e "Removal successful\n" 157 | exit 0 158 | fi 159 | 160 | 161 | # check if template.yaml exists 162 | if [ -f template.yaml ]; then 163 | remove_template_ask 164 | fi 165 | 166 | # if template.yaml doesn't exist, run these functions 167 | if [ ! -f template.yaml ]; then 168 | cp -f ./templates/samconfig.toml . 169 | 170 | choose_aws_region 171 | echo -e "AWS region specified: $aws_region\n" 172 | 173 | # replace %REGION% with aws_region 174 | sed -i "s/%REGION%/$aws_region/g" samconfig.toml 175 | 176 | choose_architecture 177 | echo -e "Architecture specified: $arch\n" 178 | lambda_timeout 179 | echo -e "Lambda timeout specified: $lambda_timeout\n" 180 | lambda_memory 181 | echo -e "Lambda memory specified: $lambda_memory\n" 182 | 183 | # generate a random API key 184 | api_key=$(openssl rand -hex 8) 185 | 186 | #check if arch is arm64 187 | if [ "$arch" == "arm64" ]; then 188 | cp ./templates/template_arm64.yaml ./template.yaml 189 | else 190 | cp ./templates/template_x86.yaml ./template.yaml 191 | fi 192 | 193 | # replace %MEMORY% with lambda_memory 194 | sed -i "s/%MEMORY%/$lambda_memory/g" template.yaml 195 | # replace %TIMEOUT% with lambda_timeout 196 | sed -i "s/%TIMEOUT%/$lambda_timeout/g" template.yaml 197 | # replace %API_KEY% with api_key 198 | sed -i "s/%API_KEY%/$api_key/g" template.yaml 199 | fi 200 | 201 | arch=$(grep -A 1 'Architectures:' template.yaml | awk '/- / {print $2}') 202 | api_key=$(grep -A 5 'Environment:' template.yaml | grep 'LEMMA_API_KEY:' | awk '{print $2}') 203 | 204 | install_tools 205 | 206 | rm -rf .aws-sam 207 | 208 | echo "Building Lemma Lambda... (Please wait, this may take a while)" 209 | sam build > build.log 2>&1 210 | # check if it fails 211 | if [ $? -ne 0 ]; then 212 | echo -e "\x1b[31mBuild failed. Check build.log for more information.\x1b[0m" 213 | exit 1 214 | fi 215 | 216 | echo -e "Build successful\n" 217 | 218 | echo "Deploying Lemma Lambda to AWS... (Please wait, this may take a while)" 219 | sam deploy > deploy.log 2>&1 220 | # check if it fails 221 | if [ $? -ne 0 ]; then 222 | echo -e "\x1b[31mDeployment failed. Check deploy.log for more information.\x1b[0m" 223 | exit 1 224 | fi 225 | 226 | echo -e "Deployment successful\n" 227 | 228 | echo -e "To remove the Lambda, run: \x1b[32m./build.sh delete\x1b[0m" 229 | echo -e "To update the Lambda with new tools re-run \x1b[32m./build.sh\x1b[0m\n" 230 | 231 | URL=$(tr -d '\n ' < deploy.log | sed -n 's/.*LEMMA_URL:\(https:\/\/[^ ]*\.aws\/\).*/\1/p') 232 | 233 | echo -e "Your Lemma Lambda URL is: \x1b[32m$URL?key=$api_key\x1b[0m" 234 | -------------------------------------------------------------------------------- /images/build.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/build.gif -------------------------------------------------------------------------------- /images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/demo.gif -------------------------------------------------------------------------------- /images/demo2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/demo2.gif -------------------------------------------------------------------------------- /images/demo3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/demo3.gif -------------------------------------------------------------------------------- /images/e1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/e1.png -------------------------------------------------------------------------------- /images/e2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/e2.png -------------------------------------------------------------------------------- /images/e3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/e3.png -------------------------------------------------------------------------------- /images/e4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/e4.png -------------------------------------------------------------------------------- /images/e5.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/e5.gif -------------------------------------------------------------------------------- /images/lemma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/images/lemma.png -------------------------------------------------------------------------------- /lemma/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vitorfhc/lemma-fork/bdc90ddb843133f891866863d46b544c83fc81e5/lemma/__init__.py -------------------------------------------------------------------------------- /lemma/__main__.py: -------------------------------------------------------------------------------- 1 | from lemma.settings import get_settings, get_profiler 2 | from lemma.pipeline import Pipeline 3 | from lemma.input_adapter import InputAdapter 4 | from lemma.output_adapter import OutputAdapter 5 | import sys 6 | 7 | 8 | def main(): 9 | settings = get_settings() 10 | 11 | if settings.remote_command == "": 12 | print("error: no remote command specified", file=sys.stderr) 13 | exit(1) 14 | 15 | if not settings.stdin_pipe_exists and settings.args.per_stdin: 16 | print("error: cannot invoke per stdin line, stdin has no pipe", file=sys.stderr) 17 | exit(1) 18 | 19 | pipe = Pipeline( 20 | settings=settings, 21 | input_adapter=InputAdapter(), 22 | output_adapter=OutputAdapter(), 23 | ) 24 | 25 | pipe.run() 26 | 27 | if __name__ == "__main__": 28 | main() -------------------------------------------------------------------------------- /lemma/input_adapter.py: -------------------------------------------------------------------------------- 1 | from lemma.settings import get_settings 2 | from select import select 3 | import sys 4 | 5 | def format_command(command, index, stdin_data): 6 | if (('%STDIN%' in command) and ("\n" in stdin_data.strip())): 7 | print("error: cannot place stdin onto remote command with newline characters", file=sys.stderr) 8 | exit(1) 9 | 10 | command = command.replace(r"%INDEX%", str(index)) 11 | command = command.replace(r"%STDIN%", stdin_data.strip()) 12 | return command 13 | 14 | class InputAdapter: 15 | def __init__(self): 16 | self.settings = get_settings() 17 | self.done = False 18 | self.stdin_closed = False 19 | self.count = 0 20 | 21 | def readline_stdin(self): 22 | # Check if there is any data to read from stdin 23 | ready_to_read, _, _ = select([sys.stdin], [], [], 1.0) 24 | if sys.stdin in ready_to_read: 25 | line = sys.stdin.readline() 26 | if line == "": 27 | self.stdin_closed = True 28 | return None 29 | return line 30 | else: 31 | return None 32 | 33 | def read_stdin(self): 34 | if self.stdin_closed: 35 | return None 36 | data = sys.stdin.read() 37 | self.stdin_closed = True 38 | return data 39 | 40 | def process(self, command_queue): 41 | if self.done: 42 | return 43 | # lambdas are either invoked per stdin line or per the invocations argument value 44 | if self.settings.args.per_stdin: 45 | line = self.readline_stdin() 46 | if line is not None: 47 | remote_command = self.settings.remote_command 48 | command_queue.put((format_command(remote_command, self.count, line),line,)) 49 | self.count += 1 50 | if self.stdin_closed: 51 | self.done = True 52 | elif self.settings.args.div_stdin: 53 | stdin_data = "" 54 | if self.settings.stdin_pipe_exists: 55 | stdin_data = self.read_stdin() 56 | remote_command = self.settings.remote_command 57 | 58 | # we need to divide the stdin data into div_stdin equal parts at the newline boundary 59 | parts = stdin_data.split('\n') 60 | parts_len = len(parts) 61 | # create step and round up to the nearest integer 62 | step = -(-parts_len // int(self.settings.args.div_stdin)) 63 | for i in range(0, parts_len, step): 64 | stdin_data = '\n'.join(parts[i:i+step]) 65 | command_queue.put((format_command(remote_command, self.count, stdin_data),stdin_data,)) 66 | self.count += 1 67 | self.done = True 68 | else: 69 | stdin_data = "" 70 | if self.settings.stdin_pipe_exists: 71 | stdin_data = self.read_stdin() 72 | 73 | remote_command = self.settings.remote_command 74 | for i in range(int(self.settings.args.invocations)): 75 | command_queue.put((format_command(remote_command, i, stdin_data),stdin_data,)) 76 | 77 | self.done = True 78 | -------------------------------------------------------------------------------- /lemma/lambda_worker.py: -------------------------------------------------------------------------------- 1 | import time 2 | import select 3 | from lemma.settings import get_settings 4 | import requests 5 | import urllib.parse 6 | import random 7 | 8 | class LambdaService: 9 | def __init__(self, worker): 10 | self.worker = worker 11 | 12 | def invoke(self, command, stdin): 13 | # URL encode the command 14 | command = urllib.parse.quote(command) 15 | body_data = stdin 16 | 17 | url = get_settings().lambda_url + '/runtool?cmd=' + command 18 | if get_settings().args.verbose: 19 | url += '&verbose=true' 20 | if get_settings().args.no_stderr: 21 | url += '&no_stderr=true' 22 | 23 | # Set the LEMMA_API_KEY cookie 24 | cookies = {'LEMMA_API_KEY': get_settings().lambda_key} 25 | 26 | # Check if feeding stdin is enabled and if it is then add the stdin to the request body 27 | if get_settings().args.omit_stdin: 28 | body_data = "" 29 | 30 | with requests.Session() as session: 31 | # Perform a POST request to the Lambda URL with streaming enabled 32 | with session.post(url, cookies=cookies, data=body_data, stream=True) as response: 33 | # Ensure the request was successful 34 | try: 35 | response.raise_for_status() 36 | except: 37 | print(response.status_code) 38 | return response.status_code 39 | 40 | # Get the X-Lemma-Timeout header 41 | timeout = response.headers.get('X-Lemma-Timeout') 42 | if timeout: 43 | timeout = float(timeout) 44 | start_time = time.time() 45 | 46 | buffer = "" 47 | sock = response.raw._fp.fp.raw._sock # Get the raw socket from the response 48 | x = response.raw.stream(4096, decode_content=False) 49 | 50 | # Process the stream in chunks 51 | while True: 52 | rlist, _, _ = select.select([sock], [], [], 0.01) 53 | # Break the loop if no more data to read 54 | if not rlist: 55 | if timeout and (time.time() - start_time) > timeout: 56 | # LEMMA timeout exceeded, bail out the thread 57 | break 58 | continue 59 | 60 | try: 61 | chunk = next(x) 62 | except StopIteration: 63 | break 64 | 65 | decoded_chunk = chunk.decode('utf-8') 66 | if get_settings().args.line_buffered: 67 | buffer += decoded_chunk 68 | while '\n' in buffer: 69 | line, buffer = buffer.split('\n', 1) 70 | self.worker.push(line + '\n') 71 | else: 72 | self.worker.push(decoded_chunk) 73 | 74 | 75 | if buffer: 76 | self.worker.push(buffer) 77 | 78 | return response.status_code 79 | 80 | class LambdaWorker: 81 | stop = False 82 | 83 | def __init__(self, command_queue, stdout_queue): 84 | self.command_queue = command_queue 85 | self.stdout_queue = stdout_queue 86 | self.idle = True 87 | 88 | def push(self, stdoutitem): 89 | self.stdout_queue.put(stdoutitem) 90 | 91 | def run(self): 92 | while True: 93 | if LambdaWorker.stop: 94 | break 95 | 96 | try: 97 | command, stdin = self.command_queue.get_nowait() 98 | except: 99 | continue 100 | 101 | self.idle = False 102 | LambdaService(self).invoke(command, stdin) 103 | self.command_queue.task_done() 104 | self.idle = True 105 | 106 | @classmethod 107 | def stop_all_workers(cls): 108 | LambdaWorker.stop = True 109 | -------------------------------------------------------------------------------- /lemma/logo.py: -------------------------------------------------------------------------------- 1 | logo = """\x1b[33m 2 | ██ ▄▄▄▄▄▄▄▄▄▄▄▄▄ ▐██ ▄█▄ ██ ▄█▄ ▄▄ 3 | ██▌ ▀▀ ▐██ ███ ███▌ ███ ███▌ ████ 4 | ▐██ ███ ███▌ ████ ███▌ ████ ▄█ ██ 5 | ██▌ ▄███▄▄▄▄▄ ▐████ █████ ▐████▄ █████ █▀ ██▌ 6 | ▐██ ▐██ ██ ▐██ ▄█▀ ███ ██ ██▄ ▄█▀ ███ ██████████ 7 | ███ ███ ███ ████▀ ██▌ ▐██ ████▀ ██▌ ▄██ ▀██ 8 | ██▌ ▄▄ ██▌ ██ ██▌ ██ ██▌ ▄██ ███ 9 | ▀██████▀▀▀ ▀█████▀▀▀ ▐██ ██▌ ██ ██▌ ██▀ ██▀\x1b[0m 10 | 11 | Response Streaming CLI Tools on AWS Lambda 12 | """ -------------------------------------------------------------------------------- /lemma/output_adapter.py: -------------------------------------------------------------------------------- 1 | from lemma.settings import get_settings 2 | 3 | class OutputAdapter: 4 | def __init__(self): 5 | self.settings = get_settings() 6 | 7 | def process(self, stdout_queue): 8 | if (stdout_queue.empty()): 9 | return 10 | 11 | data = stdout_queue.get() 12 | print(data, end='', flush=True) 13 | stdout_queue.task_done() 14 | -------------------------------------------------------------------------------- /lemma/pipeline.py: -------------------------------------------------------------------------------- 1 | import sys, time 2 | from threading import Thread 3 | from queue import Queue 4 | from lemma.lambda_worker import LambdaWorker 5 | 6 | class Pipeline: 7 | def __init__(self, settings, input_adapter, output_adapter): 8 | self.settings = settings 9 | self.input_adapter = input_adapter 10 | self.output_adapter = output_adapter 11 | self.command_queue = Queue() 12 | self.stdout_queue = Queue() 13 | self.worker_pool = [] 14 | 15 | def pool_create(self): 16 | self.worker_pool = [] 17 | for i in range(int(self.settings.args.workers)): 18 | worker = LambdaWorker(self.command_queue, self.stdout_queue) 19 | self.worker_pool.append(worker) 20 | thread = Thread(target=LambdaWorker.run, args=(worker,), daemon=True) 21 | thread.start() 22 | 23 | def pool_idle(self): 24 | return all(worker.idle for worker in self.worker_pool) 25 | 26 | def pool_stop(self): 27 | LambdaWorker.stop_all_workers() 28 | 29 | def queues_empty(self): 30 | return self.command_queue.empty() and self.stdout_queue.empty() 31 | 32 | def run(self): 33 | 34 | self.pool_create() 35 | 36 | while True: 37 | self.input_adapter.process(self.command_queue) 38 | self.output_adapter.process(self.stdout_queue) 39 | time.sleep(0.01) 40 | if self.input_adapter.done and self.queues_empty() and self.pool_idle(): 41 | self.pool_stop() 42 | break 43 | -------------------------------------------------------------------------------- /lemma/settings.py: -------------------------------------------------------------------------------- 1 | from lemma.logo import logo 2 | from functools import lru_cache 3 | from argparse import ArgumentParser, REMAINDER, RawDescriptionHelpFormatter 4 | from configparser import ConfigParser 5 | import urllib.parse 6 | import requests 7 | import os, sys 8 | import cProfile 9 | 10 | def tools(settings): 11 | print('Available Remote Tools:') 12 | url = settings.lambda_url + 'tools.json' 13 | try: 14 | response = requests.get(url, cookies={'LEMMA_API_KEY': settings.lambda_key},timeout=5) 15 | except: 16 | print(' \u001b[31m\u001b[0m', file=sys.stderr) 17 | sys.exit(1) 18 | 19 | if response.status_code != 200: 20 | print(' \u001b[31m\u001b[0m', file=sys.stderr) 21 | sys.exit(1) 22 | 23 | tools = response.json() 24 | for tool in tools: 25 | print(' \u001b[38;2;145;231;255m' + tool + '\u001b[0m') 26 | 27 | 28 | @lru_cache(maxsize=1) 29 | def get_args(): 30 | parser = ArgumentParser(description=logo,formatter_class=RawDescriptionHelpFormatter) 31 | parser.add_argument('-w', '--workers', default=1, help='Number of concurrent Lambda service workers') 32 | parser.add_argument('-l', '--lambda-url', help='Prompt user to enter a new lambda url', action='store_true') 33 | parser.add_argument('-i', '--invocations', default=1, help='The number of invocations of the remote command') 34 | parser.add_argument('-p', '--per-stdin', help='Invoke the remote command for each line of stdin (-i is ignored)', action='store_true') 35 | parser.add_argument('-d', '--div-stdin', help='Divide stdin into DIV_STDIN parts at a newline boundary and invoke on each (-i is ignored)') 36 | parser.add_argument('-o', '--omit-stdin', help='Omit stdin to the remote command stdin', action='store_true') 37 | parser.add_argument('-e', '--no-stderr', help='prevent stderr from being streamed into response', action='store_true') 38 | parser.add_argument('-b', '--line-buffered', help='Stream only line chunks to stdout', action='store_true') 39 | parser.add_argument('-v', '--verbose', help='Enable verbose remote output', action='store_true') 40 | parser.add_argument('-t', '--tools', help='List available tools', action='store_true') 41 | 42 | parser.add_argument('remote_command', help='lemma -- remote_command',nargs=REMAINDER) 43 | args = parser.parse_args() 44 | 45 | return args, parser 46 | 47 | def validate_args(settings): 48 | parser = settings.parser 49 | args = settings.args 50 | if len(sys.argv) == 1: 51 | parser.print_help() 52 | sys.stdout.write('\n') 53 | tools(settings) 54 | sys.exit(1) 55 | 56 | if args.tools: 57 | tools(settings) 58 | sys.exit(0) 59 | 60 | # validate that -d and -p are not used together 61 | if args.div_stdin and args.per_stdin: 62 | print('error: -d and -p cannot be used together', file=sys.stderr) 63 | sys.exit(1) 64 | 65 | if args.div_stdin and args.omit_stdin: 66 | print('error: -d and -o cannot be used together', file=sys.stderr) 67 | sys.exit(1) 68 | 69 | # args.div_stdin must be a non-zero positive integer 70 | if args.div_stdin: 71 | try: 72 | if int(args.div_stdin) <= 0: 73 | raise ValueError 74 | except: 75 | print('error: -d must be a non-zero positive integer', file=sys.stderr) 76 | sys.exit(1) 77 | 78 | class Settings: 79 | def __init__(self): 80 | # Parse cli arguments 81 | args, parser = get_args() 82 | 83 | self.config = ConfigParser() 84 | self._load_config() 85 | 86 | if args.lambda_url: 87 | self.ask_config() 88 | 89 | def ask_config(self): 90 | self.lambda_url = input('Please enter the URL of the Lambda service: ') 91 | 92 | def _load_config(self): 93 | newconfig = False 94 | config_dir_path = os.path.expanduser('~/.lemma') 95 | config_file_path = os.path.join(config_dir_path, 'lemma.ini') 96 | 97 | # check if config_dir_path exists 98 | if not os.path.exists(config_file_path): 99 | newconfig = True 100 | 101 | # Ensure the directory exists 102 | os.makedirs(config_dir_path, exist_ok=True) 103 | 104 | # Ensure the file exists 105 | open(config_file_path, 'a').close() 106 | 107 | self.config.read([config_file_path]) 108 | 109 | if newconfig: 110 | print('Welcome to Lemma! we could not find a configuration file, so lets create one for you.') 111 | self.ask_config() 112 | 113 | def _save_config(self): 114 | config_dir_path = os.path.expanduser('~/.lemma') 115 | config_file_path = os.path.join(config_dir_path, 'lemma.ini') 116 | 117 | with open(config_file_path, 'w') as configfile: 118 | self.config.write(configfile) 119 | 120 | @property 121 | def args(self): 122 | args, _ = get_args() 123 | return args 124 | 125 | @property 126 | def parser(self): 127 | _, parser = get_args() 128 | return parser 129 | 130 | @property 131 | def remote_command(self): 132 | remote_command = '--'.join(((' '.join(self.args.remote_command)).split('--')[1:])) 133 | return remote_command.strip() 134 | 135 | @property 136 | def lambda_url(self): 137 | lurl = self.config.get('DEFAULT', 'lambda_url', fallback=None) 138 | 139 | if lurl is None: 140 | # errpr no lambda url 141 | print('error: no lambda url specified', file=sys.stderr) 142 | 143 | # parse the url and get the hostname 144 | parsed = urllib.parse.urlparse(lurl) 145 | return parsed.scheme + '://' + parsed.netloc + '/' 146 | 147 | @property 148 | def lambda_key(self): 149 | lurl = self.config.get('DEFAULT', 'lambda_url', fallback=None) 150 | 151 | if lurl is None: 152 | # errpr no lambda url 153 | print('error: no lambda url specified', file=sys.stderr) 154 | 155 | # parse the url and get the query variable named "key" 156 | parsed = urllib.parse.urlparse(lurl) 157 | query = urllib.parse.parse_qs(parsed.query) 158 | return query.get('key', [''])[0] 159 | 160 | 161 | @lambda_url.setter 162 | def lambda_url(self, value): 163 | self.config.set('DEFAULT', 'lambda_url', value) 164 | self._save_config() 165 | 166 | @property 167 | def stdin_pipe_exists(self) -> bool: 168 | return not sys.stdin.isatty() 169 | 170 | 171 | 172 | @lru_cache(maxsize=1) 173 | def get_settings()->Settings: 174 | s = Settings() 175 | validate_args(s) 176 | return s 177 | 178 | @lru_cache(maxsize=1) 179 | def get_profiler(): 180 | return cProfile.Profile() -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup( 3 | name='lemmacli', 4 | version='0.1.0', 5 | packages=['lemma'], 6 | description='Run commandline tools on AWS Lambda', 7 | url='https://github.com/defparam/lemma', 8 | author='defparam', 9 | license='Apache 2.0', 10 | include_package_data=True, 11 | install_requires=[ 12 | "requests", 13 | ], 14 | entry_points={ 15 | 'console_scripts': [ 16 | 'lemma=lemma.__main__:main', 17 | ], 18 | }, 19 | classifiers=[ 20 | 'Programming Language :: Python :: 3', 21 | 'Operating System :: OS Independent', 22 | ], 23 | python_requires='>=3.6', 24 | ) 25 | -------------------------------------------------------------------------------- /templates/samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default.deploy.parameters] 3 | stack_name = "lemma" 4 | resolve_s3 = true 5 | s3_prefix = "lemma" 6 | region = "%REGION%" 7 | capabilities = "CAPABILITY_IAM" 8 | image_repositories = [] 9 | -------------------------------------------------------------------------------- /templates/template_arm64.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | Lemma Lambda Function 5 | 6 | Globals: 7 | Function: 8 | Timeout: %TIMEOUT% 9 | 10 | Resources: 11 | LemmaFunction: 12 | Type: AWS::Serverless::Function 13 | Properties: 14 | CodeUri: app/ 15 | Handler: run.sh 16 | Runtime: python3.12 17 | MemorySize: %MEMORY% 18 | Architectures: 19 | - arm64 20 | Environment: 21 | Variables: 22 | AWS_LAMBDA_EXEC_WRAPPER: /opt/bootstrap 23 | AWS_LWA_INVOKE_MODE: response_stream 24 | LEMMA_API_KEY: %API_KEY% 25 | LEMMA_TIMEOUT: %TIMEOUT% 26 | HOME: /tmp 27 | PORT: 8000 28 | Layers: 29 | - !Sub arn:aws:lambda:${AWS::Region}:753240598075:layer:LambdaAdapterLayerArm64:22 30 | FunctionUrlConfig: 31 | AuthType: NONE 32 | InvokeMode: RESPONSE_STREAM 33 | 34 | Outputs: 35 | LemmaFunctionUrl: 36 | Description: "Lemma Lambda Function URL" 37 | Value: !Sub "LEMMA_URL:${LemmaFunctionUrl.FunctionUrl}" 38 | LemmaFunction: 39 | Description: "Lemma Lambda Function ARN" 40 | Value: !GetAtt LemmaFunction.Arn 41 | -------------------------------------------------------------------------------- /templates/template_x86.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | Lemma Lambda Function 5 | 6 | Globals: 7 | Function: 8 | Timeout: %TIMEOUT% 9 | 10 | Resources: 11 | LemmaFunction: 12 | Type: AWS::Serverless::Function 13 | Properties: 14 | CodeUri: app/ 15 | Handler: run.sh 16 | Runtime: python3.12 17 | MemorySize: %MEMORY% 18 | Architectures: 19 | - x86_64 20 | Environment: 21 | Variables: 22 | AWS_LAMBDA_EXEC_WRAPPER: /opt/bootstrap 23 | AWS_LWA_INVOKE_MODE: response_stream 24 | LEMMA_API_KEY: %API_KEY% 25 | LEMMA_TIMEOUT: %TIMEOUT% 26 | HOME: /tmp 27 | PORT: 8000 28 | Layers: 29 | - !Sub arn:aws:lambda:${AWS::Region}:753240598075:layer:LambdaAdapterLayerX86:22 30 | FunctionUrlConfig: 31 | AuthType: NONE 32 | InvokeMode: RESPONSE_STREAM 33 | 34 | Outputs: 35 | LemmaFunctionUrl: 36 | Description: "Lemma Lambda Function URL" 37 | Value: !Sub "LEMMA_URL:${LemmaFunctionUrl.FunctionUrl}" 38 | LemmaFunction: 39 | Description: "Lemma Lambda Function ARN" 40 | Value: !GetAtt LemmaFunction.Arn 41 | -------------------------------------------------------------------------------- /tools/bin/download: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import urllib.request 4 | 5 | def download_file(url, destination): 6 | try: 7 | urllib.request.urlretrieve(url, destination) 8 | except Exception as e: 9 | print(f"An error occurred: {e}", file=sys.stderr) 10 | 11 | if __name__ == "__main__": 12 | if len(sys.argv) != 3: 13 | print("Usage: python download.py ", file=sys.stderr) 14 | sys.exit(1) 15 | 16 | url = sys.argv[1] 17 | destination = sys.argv[2] 18 | 19 | download_file(url, destination) 20 | -------------------------------------------------------------------------------- /tools/config/.gau.toml: -------------------------------------------------------------------------------- 1 | threads = 2 2 | verbose = false 3 | retries = 15 4 | subdomains = false 5 | parameters = false 6 | providers = ["wayback","commoncrawl","otx","urlscan"] 7 | blacklist = ["ttf","woff","svg","png","jpg"] 8 | json = false 9 | 10 | [urlscan] 11 | apikey = "" 12 | 13 | [filters] 14 | from = "" 15 | to = "" 16 | matchstatuscodes = [] 17 | matchmimetypes = [] 18 | filterstatuscodes = [] 19 | filtermimetypes = ["image/png", "image/jpg", "image/svg+xml"] 20 | -------------------------------------------------------------------------------- /tools/config/dirsearch.ini: -------------------------------------------------------------------------------- 1 | [general] 2 | threads = 25 3 | recursive = False 4 | deep-recursive = False 5 | force-recursive = False 6 | recursion-status = 200-399,401,403 7 | max-recursion-depth = 0 8 | exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/ 9 | random-user-agents = False 10 | max-time = 0 11 | exit-on-error = False 12 | # subdirs = /,api/ 13 | # include-status = 200-299,401 14 | # exclude-status = 400,500-999 15 | # exclude-sizes = 0b,123gb 16 | # exclude-text = "Not found" 17 | # exclude-regex = "^403$" 18 | # exclude-redirect = "*/error.html" 19 | # exclude-response = 404.html 20 | # skip-on-status = 429,999 21 | 22 | [dictionary] 23 | default-extensions = php,aspx,jsp,html,js 24 | force-extensions = False 25 | overwrite-extensions = False 26 | lowercase = False 27 | uppercase = False 28 | capitalization = False 29 | # exclude-extensions = old,log 30 | # prefixes = .,admin 31 | # suffixes = ~,.bak 32 | # wordlists = /path/to/wordlist1.txt,/path/to/wordlist2.txt 33 | 34 | [request] 35 | http-method = get 36 | follow-redirects = False 37 | # headers-file = /path/to/headers.txt 38 | # user-agent = MyUserAgent 39 | # cookie = SESSIONID=123 40 | 41 | [connection] 42 | timeout = 7.5 43 | delay = 0 44 | max-rate = 0 45 | max-retries = 1 46 | ## By disabling `scheme` variable, dirsearch will automatically identify the URI scheme 47 | # scheme = http 48 | # proxy = localhost:8080 49 | # proxy-file = /path/to/proxies.txt 50 | # replay-proxy = localhost:8000 51 | 52 | [advanced] 53 | crawl = False 54 | 55 | [view] 56 | full-url = False 57 | quiet-mode = False 58 | color = True 59 | show-redirects-history = False 60 | 61 | [output] 62 | ## Support: plain, simple, json, xml, md, csv, html, sqlite 63 | report-format = plain 64 | autosave-report = False 65 | autosave-report-folder = /tmp 66 | # log-file = /path/to/dirsearch.log 67 | # log-file-size = 50000000 -------------------------------------------------------------------------------- /tools/config/test.txt: -------------------------------------------------------------------------------- 1 | 11111111111111111 2 | 22222222222222222 3 | 33333333333333333 4 | 44444444444444444 5 | 55555555555555555 6 | 66666666666666666 7 | 77777777777777777 8 | 88888888888888888 9 | 99999999999999999 10 | 00000000000000000 11 | -------------------------------------------------------------------------------- /tools/demo: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import time 4 | 5 | inp = ' '.join(sys.argv[1:]) 6 | 7 | if (len(inp.strip()) == 0): 8 | inp = "Hello World! I am executing on AWS Lambda. This is a demo of response streaming." 9 | 10 | sys.stdout.write("\x1b[42m") 11 | for char in inp: 12 | sys.stdout.write(char) 13 | sys.stdout.flush() 14 | time.sleep(0.02) 15 | 16 | sys.stdout.write("\x1b[0m") 17 | 18 | sys.stdout.write("\n") 19 | sys.stdout.flush() 20 | -------------------------------------------------------------------------------- /tools/ffuf: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export XDG_DATA_HOME=/tmp/.local/share 4 | export XDG_CONFIG_HOME=/tmp/.config 5 | export XDG_CACHE_HOME=/tmp/.cache 6 | 7 | # get the current directory of where this script is located 8 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 9 | 10 | # run ffuf with the new environment variables and the provided arguments 11 | $DIR/bin/ffuf "$@" -------------------------------------------------------------------------------- /tools/gau: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cp -f ./tools/config/.gau.toml /tmp/.gau.toml 4 | ./tools/bin/gau $@ -------------------------------------------------------------------------------- /tools/install_tools.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Function to display usage message 4 | usage() { 5 | echo "Usage: $0 " 6 | echo "arch must be either x86_64 or arm64" 7 | exit 1 8 | } 9 | 10 | # Check if argument is provided 11 | if [ -z "$1" ]; then 12 | echo "Error: No architecture specified." 13 | usage 14 | fi 15 | 16 | # Check if the argument is valid 17 | if [ "$1" != "x86_64" ] && [ "$1" != "arm64" ]; then 18 | echo "Error: Invalid architecture specified." 19 | usage 20 | fi 21 | 22 | # If the argument is valid, proceed with the script 23 | arch="$1" 24 | echo "Architecture specified: $arch" 25 | #cd .. 26 | rm -rf ./app/tools 27 | rm -f ./app/tool_requirements.txt 28 | cp -rf ./tools ./app/ 29 | mkdir -p ./app/tools/wordlists 30 | touch ./app/tool_requirements.txt 31 | 32 | if [ "$arch" == "x86_64" ]; then 33 | 34 | echo "Installing ffuf..." 35 | tmpdir=$(mktemp -d) 36 | wget https://github.com/ffuf/ffuf/releases/download/v2.1.0/ffuf_2.1.0_linux_amd64.tar.gz -O $tmpdir/ffuf.tar.gz > /dev/null 2>&1 37 | tar -xvf $tmpdir/ffuf.tar.gz -C $tmpdir > /dev/null 2>&1 38 | mv $tmpdir/ffuf ./app/tools/bin/ 39 | rm -rf $tmpdir 40 | 41 | echo "Installing httpx..." 42 | tmpdir=$(mktemp -d) 43 | wget https://github.com/projectdiscovery/httpx/releases/download/v1.6.5/httpx_1.6.5_linux_amd64.zip -O $tmpdir/httpx.zip > /dev/null 2>&1 44 | unzip $tmpdir/httpx.zip -d $tmpdir > /dev/null 2>&1 45 | mv $tmpdir/httpx ./app/tools 46 | rm -rf $tmpdir 47 | 48 | echo "Installing gau..." 49 | tmpdir=$(mktemp -d) 50 | wget https://github.com/lc/gau/releases/download/v2.2.3/gau_2.2.3_linux_amd64.tar.gz -O $tmpdir/gau.tar.gz > /dev/null 2>&1 51 | tar -xvf $tmpdir/gau.tar.gz -C $tmpdir > /dev/null 2>&1 52 | mv $tmpdir/gau ./app/tools/bin/ 53 | rm -rf $tmpdir 54 | 55 | echo "Installing subfinder..." 56 | tmpdir=$(mktemp -d) 57 | wget https://github.com/projectdiscovery/subfinder/releases/download/v2.6.6/subfinder_2.6.6_linux_amd64.zip -O $tmpdir/subfinder.zip > /dev/null 2>&1 58 | unzip $tmpdir/subfinder.zip -d $tmpdir > /dev/null 2>&1 59 | mv $tmpdir/subfinder ./app/tools 60 | rm -rf $tmpdir 61 | 62 | echo "Installing dnsx..." 63 | tmpdir=$(mktemp -d) 64 | wget https://github.com/projectdiscovery/dnsx/releases/download/v1.2.1/dnsx_1.2.1_linux_amd64.zip -O $tmpdir/dnsx.zip > /dev/null 2>&1 65 | unzip $tmpdir/dnsx.zip -d $tmpdir > /dev/null 2>&1 66 | mv $tmpdir/dnsx ./app/tools 67 | rm -rf $tmpdir 68 | 69 | echo "Installing nuclei..." 70 | tmpdir=$(mktemp -d) 71 | wget https://github.com/projectdiscovery/nuclei/releases/download/v3.2.9/nuclei_3.2.9_linux_amd64.zip -O $tmpdir/nuclei.zip > /dev/null 2>&1 72 | unzip $tmpdir/nuclei.zip -d $tmpdir > /dev/null 2>&1 73 | mv $tmpdir/nuclei ./app/tools/bin 74 | rm -rf $tmpdir 75 | 76 | echo "Installing katana..." 77 | tmpdir=$(mktemp -d) 78 | wget https://github.com/projectdiscovery/katana/releases/download/v1.1.0/katana_1.1.0_linux_amd64.zip -O $tmpdir/katana.zip > /dev/null 2>&1 79 | unzip $tmpdir/katana.zip -d $tmpdir > /dev/null 2>&1 80 | mv $tmpdir/katana ./app/tools 81 | rm -rf $tmpdir 82 | 83 | echo "Installing shortscan..." 84 | git clone https://github.com/bitquark/shortscan.git > /dev/null 2>&1 85 | cd shortscan 86 | go mod tidy > /dev/null 2>&1 87 | GOARCH=amd64 go build -o ../app/tools/shortscan ./cmd/shortscan > /dev/null 2>&1 88 | cd .. 89 | rm -rf shortscan 90 | 91 | tmpdir=$(mktemp -d) 92 | wget http://ftp.us.debian.org/debian/pool/main/b/busybox/busybox_1.30.1-4_amd64.deb -O $tmpdir/busybox.deb > /dev/null 2>&1 93 | dpkg -x $tmpdir/busybox.deb $tmpdir > /dev/null 2>&1 94 | mv $tmpdir/bin/busybox ./app/tools/bin/ 95 | rm -rf $tmpdir 96 | 97 | elif [ "$arch" == "arm64" ]; then 98 | 99 | echo "Installing ffuf..." 100 | tmpdir=$(mktemp -d) 101 | wget https://github.com/ffuf/ffuf/releases/download/v2.1.0/ffuf_2.1.0_linux_arm64.tar.gz -O $tmpdir/ffuf.tar.gz > /dev/null 2>&1 102 | tar -xvf $tmpdir/ffuf.tar.gz -C $tmpdir > /dev/null 2>&1 103 | mv $tmpdir/ffuf ./app/tools/bin/ 104 | rm -rf $tmpdir 105 | 106 | echo "Installing httpx..." 107 | tmpdir=$(mktemp -d) 108 | wget https://github.com/projectdiscovery/httpx/releases/download/v1.6.5/httpx_1.6.5_linux_arm64.zip -O $tmpdir/httpx.zip > /dev/null 2>&1 109 | unzip $tmpdir/httpx.zip -d $tmpdir > /dev/null 2>&1 110 | mv $tmpdir/httpx ./app/tools 111 | rm -rf $tmpdir 112 | 113 | echo "Installing gau..." 114 | tmpdir=$(mktemp -d) 115 | wget https://github.com/lc/gau/releases/download/v2.2.3/gau_2.2.3_linux_arm64.tar.gz -O $tmpdir/gau.tar.gz > /dev/null 2>&1 116 | tar -xvf $tmpdir/gau.tar.gz -C $tmpdir > /dev/null 2>&1 117 | mv $tmpdir/gau ./app/tools/bin/ 118 | rm -rf $tmpdir 119 | 120 | echo "Installing subfinder..." 121 | tmpdir=$(mktemp -d) 122 | wget https://github.com/projectdiscovery/subfinder/releases/download/v2.6.6/subfinder_2.6.6_linux_arm64.zip -O $tmpdir/subfinder.zip > /dev/null 2>&1 123 | unzip $tmpdir/subfinder.zip -d $tmpdir > /dev/null 2>&1 124 | mv $tmpdir/subfinder ./app/tools 125 | rm -rf $tmpdir 126 | 127 | echo "Installing dnsx..." 128 | tmpdir=$(mktemp -d) 129 | wget https://github.com/projectdiscovery/dnsx/releases/download/v1.2.1/dnsx_1.2.1_linux_arm64.zip -O $tmpdir/dnsx.zip > /dev/null 2>&1 130 | unzip $tmpdir/dnsx.zip -d $tmpdir > /dev/null 2>&1 131 | mv $tmpdir/dnsx ./app/tools 132 | rm -rf $tmpdir 133 | 134 | echo "Installing nuclei..." 135 | tmpdir=$(mktemp -d) 136 | wget https://github.com/projectdiscovery/nuclei/releases/download/v3.2.9/nuclei_3.2.9_linux_arm64.zip -O $tmpdir/nuclei.zip > /dev/null 2>&1 137 | unzip $tmpdir/nuclei.zip -d $tmpdir > /dev/null 2>&1 138 | mv $tmpdir/nuclei ./app/tools/bin 139 | rm -rf $tmpdir 140 | 141 | echo "Installing katana..." 142 | tmpdir=$(mktemp -d) 143 | wget https://github.com/projectdiscovery/katana/releases/download/v1.1.0/katana_1.1.0_linux_arm64.zip -O $tmpdir/katana.zip > /dev/null 2>&1 144 | unzip $tmpdir/katana.zip -d $tmpdir > /dev/null 2>&1 145 | mv $tmpdir/katana ./app/tools 146 | rm -rf $tmpdir 147 | 148 | echo "Installing shortscan..." 149 | git clone https://github.com/bitquark/shortscan.git > /dev/null 2>&1 150 | cd shortscan 151 | go mod tidy > /dev/null 2>&1 152 | GOARCH=arm64 go build -o ../app/tools/shortscan ./cmd/shortscan > /dev/null 2>&1 153 | cd .. 154 | rm -rf shortscan 155 | 156 | tmpdir=$(mktemp -d) 157 | wget http://ftp.us.debian.org/debian/pool/main/b/busybox/busybox_1.30.1-4_arm64.deb -O $tmpdir/busybox.deb > /dev/null 2>&1 158 | dpkg -x $tmpdir/busybox.deb $tmpdir > /dev/null 2>&1 159 | mv $tmpdir/bin/busybox ./app/tools/bin/ 160 | rm -rf $tmpdir 161 | 162 | fi 163 | 164 | echo "Installing smuggler..." 165 | git clone https://github.com/defparam/smuggler ./app/tools/bin/smuggler > /dev/null 2>&1 166 | 167 | echo "Installing SecLists's common.txt wordlist..." 168 | wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/common.txt -O ./app/tools/wordlists/common.txt > /dev/null 2>&1 169 | 170 | rm -rf ./app/tools/install_tools.sh 171 | 172 | 173 | -------------------------------------------------------------------------------- /tools/nuclei: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | 4 | # Disable download from the default nuclei-templates project 5 | export DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD=true 6 | 7 | # Disable download from public / private GitHub project(s) 8 | export DISABLE_NUCLEI_TEMPLATES_GITHUB_DOWNLOAD=true 9 | 10 | # Disable download from public / private GitLab project(s) 11 | export DISABLE_NUCLEI_TEMPLATES_GITLAB_DOWNLOAD=true 12 | 13 | # Disable download from public / private AWS Bucket(s) 14 | export DISABLE_NUCLEI_TEMPLATES_AWS_DOWNLOAD=true 15 | 16 | # Disable download from public / private Azure Blob Storage 17 | export DISABLE_NUCLEI_TEMPLATES_AZURE_DOWNLOAD=true 18 | 19 | 20 | $DIR/bin/download https://github.com/projectdiscovery/nuclei-templates/archive/refs/tags/v9.9.1.zip /tmp/nuclei-templates.zip > /dev/null 2>&1 21 | $DIR/bin/busybox unzip -d /tmp /tmp/nuclei-templates.zip > /dev/null 2>&1 22 | 23 | # This will create a /tmp/nuclei-templates-X.X.X directory, change it to /tmp/nuclei-templates 24 | mv /tmp/nuclei-templates-* /tmp/nuclei-templates > /dev/null 2>&1 25 | $DIR/bin/nuclei $@ -------------------------------------------------------------------------------- /tools/runner: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | $@ -------------------------------------------------------------------------------- /tools/smuggler: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | /usr/bin/env python3 $DIR/bin/smuggler/smuggler.py $@ --------------------------------------------------------------------------------