├── .gitignore ├── CONTRIBUTING.md ├── README.md ├── docker-compose-mongo-only.yml ├── elasticpwn ├── .gitignore ├── docker-compose.localdev.yml ├── elasticpwn.code-workspace ├── go.mod ├── go.sum ├── localdev.Dockerfile ├── lookup-addrs │ ├── .gitignore │ ├── README.md │ ├── build-binary.sh │ ├── lookup-addrs.go │ └── utils.go ├── main.go ├── plugins │ ├── elastic-util.go │ ├── elasticsearch-plugin.go │ ├── kibana-plugin.go │ ├── plugin.go │ ├── report-generate-plugin.go │ └── report-view-plugin.go └── util │ ├── concurrency.go │ ├── file.go │ ├── http.go │ ├── log.go │ ├── mongo.go │ ├── regex.go │ ├── regex_test.go │ ├── util.go │ └── validators.go ├── report ├── build-frontend.sh ├── dev.sh ├── elasticpwn-backend │ ├── .env.example │ ├── .gitignore │ ├── go.mod │ ├── go.sum │ ├── handlers.go │ └── main.go └── frontend │ ├── .env.local.example │ ├── .eslintrc.json │ ├── .gitignore │ ├── .nvmrc │ ├── README.md │ ├── components │ ├── layout │ │ └── layout.tsx │ ├── quickActionButtons │ │ └── quickActionButtons.tsx │ └── spacer │ │ └── spacer.tsx │ ├── config │ └── env.ts │ ├── global.d.ts │ ├── hooks │ ├── useOnReviewedAndOnPass.ts │ └── usePersistentDatabaseServerAvailableStatus.ts │ ├── next-env.d.ts │ ├── next.config.js │ ├── package-lock.json │ ├── package.json │ ├── pages │ ├── _app.tsx │ ├── index.tsx │ └── reports │ │ └── [id].tsx │ ├── public │ ├── favicon.ico │ └── vercel.svg │ ├── styles │ ├── Home.module.css │ ├── fragments.ts │ └── globals.css │ ├── templates │ └── localFragments │ │ ├── Indices.tsx │ │ └── PreInfo.tsx │ ├── tsconfig.json │ ├── types │ └── elastic.ts │ └── util │ ├── api.ts │ ├── filesize.ts │ ├── localStorage.ts │ ├── mongo.ts │ ├── react-essentials.tsx │ └── string.ts └── screenshots ├── REPORT_ROOT_SCREENSHOT.png ├── REPORT_SCREENSHOT.png └── WORKSPACE_INSTRUCTION.png /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | elasticsearch-sample-input.txt 3 | node_modules 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | data/dump* 20 | data 21 | history.csv 22 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | As always, PRs are welcome. 2 | 3 | # Spirit 4 | Please bear this spirit in mind when thinking of a fix/new feature proposal/etc: this tool is intended for a user who wants to _analyse_ large OSINT data via semi-automated means. This is the very reason why, unlike other tools, it is designed to request detailed information as much as possible (for example, requesting POST `index_name/_search` for each index). Therefore, please help craft this tool that way when contributing. 5 | 6 | # Structure 7 | `/elasticpwn` contains separate folders: 8 | - `/lookup-addrs`: provides utilities for getting info about an URL. This package can be built as a standalone module as well. 9 | - `/util`: contains shared, commonly used tools across packages 10 | - `/plugins`: this is where all plugins reside in. Any new plugins should be built in this folder too. Plugins will be exported and be used in `/elasticpwn`. Plugins should be abstracted well and should only expose as few public methods as possible. 11 | 12 | ## kibana/elasticsearch plugin 13 | - If you don't know already, elasticsearch is a tool for collecting, analazying and viewing large data, and kibana is the frontend of it. 14 | - Sometimes, an IP would have an elasticsearch but kibana port open. In other occassions, it could be reverse. Otherwise, it could have both open. This is the reason that both plugins are needed. 15 | - Kibana's got dev tools page, and it allows user to send a request to elasticsearch via proxy. So we are using that proxy API. 16 | - Elasticsearch is more straightforward; it has [official API documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/rest-apis.html). There is [an official Golang client](https://github.com/elastic/go-elasticsearch) for elasticsearch, but it's not used here because all we need to do for this tool is to query very few API endpoints and it takes not too much effort to do that. 17 | - Elasticsearch can have many useless indices. These are listed in `elastic-util.go` and automatically filtered out when querying indices. 18 | 19 | # Code 20 | This is my first project using GoLang. It's very possible that I've made stupid mistakes. Please help fix them. 21 | 22 | # How to get started on developing locally 23 | 24 | ``` 25 | $ git clone https://github.com/9oelM/elasticpwn.git 26 | 27 | $ cd elasticpwn 28 | 29 | $ go version # at least 1.17 30 | go version go1.17.2 linux/amd64 31 | 32 | $ go mod tidy 33 | 34 | # alternatively, you can use any other tools that can gather data about it. For example, Google dork. 35 | $ shodan download --limit 300 elasticsearch-sample elasticsearch # you may need to add more keywords to easily find 'open' elasticsearch instances 36 | 37 | $ shodan parse --fields ip_str,port --separator : elasticsearch-sample.gz > elasticsearch-sample.txt 38 | 39 | $ go build -v 40 | 41 | # input file must be a list of URLs pointing to a elasticsearch|kibana instances. For example: 42 | # 123.123.123.123:9200 43 | # 4.5.6.7:9200 44 | # and so on. 45 | $ ./elasticpwn elasticsearch -f elasticsearch-sample.txt -t 12 -of elasticsearch-sample.json -om json 46 | 47 | # or use mongodb to store result 48 | $ ./elasticpwn elasticsearch -f elasticsearch-sample.txt -murl mongodb://root:example@172.17.0.1:27017/ -t 12 -om mongo 49 | ``` 50 | 51 | # Debug inside Docker container 52 | Sometimes you may wanna dive into docker container to inspect mongo directly. You may want to use these commands below in that case. 53 | 54 | ``` 55 | docker container ls # find which container is responsible for mongodb 56 | 57 | docker exec -it [container-id] /bin/bash # enter docker container shell 58 | 59 | mongo -host "mongodb://root:example@mongo:27017/" -u root -p example # login to console 60 | ``` 61 | Alternatively, you could use a tool like DataGrip to query from a separate program. 62 | 63 | # IDE 64 | Visual Studio Code is highly recommended. [Note that gopls does not support multi-package environment without any config](https://github.com/golang/go/issues/32394). [Please open a workspace on VSCode and add respective package folder (lookup-addrs, main, plugins, ...) to the workspace](https://github.com/golang/go/issues/32394#issuecomment-498385140). Therefore, find `elasticpwn/elasticpwn.code-workspace` file in this repository, and then open that workspace on Visual Studio Code to avoid any weird errors: 65 | 66 | ![WORKSPACE_INSTRUCTION.png](./screenshots/WORKSPACE_INSTRUCTION.png) 67 | 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elasticpwn 2 | 3 | Quickly collects data from exposed Elasticsearch or Kibana instances and generates a report to be reviewed. It mainly aims for sensitive data exposure and content discovery. 4 | 5 | # Table of contents 6 | * [elasticpwn](#elasticpwn) 7 | * [Table of contents](#table-of-contents) 8 | * [Rationale](#rationale) 9 | * [Collecting data](#collecting-data) 10 | * [Generating a report](#generating-a-report) 11 | * [Generating a report with a persistent MongoDB database](#generating-a-report-with-a-persistent-mongodb-database) 12 | * [Generating a report without a persistent MongoDB database](#generating-a-report-without-a-persistent-mongodb-database) 13 | * [Notes about performance](#notes-about-performance) 14 | * [Threads (-t option)](#threads--t-option) 15 | * [Maximum number of indices to request (-max-i option)](#maximum-number-of-indices-to-request--max-i-option) 16 | * [Maximum size of an index to request (-max-is option)](#maximum-size-of-an-index-to-request--max-is-option) 17 | * [Generating a report with many pages](#generating-a-report-with-many-pages) 18 | * [Full CLI reference](#full-cli-reference) 19 | * [elasticpwn](#elasticpwn-1) 20 | * [elasticpwn-backend](#elasticpwn-backend) 21 | * [Contributing](#contributing) 22 | 23 | # Rationale 24 | 25 | Why do you need this project? If you are hunting for sensitive data exposure or content discovery, Elasticsearch and Kibana could be your target, because they do not have an authentication turned on by default and there are thousands of exposed instances on the Internet. But as you try to look into the data acquired from these instances, you need a lot of ad-hoc data processing and manual work to find out sensitive pieces of data. To be able to easily do that, we need some neat way to collect data, process it, and review them all at once. This is what `elasticpwn` tries to achieve. 26 | 27 | This project is different from other similar OSS projects in that they actually request detailed indices info by using **[`_search` APIs](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html).** This greatly reduces the burden of visiting each instance manually and querying the data of an index that looks interesting. Also, it automatically ignores data that looks quite useless, because many open Elasticsearch and Kibana instances may have sample/boilerplate data which are just useless to inspect. The list of this should be constantly updated. [More details in CONTRIBUTING.md](CONTRIBUTING.md). 28 | 29 | # Collecting data 30 | 1. You need `go^1.17` to install `elasticpwn`. 31 | 1. Install `elasticpwn`: 32 | ```bash 33 | go install github.com/9oelM/elasticpwn/elasticpwn@latest 34 | ``` 35 | 1. You will need to have a list of URLs to try against. Get them from OSINT platforms like [shodan.io](https://shodan.io) or [binaryedge.io](https://binaryedge.io). Download them and put them into a text file, with each ip and port written as follows: 36 | ``` 37 | 123.123.123.123:80 38 | 124.124.124.124:443 39 | ... and so on. 40 | ``` 41 | 42 | Various types of inputs are accepted. All of the formats below are supported. Just make sure that you don't have any empty lines in the file, as this case is not explicitly handled and is still on the todo list. 43 | 44 | ``` 45 | https://123.123.123.123 46 | http://123.123.123.123 47 | http://123.123.123.123:80 48 | http://123.123.123.123:443 49 | http://123.123.123.123:5601 50 | 123.123.123.123:5601 51 | ``` 52 | 53 | `elasticpwn` will also retry with `http` if the server explicitly responds with the error message containing `server gave HTTP response to HTTPS client`, and `https` if the server explicitly responds with the error message containing `server gave HTTPS response to HTTP client`. 54 | 1. Think about which option to choose for storing output. If you are collecting a really large sum of data, from say, 2000 instances of kibana, then you would probably need to use `mongo` instead of `json`, because `mongo` is the only option where you could generate a report. Otherwise, choose `json` or `plain`. If you will use `mongo`, preferrably launch a local mongodb instance. You can easily do it by using docker-compose file provided at the root of this repository: `curl https://raw.githubusercontent.com/9oelM/elasticpwn/main/docker-compose-mongo-only.yml -o docker-compose-mongo-only.yml && docker-compose -f docker-compose-mongo-only.yml up -d` 55 | 1. Run `elasticpwn` and wait for the data collection to be finished. For example: 56 | ``` 57 | elasticpwn elasticsearch -f list_of_elasticsearch_instances.txt -murl mongodb://root:example@172.17.0.1:27017/ -of mongo -t 12 58 | ``` 59 | if you are not going for mongo, then it would be something like: 60 | ``` 61 | elasticpwn elasticsearch -f list_of_elasticsearch_instances.txt -of json -t 12 62 | ``` 63 | 1. After it is finished, check data is properly collected. 64 | 65 | # Generating a report 66 | 67 | **Generating a report expects pre-installation of `npm` and `node`**. 68 | 69 | Reading the report is a convenient way to review data obtained from the search done by the CLI. The report contains data not limited to: 70 | - URL of the elasticsearch/kibana instance 71 | - indices 72 | - detailed info about each index (POST `//_search` result) 73 | - interesting information (may be relevant to sensitive information disclosure) 74 | - nodes 75 | - allocations 76 | - aliases 77 | 78 | Currently, elasticsearch and kibana plugins do not collect any more data than this despite the ability to do so, because other data do not usually include sensitive information. If you think this is not the case, please open an issue. 79 | 80 | Due to performance reasons, generating an report is only possible by querying data from mongodb. **JSON backend is not supported.** 81 | 82 | This is a little preview of how a report will look like: 83 | 84 | ![report root](./screenshots/REPORT_ROOT_SCREENSHOT.png) 85 | ![report](./screenshots/REPORT_SCREENSHOT.png) 86 | 87 | # Generating a report with a persistent MongoDB database 88 | `elasticpwn` offers a way to store the history of the reviewed IPs in a database, so that you will know you don't have to review them again in the future if you encounter them. This is different from the database that data collected from elasticsearch/kibana instances are put into. 89 | 90 | 1. Generate the report. This will create a directory called `report` under CWD: 91 | ```bash 92 | # -cn should be either elasticsearch or kibana 93 | # -murl should be where the collected data is stored at 94 | # -dn should be the url where elasticpwn-backend will be hosted 95 | elasticpwn report generate -cn elasticsearch -murl mongodb://root:example@172.17.0.1:27017/ -dn http://localhost:9292 96 | ``` 97 | 1. To connect to a persistent MongoDB database (you will store the list of all IPs you reviewed), install `elasticpwn-backend`: 98 | ``` 99 | go install github.com/9oelM/elasticpwn/report/elasticpwn-backend@latest 100 | ``` 101 | 1. Set up a persistent MongoDB database. It could be your local datbase, or a remote one. I would highly recommend signing up for a free account for [MongoDB Atlas](https://www.mongodb.com/cloud/atlas/register) because it offers 500MB for free, and there is no chance that you will reach maximum usage just by putting the list of all IPs you reviewed (note that it won't store data from the indices and so on - only IPs are stored). 102 | 1. Get the mongodb URI from MongoDB Atlas or somewhere else you managed to use, for example: `mongodb+srv://my-mongo:mypassword@my-mongodb-name.tn3al.mongodb.net/defaultcollectionname?retryWrites=true&w=majority`. 103 | 1. Launch server that connects with MongoDB URI above. You can also provide values through `.env.local` file. See [`/report/backend/.env.example`](./report/backend/.env.example). Below is an example command that serves reviewed urls of elasticsearch: 104 | ```bash 105 | elasticpwn-backend -mongodbUri=mongodb+srv://username:pw@somewhere.mongodb.net/default-collection-name?retryWrites=true&w=majority -databaseCollectionName=elasticsearch_reviewed_urls -databaseName elasticpwn -port 9292 106 | ``` 107 | 1. If everything is successfully bootstrapped, you should see some logs coming up: 108 | ```bash 109 | 2021/12/28 05:38:54 Connected to database 110 | 2021/12/28 05:38:54 Inserting 111 | 2021/12/28 05:38:54 Test url already created before 112 | [GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached. 113 | 114 | [GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production. 115 | - using env: export GIN_MODE=release 116 | - using code: gin.SetMode(gin.ReleaseMode) 117 | 118 | [GIN-debug] GET /ping --> main.PingHandler (4 handlers) 119 | [GIN-debug] GET /urls --> main.GetUrlsHandlerGenerator.func1 (4 handlers) 120 | [GIN-debug] POST /urls --> main.PostUrlsHandlerGenerator.func1 (4 handlers) 121 | [GIN-debug] DELETE /urls --> main.DeleteUrlsHandlerGenerator.func1 (4 handlers) 122 | [GIN-debug] Listening and serving HTTP on 0.0.0.0:9292 123 | ``` 124 | Run `curl http://localhost:9292/urls` or `curl http://localhost:9292/ping` to check if it's running correctly. 125 | 1. With this server bootstrapped and running, view the report at the same time: 126 | ``` 127 | elasticpwn report view -d ./path-to-report -p 9999 128 | ``` 129 | Then, you will be able to see the report at http://localhost:9999. 130 | 131 | # Generating a report without a persistent MongoDB database 132 | If you are just running `elasticpwn` for once or twice, generating a report without a persistent database may be the right choice. 133 | 134 | 1. Generate the report by running the command below (After collecting data by running `elasticpwn elasticsearch|kibana`). This will create a directory called `report` under CWD: 135 | ```bash 136 | # -cn should be either elasticsearch or kibana 137 | # -murl should be where the collected data is stored at 138 | elasticpwn report generate -cn elasticsearch -murl mongodb://root:example@172.17.0.1:27017/ 139 | ``` 140 | 141 | 1. View the report: 142 | ```bash 143 | elasticpwn report view -d ./path-to-report -p 9999 144 | ``` 145 | Then, you will be able to see the report at http://localhost:9999. 146 | 147 | # Notes about performance 148 | ## Threads (`-t` option) 149 | `elasticpwn` goes through extensive regex matching work to find interesting words that may be relevant to sensitive information disclosure. Therefore it is recommended to keep the number of threads at about the number of your computer's cores (output from the command `nproc`). Otherwise, the program may crash or slow down. 150 | 151 | For example, if the number of cores of your computer is 12, keep the number of threads at about 12~24. 152 | 153 | ## Maximum number of indices to request (`-max-i` option) 154 | If this is too large, it might cause MongoDB to reject insertion of data due to its size. Stick with the default option if you are unsure. 155 | 156 | ## Maximum size of an index to request (`-max-is` option) 157 | If this is too large, it might cause MongoDB to reject insertion of data due to its size. Stick with the default option if you are unsure. This can also affect RAM and CPU usage. 158 | 159 | ## Generating a report with many pages 160 | If you are collecting data from.. say, 5000 instances, generating a report with that many pages with _javascript_ could be a bit of challenge. Next.js is used to create it (Gatsby.js failed because it could not hold this much data and would just fail due to memory shortage). If you are collecting very much of data, `elasticpwn` could expect 4GB to 8GB of vacant RAM. 161 | 162 | # Full CLI reference 163 | 164 | ## `elasticpwn` 165 | ``` 166 | Usage: elasticpwn [elasticsearch|kibana [...plugin options] or elasticpwn report [generate|view] [...plugin options] 167 | [elasticsearch] plugin options: 168 | -f string 169 | [REQUIRED] path to a file with urls (url per line) 170 | -max-i int 171 | maximum number of indices to request. 172 | If you intend to set this as a high number, make sure you've got enough storage. 173 | If you don't know what an index is, 174 | refer to elasticserach docs at https://www.elastic.co/blog/what-is-an-elasticsearch-index (default 5) 175 | -max-is int 176 | maximum size of an index to request. 177 | If you intend to set this as a high number, make sure you've got enough storage. 178 | If you don't know what 'size' is, 179 | please refer to elasticsearch docs on 180 | '/_cat/_search?size=' at 181 | https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html (default 70) 182 | -murl string 183 | [OPTIONAL] needed only when -o=mongo is selected. 184 | mongodb url with username and pw included. 185 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this. 186 | (default "mongodb://root:example@172.17.0.1:27017/") 187 | -of string 188 | [OPTIONAL] output file name. Ignored when mongo option is chosen. (default "elasticsearch.json") 189 | -om string 190 | [OPTIONAL] output mode. json|mongo|plain. 191 | mongo option will require a mongo server to be up. 192 | plain mode will output json-like object to each line finishing with a comma. 193 | For mongo, Local docker mongo instance is recommneded. (check docker-compose.yml and docs) (default "json") 194 | -t int 195 | [OPTIONAL] number of threads when running a plugin (default 8) 196 | [kibana] plugin options: 197 | -f string 198 | [REQUIRED] path to a file with urls (url per line) 199 | -max-i int 200 | maximum number of indices to request. 201 | If you intend to set this as a high number, make sure you've got enough storage. 202 | If you don't know what an index is, 203 | refer to elasticserach docs at 204 | https://www.elastic.co/blog/what-is-an-elasticsearch-index (default 5) 205 | -max-is int 206 | maximum size of an index to request. 207 | If you intend to set this as a high number, make sure you've got enough storage. 208 | If you don't know what 'size' is, please refer to elasticsearch docs on 209 | '/_cat/_search?size=' at 210 | https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html (default 70) 211 | -murl string 212 | [OPTIONAL] needed only when -o=mongo is selected. 213 | mongodb url with username and pw included. 214 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this. 215 | (default "mongodb://root:example@172.17.0.1:27017/") 216 | -of string 217 | [OPTIONAL] output file name. Ignored when mongo option is chosen. (default "kibana.json") 218 | -om string 219 | [OPTIONAL] output mode. json|mongo|plain. 220 | mongo option will require a mongo server to be up. 221 | plain mode will output json-like object to each line finishing with a comma. 222 | For mongo, local docker mongo instance is recommneded. (check docker-compose.yml and docs) (default "json") 223 | -t int 224 | [OPTIONAL] number of threads when running a plugin (default 8) 225 | [report] generate plugin options: 226 | -cn string 227 | [REQUIRED] 228 | must be elasticsearch|kibana. Collection name of the mongodb database to be used. 229 | -dn string 230 | [OPTIONAL] 231 | backend url of persistent database server being used. This should be the url where elasticpwn-backend is hosted. (default "http://localhost:9292") 232 | -murl string 233 | [OPTIONAL] 234 | mongodb url with username and pw included from which gathered data can be accessed. 235 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this. (default "mongodb://root:example@172.17.0.1:27017/") 236 | [report] view plugin options: 237 | -d string 238 | [OPTIONAL] the directory of generated report (default "./report") 239 | -p string 240 | [OPTIONAL] local port to serve report page from (default "9999") 241 | ``` 242 | 243 | ## `elasticpwn-backend` 244 | 245 | ``` 246 | Usage of elasticpwn-bakcend: 247 | -databaseCollectionName string 248 | mongodb collection name. Should be any one of: elasticsearch_reviewed_urls|elasticsearch_reviewed_urls_dev|kibana_reviewed_urls|kibana_reviewed_urls_dev 249 | -databaseName string 250 | mongodb db name 251 | -mongodbUri string 252 | mongodb URI. Example: mongodb+srv://username:pw@somewhere.mongodb.net/default-collection-name?retryWrites=true&w=majority 253 | -port string 254 | port at which the server will run. (default "9292") 255 | ``` 256 | 257 | # Contributing 258 | See [CONTRIBUTING.md](./CONTRIBUTING.md). 259 | -------------------------------------------------------------------------------- /docker-compose-mongo-only.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | 5 | mongo: 6 | image: mongo:5.0.3 7 | ports: 8 | - "27017:27017" # to connect from my laptop 9 | environment: 10 | MONGO_INITDB_ROOT_USERNAME: root 11 | MONGO_INITDB_ROOT_PASSWORD: example 12 | -------------------------------------------------------------------------------- /elasticpwn/.gitignore: -------------------------------------------------------------------------------- 1 | main -------------------------------------------------------------------------------- /elasticpwn/docker-compose.localdev.yml: -------------------------------------------------------------------------------- 1 | # this compose file is only meant to be used for local development 2 | version: '3.8' 3 | 4 | services: 5 | 6 | mongo: 7 | image: mongo:5.0.3 8 | ports: 9 | - "27017:27017" # to connect from my laptop 10 | restart: always 11 | environment: 12 | MONGO_INITDB_ROOT_USERNAME: root 13 | MONGO_INITDB_ROOT_PASSWORD: example 14 | elasticpwn: 15 | depends_on: 16 | - mongo 17 | build: 18 | context: . 19 | dockerfile: localdev.Dockerfile 20 | restart: always 21 | volumes: 22 | - ./:/go/src/app 23 | environment: 24 | ME_CONFIG_MONGODB_ADMINUSERNAME: root 25 | ME_CONFIG_MONGODB_ADMINPASSWORD: example 26 | ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/ -------------------------------------------------------------------------------- /elasticpwn/elasticpwn.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | }, 6 | ] 7 | } -------------------------------------------------------------------------------- /elasticpwn/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/9oelM/elasticpwn/elasticpwn 2 | 3 | go 1.17 4 | 5 | require go.mongodb.org/mongo-driver v1.8.2 6 | 7 | require ( 8 | github.com/go-stack/stack v1.8.0 // indirect 9 | github.com/golang/snappy v0.0.1 // indirect 10 | github.com/klauspost/compress v1.13.6 // indirect 11 | github.com/pkg/errors v0.9.1 // indirect 12 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 13 | github.com/xdg-go/scram v1.0.2 // indirect 14 | github.com/xdg-go/stringprep v1.0.2 // indirect 15 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 16 | golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect 17 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect 18 | golang.org/x/text v0.3.5 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /elasticpwn/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 5 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 6 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 7 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 8 | github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= 9 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 11 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 12 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 13 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 14 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 15 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 16 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 17 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 18 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 19 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 22 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= 24 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 25 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 26 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 27 | github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= 28 | github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= 29 | github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= 30 | github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= 31 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= 32 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 33 | go.mongodb.org/mongo-driver v1.8.2 h1:8ssUXufb90ujcIvR6MyE1SchaNj0SFxsakiZgxIyrMk= 34 | go.mongodb.org/mongo-driver v1.8.2/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= 35 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 36 | golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= 37 | golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 38 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 39 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 40 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 41 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 42 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 43 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 44 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 45 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 46 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 47 | golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= 48 | golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 49 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 50 | golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 51 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 52 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 53 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 54 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 56 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 57 | -------------------------------------------------------------------------------- /elasticpwn/localdev.Dockerfile: -------------------------------------------------------------------------------- 1 | # this image is only meant to be used for local development 2 | FROM golang:1.17-buster 3 | 4 | MAINTAINER Joel Mun <9oelm@wearehackerone.com> 5 | 6 | WORKDIR /go/src/app 7 | COPY . . 8 | 9 | RUN go mod download 10 | 11 | RUN go get github.com/githubnemo/CompileDaemon 12 | 13 | # watches file change to build again 14 | ENTRYPOINT CompileDaemon --build="go build -v" --command=./request 15 | -------------------------------------------------------------------------------- /elasticpwn/lookup-addrs/.gitignore: -------------------------------------------------------------------------------- 1 | lookup-addrs -------------------------------------------------------------------------------- /elasticpwn/lookup-addrs/README.md: -------------------------------------------------------------------------------- 1 | # lookup-addrs 2 | 3 | Performs quick intel of IP addresses supplied. 4 | If possible, grabs: 5 | - subject name of SSL certificate 6 | - organization name of SSL certificate 7 | - ip addr lookup result (equivalent to nslookup result), which is cloud hosting provider 8 | 9 | Works quite similarly to `shodan parse`, but much simpler version. 10 | 11 | ## Usage 12 | - Input a file with IPs in it. Doesn't matter if it's in the form of a complicated URL like `https://3.3.3.3:12456/asldfkjdklsajweaf?125125=1`. 13 | - It will just simply pull out IPs by using regex (fine print: therefore, it will basically pull out anything that looks like an IP, even if it is in the querystring of the URL, for example) 14 | 15 | # Build 16 | ``` 17 | go build 18 | ``` 19 | 20 | # Run 21 | ``` 22 | Usage of lookup-addrs: 23 | -inputFilePath string 24 | Path to the text file that contains list of URLs (default "./urls.txt") 25 | -outputFilePath string 26 | Path to output CSV file (default "./out.csv") 27 | -threads int 28 | Number of threads to use (default 20) 29 | -verbose 30 | Verbosity (default true) 31 | Example: 32 | ./lookup-addrs -threads=30 -inputFilePath=input.txt -outputFilePath=output.csv -verbose=false 33 | 34 | Note that the boolean flag should be fed as -isVerbose=false. -isVerbose false won't get it to work. 35 | ``` 36 | 37 | # Example input & output 38 | Input: 39 | 40 | ``` 41 | 100.24.148.45:443 42 | 100.24.154.46:80 43 | 100.24.158.213:80 44 | 100.24.169.157:80 45 | 100.24.176.122:80 46 | 100.24.176.59:80 47 | 100.24.197.246:80 48 | 100.24.202.245:80 49 | 100.24.214.180:80 50 | 100.24.217.207:443 51 | 100.24.222.19:443 52 | 100.24.230.119:80 53 | 100.24.241.246:80 54 | 100.24.251.5:80 55 | 100.24.77.124:80 56 | 100.24.78.201:443 57 | 100.24.83.233:80 58 | 100.24.85.28:80 59 | 100.24.93.146:80 60 | 100.25.106.49:443 61 | 100.25.176.220:80 62 | 100.25.185.7:443 63 | 100.25.186.116:443 64 | 100.25.186.116:80 65 | 100.25.194.119:443 66 | 100.25.196.233:80 67 | 100.25.197.121 68 | ``` 69 | 70 | ``` 71 | 100.24.148.45:443,,data.infopriceti.com.br, 72 | 100.24.154.46:80,,, 73 | 100.24.158.213:80,,, 74 | 100.24.169.157:80,,, 75 | 100.24.176.122:80,,, 76 | 100.24.176.59:80,,, 77 | 100.24.197.246:80,,, 78 | 100.24.202.245:80,,, 79 | 100.24.214.180:80,,, 80 | 100.24.217.207:443,,sbopslive.servisbotconnect.com, 81 | 100.24.222.19:443,,operations.vidgo.veygo.co, 82 | 100.24.230.119:80,,, 83 | 100.24.241.246:80,,, 84 | 100.24.251.5:80,,, 85 | 100.24.77.124:80,,, 86 | 100.24.78.201:443,,fivana.com, 87 | 100.24.83.233:80,,, 88 | 100.24.85.28:80,,, 89 | 100.24.93.146:80,,, 90 | 100.25.106.49:443,,oncoramedical.com, 91 | 100.25.176.220:80,,, 92 | 100.25.185.7:443,,accessclinicaltrials.niaid.nih.gov,National Institutes of Health,Entrust, Inc.,Entrust, Inc. 93 | 100.25.186.116:443,,gct-internal.net, 94 | 100.25.186.116:80,,, 95 | 100.25.194.119:443,,grafana.prudentte.com.br, 96 | 100.25.196.233:80,,, 97 | 100.25.197.121,,, 98 | ``` -------------------------------------------------------------------------------- /elasticpwn/lookup-addrs/build-binary.sh: -------------------------------------------------------------------------------- 1 | # Go does not support building binary from non-main package. This is a little script to build standalone binary. 2 | mkdir ./tmp-binary-build 3 | 4 | cp *.go ./tmp-binary-build 5 | 6 | cd tmp-binary-build 7 | 8 | go mod init github.com/9oelM/elasticpwn/src/tmp-binary-build/lookup-addrs 9 | 10 | go mod edit -replace github.com/9oelM/elasticpwn/elasticpwn/util=../../util 11 | 12 | go mod tidy 13 | 14 | # Replace all occurences 15 | sed -i 's/package EPLookup_addrs/package main/g' *.go 16 | 17 | go build -v 18 | 19 | cp lookup-addrs ../ 20 | 21 | cd .. 22 | 23 | rm -rf ./tmp-binary-build 24 | -------------------------------------------------------------------------------- /elasticpwn/lookup-addrs/lookup-addrs.go: -------------------------------------------------------------------------------- 1 | package EPLookup_addrs 2 | 3 | import ( 4 | "crypto/tls" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 14 | ) 15 | 16 | /* 17 | Common name != Subject name 18 | 19 | https://stackoverflow.com/questions/5935369/how-do-common-names-cn-and-subject-alternative-names-san-work-together/29600674 20 | 21 | Important: origin needs correct port number to access HTTP(S) service. 22 | 23 | Example: getSslCertificateInfo("https://google.com:443") 24 | 25 | Default timeout: 10 secs 26 | */ 27 | func GetSslCertificateInfo(origin string) (maybeValidUrls string, maybeValidOrgs string) { 28 | tlsConf := &tls.Config{ 29 | InsecureSkipVerify: true, 30 | } 31 | conn, err := tls.DialWithDialer( 32 | &net.Dialer{ 33 | Timeout: time.Second * 5, 34 | }, 35 | "tcp", 36 | origin, 37 | tlsConf, 38 | ) 39 | if err != nil { 40 | return "", "" 41 | } 42 | defer conn.Close() 43 | certs := conn.ConnectionState().PeerCertificates 44 | for _, cert := range certs { 45 | // also will match URL inside a wildcard domain, like *.example.com -> .example.com 46 | maybeValidUrl := UrlRegex.FindString(cert.Subject.CommonName) 47 | // prevent dups 48 | if maybeValidUrl != "" && !strings.Contains(maybeValidUrls, maybeValidUrl) { 49 | maybeValidUrls += strings.TrimPrefix(maybeValidUrl, ".") + "," 50 | } 51 | // prevent dups 52 | for _, org := range EPUtils.Unique(cert.Subject.Organization) { 53 | if EPUtils.Contains(strings.TrimSpace(org), EXCLUDE_SSL_ORGS) == -1 { 54 | maybeValidOrgs += strings.TrimSpace(org) + "," 55 | } 56 | } 57 | } 58 | return strings.TrimSuffix(maybeValidUrls, ","), strings.TrimSuffix(maybeValidOrgs, ",") 59 | } 60 | 61 | func GetIpInfo(ipWithMaybePortNum string) (string, string, string, string) { 62 | ipSplit := strings.Split(ipWithMaybePortNum, ":") 63 | ip := ipSplit[0] 64 | 65 | wg := sync.WaitGroup{} 66 | subjectUrlsChan := make(chan string) 67 | organizationsChan := make(chan string) 68 | cloudHostingProvidersChan := make(chan string) 69 | cnameChan := make(chan string) 70 | wg.Add(1) 71 | go func(cloudHostingProvidersChan chan string) { 72 | defer wg.Done() 73 | validDomains, err := net.LookupAddr(ip) 74 | if err == nil { 75 | for _, domain := range validDomains { 76 | if EPUtils.ContainsEndsWith(domain, NOT_REALLY_INTERESTING_DOMAINS) == -1 { 77 | result := fmt.Sprintf("%s,%s\n", ipWithMaybePortNum, domain) 78 | cloudHostingProvidersChan <- result 79 | return 80 | } else { 81 | cloudHostingProvidersChan <- "" 82 | } 83 | } 84 | } else { 85 | // silently ignore error, otherwise there will be too many error outputs 86 | cloudHostingProvidersChan <- "" 87 | } 88 | }(cloudHostingProvidersChan) 89 | wg.Add(1) 90 | go func(organizationsChan chan string, subjectUrlsChan chan string) { 91 | defer wg.Done() 92 | subjectUrls, organizations := GetSslCertificateInfo(ipWithMaybePortNum) 93 | if subjectUrls != "" || organizations != "" { 94 | subjectUrlsChan <- subjectUrls 95 | organizationsChan <- organizations 96 | } else { 97 | subjectUrlsChan <- "" 98 | organizationsChan <- "" 99 | } 100 | }(organizationsChan, subjectUrlsChan) 101 | wg.Add(1) 102 | go func(cnameChan chan string) { 103 | defer wg.Done() 104 | cname, err := net.LookupCNAME(ip) 105 | 106 | if cname != "" || err == nil { 107 | fmt.Println(cname) 108 | cnameChan <- cname 109 | } else { 110 | cnameChan <- "" 111 | } 112 | }(cnameChan) 113 | 114 | cloudHostingProvider, subjectUrls, organizations, cname := <-cloudHostingProvidersChan, <-subjectUrlsChan, <-organizationsChan, <-cnameChan 115 | wg.Wait() 116 | 117 | return cloudHostingProvider, subjectUrls, organizations, cname 118 | } 119 | 120 | func parseFlagsAndInit() (inputFilePath *string, outputFilePath *string, numThreads *int, logger func(string)) { 121 | flagSet := flag.NewFlagSet("lookup-addrs", flag.ContinueOnError) 122 | inputFilePath = flagSet.String("inputFilePath", "./urls.txt", "Path to the text file that contains list of URLs") 123 | outputFilePath = flagSet.String("outputFilePath", "./out.csv", "Path to output CSV file") 124 | numThreads = flagSet.Int("threads", 20, "Number of threads to use") 125 | isVerbose := flagSet.Bool("verbose", true, "Verbosity") 126 | 127 | if err := flagSet.Parse(os.Args[1:]); err != nil { 128 | fmt.Println("Failed to parse flags.") 129 | fmt.Println(`Example: 130 | ./lookup-addrs -threads=30 -inputFilePath=input.txt -outputFilePath=output.csv -verbose=false`) 131 | fmt.Println("Note that the boolean flag should be fed as -isVerbose=false. -isVerbose false won't get it to work.") 132 | panic(err) 133 | } 134 | logger = CreateMaybeVerboseEPLogger(*isVerbose) 135 | 136 | if flagSet.Parsed() { 137 | logger(fmt.Sprintf("Received inputFilePath: %s, outputFilePath: %s, verbose: %v, threads: %v", *inputFilePath, *outputFilePath, *isVerbose, *numThreads)) 138 | } else { 139 | panic("Flags were not parsed") 140 | } 141 | 142 | return 143 | } 144 | 145 | func main() { 146 | inputFilePath, outputFilePath, numThreads, logger := parseFlagsAndInit() 147 | urls := EPUtils.ReadUrlsFromFile(*inputFilePath) 148 | allIps := IpWithOptionalPortRegex.FindAllString(urls, -1) 149 | 150 | if allIps == nil { 151 | logger(fmt.Sprintf("No valid IPs found from file %s. Check again.", *inputFilePath)) 152 | return 153 | } 154 | 155 | concurrentGoroutines := make(chan struct{}, *numThreads) 156 | f, err := os.Create(*outputFilePath) 157 | EPUtils.ExitOnError(err) 158 | defer f.Close() 159 | 160 | var wg sync.WaitGroup 161 | for _, ipWithMaybePortNum := range allIps { 162 | wg.Add(1) 163 | go func(ipWithMaybePortNum string, f *os.File) { 164 | defer wg.Done() 165 | 166 | concurrentGoroutines <- struct{}{} 167 | 168 | cloudHostingProvider, subjectUrls, organization, cname := GetIpInfo(ipWithMaybePortNum) 169 | result := strings.ReplaceAll(fmt.Sprintf("%s,%s,%s,%s,%s", ipWithMaybePortNum, cloudHostingProvider, subjectUrls, organization, cname), "\n", "") 170 | if cloudHostingProvider != "" || subjectUrls != "" || organization != "" || cname != "" { 171 | EPUtils.EPLogger(result) 172 | } 173 | _, err = fmt.Fprintf(f, string(result)+"\n") 174 | <-concurrentGoroutines 175 | }(ipWithMaybePortNum, f) 176 | } 177 | wg.Wait() 178 | logger("Finished lookup of addresses") 179 | } 180 | -------------------------------------------------------------------------------- /elasticpwn/lookup-addrs/utils.go: -------------------------------------------------------------------------------- 1 | package EPLookup_addrs 2 | 3 | import ( 4 | "regexp" 5 | 6 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 7 | ) 8 | 9 | var EXCLUDE_SSL_ORGS = []string{ 10 | "Digicert", 11 | "DigiCert", 12 | "Sectigo", 13 | "SECTIGO", 14 | "Let's Encrypt", 15 | "GlobalSign", 16 | "Amazon", 17 | "USERTRUST", 18 | "Internet Security Research Group", 19 | "Google Trust Services", 20 | "IdenTrust", 21 | "GoDaddy.com", 22 | "The Go Daddy Group", 23 | "Starfield Technologies", 24 | "Comodo", 25 | "COMODO", 26 | "Acme", 27 | "ACME", 28 | } 29 | 30 | var NOT_REALLY_INTERESTING_DOMAINS = []string{ 31 | "amazonaws.com.", 32 | "googleusercontent.com.", 33 | "linode.com.", 34 | "awsglobalaccelerator.com.", 35 | "vultr.com.", 36 | "fios.verizon.net.", 37 | } 38 | 39 | var IpWithOptionalPortRegex = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(:[0-9]+)?`) 40 | var UrlRegex = regexp.MustCompile(`[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)`) 41 | 42 | func CreateMaybeVerboseEPLogger(isVerbose bool) func(string) { 43 | if isVerbose { 44 | return EPUtils.EPLogger 45 | } else { 46 | return func(a string) {} 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /elasticpwn/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "path/filepath" 10 | "strings" 11 | "syscall" 12 | "time" 13 | 14 | EPPlugins "github.com/9oelM/elasticpwn/elasticpwn/plugins" 15 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 16 | ) 17 | 18 | // elasticsearch and kibana plugins have the exact same fields as of now. 19 | // The reason for the separation of the plugins and the flags is that 20 | // elasticsearch and kibana carry a bit of different traits, so I am a bit afraid 21 | // if anything that's different between them might cause a change in the code in the future. 22 | // therefore I just separated them into two different plugins and flagsets. So please be reminded that this design is by intention. 23 | type Elasticpwn struct { 24 | mode string 25 | elasticSearchPlugin *EPPlugins.ElasticSearchPlugin 26 | kibanaPlugin *EPPlugins.KibanaPlugin 27 | reportGeneratePlugin *EPPlugins.ReportGeneratePlugin 28 | reportViewPlugin *EPPlugins.ReportViewPlugin 29 | } 30 | 31 | var EP_OUTPUT_MODES = []string{"mongo", "json", "plain"} 32 | 33 | func printBasicInstruction( 34 | esPluginFs *flag.FlagSet, 35 | kibanaPluginFs *flag.FlagSet, 36 | reportGeneratePluginFs *flag.FlagSet, 37 | reportViewPluginFs *flag.FlagSet, 38 | ) { 39 | fmt.Println("Usage: elasticpwn [elasticsearch|kibana [...plugin options] or elasticpwn report [generate|view] [...plugin options]") 40 | fmt.Println("[elasticsearch] plugin options:") 41 | esPluginFs.PrintDefaults() 42 | fmt.Println("[kibana] plugin options:") 43 | kibanaPluginFs.PrintDefaults() 44 | fmt.Println("[report] generate plugin options:") 45 | reportGeneratePluginFs.PrintDefaults() 46 | fmt.Println("[report] view plugin options:") 47 | reportViewPluginFs.PrintDefaults() 48 | } 49 | 50 | func initializeElasticpwn() *Elasticpwn { 51 | ESPluginFs := flag.NewFlagSet("elasticsearch-plugin", flag.ContinueOnError) 52 | ESPluginInputFilePath := ESPluginFs.String("f", "", "[REQUIRED] path to a file with urls (url per line)") 53 | ESPluginThreadsNum := ESPluginFs.Int("t", 8, "[OPTIONAL] number of threads when running a plugin") 54 | ESPluginOutputMode := ESPluginFs.String("om", "json", `[OPTIONAL] output mode. json|mongo|plain. 55 | mongo option will require a mongo server to be up. 56 | plain mode will output json-like object to each line finishing with a comma. 57 | For mongo, Local docker mongo instance is recommneded. (check docker-compose.yml and docs)`) 58 | ESPluginOutputFilePath := ESPluginFs.String("of", "elasticsearch.json", "[OPTIONAL] output file name. Ignored when mongo option is chosen.") 59 | ESPluginMongoUrl := ESPluginFs.String("murl", "mongodb://root:example@172.17.0.1:27017/", `[OPTIONAL] needed only when -o=mongo is selected. 60 | mongodb url with username and pw included. 61 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this. 62 | `) 63 | // some instances have very many indices, and probably your hard drive will explode if you fetch all 64 | ESPluginMaxIndicesToCollect := ESPluginFs.Int("max-i", 5, `maximum number of indices to request. 65 | If you intend to set this as a high number, make sure you've got enough storage. 66 | If you don't know what an index is, 67 | refer to elasticserach docs at https://www.elastic.co/blog/what-is-an-elasticsearch-index`) 68 | // will return 'X' hits from the index 69 | ESPluginMaxIndexSizeToCollect := ESPluginFs.Int("max-is", 70, `maximum size of an index to request. 70 | If you intend to set this as a high number, make sure you've got enough storage. 71 | If you don't know what 'size' is, 72 | please refer to elasticsearch docs on 73 | '/_cat/_search?size=' at 74 | https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html`) 75 | 76 | // kibana plugin has the same options as elasticserach plugin right now, 77 | // but differences in their behaviors may always make the different, so leave it as it is. 78 | // as a result, there are many duplicates between es and kibana plugins at least in this file. 79 | kibanaPluginFs := flag.NewFlagSet("kibana-plugin", flag.ContinueOnError) 80 | kibanaPluginInputFilePath := kibanaPluginFs.String("f", "", "[REQUIRED] path to a file with urls (url per line)") 81 | kibanaPluginThreadsNum := kibanaPluginFs.Int("t", 8, "[OPTIONAL] number of threads when running a plugin") 82 | kibanaPluginOutputMode := kibanaPluginFs.String("om", "json", `[OPTIONAL] output mode. json|mongo|plain. 83 | mongo option will require a mongo server to be up. 84 | plain mode will output json-like object to each line finishing with a comma. 85 | For mongo, local docker mongo instance is recommneded. (check docker-compose.yml and docs)`) 86 | kibanaPluginOutputFilePath := kibanaPluginFs.String("of", "kibana.json", "[OPTIONAL] output file name. Ignored when mongo option is chosen.") 87 | kibanaPluginMongoUrl := kibanaPluginFs.String("murl", "mongodb://root:example@172.17.0.1:27017/", `[OPTIONAL] needed only when -o=mongo is selected. 88 | mongodb url with username and pw included. 89 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this. 90 | `) 91 | // some instances have very many indices, and probably your hard drive will explode if you fetch all 92 | kibanaPluginMaxIndicesToCollect := kibanaPluginFs.Int("max-i", 5, `maximum number of indices to request. 93 | If you intend to set this as a high number, make sure you've got enough storage. 94 | If you don't know what an index is, 95 | refer to elasticserach docs at 96 | https://www.elastic.co/blog/what-is-an-elasticsearch-index`) 97 | // will return 'X' hits from the index 98 | kibanaPluginMaxIndexSizeToCollect := kibanaPluginFs.Int("max-is", 70, `maximum size of an index to request. 99 | If you intend to set this as a high number, make sure you've got enough storage. 100 | If you don't know what 'size' is, please refer to elasticsearch docs on 101 | '/_cat/_search?size=' at 102 | https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html`) 103 | 104 | reportGeneratePluginFs := flag.NewFlagSet("report-generate-plugin", flag.ContinueOnError) 105 | reportGeneratePluginMongoUrl := reportGeneratePluginFs.String("murl", "mongodb://root:example@172.17.0.1:27017/", `[OPTIONAL] 106 | mongodb url with username and pw included from which gathered data can be accessed. 107 | Note that 172.17.0.1 is usually default docker host IP. You may want to change this.`) 108 | reportGeneratePluginCollectionName := reportGeneratePluginFs.String("cn", "", `[REQUIRED] 109 | must be elasticsearch|kibana. Collection name of the mongodb database to be used.`) 110 | reportGeneratePluginServerRootUrl := reportGeneratePluginFs.String("dn", "http://localhost:9292", `[OPTIONAL] 111 | backend url of persistent database server being used. This should be the url where elasticpwn-backend is hosted.`) 112 | 113 | reportViewPluginFs := flag.NewFlagSet("report-view-plugin", flag.ContinueOnError) 114 | reportViewPluginReportDirectory := reportViewPluginFs.String("d", filepath.FromSlash("./report"), "[OPTIONAL] the directory of generated report") 115 | reportViewPluginServeAtPort := reportViewPluginFs.String("p", "9999", "[OPTIONAL] local port to serve report page from") 116 | 117 | if len(os.Args) <= 1 { 118 | fmt.Println("Plugin is not selected. Please try again.") 119 | } 120 | if len(os.Args) <= 1 { 121 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 122 | os.Exit(1) 123 | } 124 | EPMode := os.Args[1] 125 | switch EPMode { 126 | case "elasticsearch": 127 | { 128 | if err := ESPluginFs.Parse(os.Args[2:]); err != nil { 129 | fmt.Println("Wrong options given. Please try again.\nexample: elasticpwn elasticsearch -t=40 -om=json -f=urls.txt") 130 | os.Exit(1) 131 | } 132 | } 133 | case "kibana": 134 | { 135 | if err := kibanaPluginFs.Parse(os.Args[2:]); err != nil { 136 | fmt.Println("Wrong options given. Please try again.\nexample: elasticpwn kibana -t=40 -om=json -f=urls.txt") 137 | os.Exit(1) 138 | } 139 | } 140 | case "report": 141 | { 142 | if len(os.Args) <= 2 { 143 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 144 | os.Exit(1) 145 | } 146 | EPReportMode := os.Args[2] 147 | 148 | switch EPReportMode { 149 | case "generate": 150 | { 151 | if err := reportGeneratePluginFs.Parse(os.Args[3:]); err != nil { 152 | fmt.Println("Wrong options given. Please try again.\nexample: elasticpwn report generate -murl mongodb://root:example@172.17.0.1:27017/") 153 | os.Exit(1) 154 | } 155 | } 156 | case "view": 157 | { 158 | if err := reportViewPluginFs.Parse(os.Args[3:]); err != nil { 159 | fmt.Println("Wrong options given. Please try again.\nexample: elasticpwn report view -d ./report") 160 | os.Exit(1) 161 | } 162 | } 163 | default: 164 | { 165 | EPUtils.EPLogger(fmt.Sprintf("\"%v\" is not a valid mode. should be either \"generate\" or \"view\".", EPReportMode)) 166 | os.Exit(1) 167 | } 168 | } 169 | } 170 | default: 171 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 172 | os.Exit(1) 173 | } 174 | 175 | switch { 176 | case ESPluginFs.Parsed(): 177 | { 178 | needsExit := false 179 | 180 | needsExit = EPUtils.ValidateStringFlag( 181 | *ESPluginInputFilePath, 182 | "", 183 | "-f", 184 | ) || 185 | EPUtils.ValidatePositiveInt( 186 | *ESPluginThreadsNum, 187 | "-t", 188 | ) || 189 | EPUtils.ValidatePositiveInt( 190 | *ESPluginMaxIndexSizeToCollect, 191 | "-max-is", 192 | ) || 193 | EPUtils.ValidatePositiveInt( 194 | *ESPluginMaxIndicesToCollect, 195 | "-max-i", 196 | ) 197 | 198 | switch { 199 | case EPUtils.Contains(*ESPluginOutputMode, EP_OUTPUT_MODES) == -1: 200 | { 201 | fmt.Printf("-om received an invalid value: %v. Possible values: %v\n", *ESPluginOutputMode, strings.Join(EP_OUTPUT_MODES, ",")) 202 | needsExit = true 203 | break 204 | } 205 | case *ESPluginOutputFilePath != "elasticsearch.json" && *ESPluginOutputMode == "mongo": 206 | { 207 | EPUtils.EPLogger(fmt.Sprintf("-of option was set as %v but will be ignored because -o option was set as mongo\n", *ESPluginOutputFilePath)) 208 | } 209 | case *ESPluginOutputMode == "mongo" && *ESPluginMongoUrl == "mongodb://root:example@mongo:27017/": 210 | { 211 | EPUtils.EPLogger(fmt.Sprintf("-of option selected as monngo. Proceeding with default mongo URL: %v. If you intend to connect to another url, please specify it with -murl option.", *ESPluginMongoUrl)) 212 | } 213 | } 214 | if needsExit { 215 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 216 | os.Exit(1) 217 | } 218 | } 219 | case kibanaPluginFs.Parsed(): 220 | { 221 | needsExit := false 222 | 223 | needsExit = EPUtils.ValidateStringFlag( 224 | *kibanaPluginInputFilePath, 225 | "", 226 | "-f", 227 | ) || 228 | EPUtils.ValidatePositiveInt( 229 | *kibanaPluginThreadsNum, 230 | "-t", 231 | ) || 232 | EPUtils.ValidatePositiveInt( 233 | *kibanaPluginMaxIndexSizeToCollect, 234 | "-max-is", 235 | ) || 236 | EPUtils.ValidatePositiveInt( 237 | *kibanaPluginMaxIndicesToCollect, 238 | "-max-i", 239 | ) 240 | 241 | if needsExit { 242 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 243 | os.Exit(1) 244 | } 245 | } 246 | case reportGeneratePluginFs.Parsed(): 247 | { 248 | needsExit := false 249 | 250 | needsExit = EPUtils.ValidateStringFlag(*reportGeneratePluginCollectionName, "", "-cn") 251 | 252 | if *reportGeneratePluginCollectionName != "elasticsearch" && *reportGeneratePluginCollectionName != "kibana" { 253 | EPUtils.EPLogger(fmt.Sprintf("-cn option: \"%v\" is not a valid collection name. should be either \"kibana\" or \"elasticsearch\".", *reportGeneratePluginCollectionName)) 254 | needsExit = true 255 | } 256 | 257 | if needsExit { 258 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 259 | os.Exit(1) 260 | } 261 | } 262 | case reportViewPluginFs.Parsed(): 263 | { 264 | needsExit := false 265 | 266 | if needsExit { 267 | printBasicInstruction(ESPluginFs, kibanaPluginFs, reportGeneratePluginFs, reportViewPluginFs) 268 | os.Exit(1) 269 | } 270 | } 271 | default: 272 | { 273 | // should never get here 274 | panic("not a valid plugin name") 275 | } 276 | } 277 | 278 | return &Elasticpwn{ 279 | mode: EPMode, 280 | elasticSearchPlugin: &EPPlugins.ElasticSearchPlugin{ 281 | InputFilePath: *ESPluginInputFilePath, 282 | ThreadsNum: *ESPluginThreadsNum, 283 | OutputMode: *ESPluginOutputMode, 284 | OutputFilePath: *ESPluginOutputFilePath, 285 | MongoUrl: *ESPluginMongoUrl, 286 | EsPluginMaxIndices: *ESPluginMaxIndicesToCollect, 287 | EsPluginMaxIndexSize: *ESPluginMaxIndexSizeToCollect, 288 | }, 289 | kibanaPlugin: &EPPlugins.KibanaPlugin{ 290 | InputFilePath: *kibanaPluginInputFilePath, 291 | ThreadsNum: *kibanaPluginThreadsNum, 292 | OutputMode: *kibanaPluginOutputMode, 293 | OutputFilePath: *kibanaPluginOutputFilePath, 294 | MongoUrl: *kibanaPluginMongoUrl, 295 | MaxIndices: *kibanaPluginMaxIndicesToCollect, 296 | MaxIndexSize: *kibanaPluginMaxIndexSizeToCollect, 297 | }, 298 | reportGeneratePlugin: &EPPlugins.ReportGeneratePlugin{ 299 | MongoUrl: *reportGeneratePluginMongoUrl, 300 | CollectionName: *reportGeneratePluginCollectionName, 301 | ServerRootUrl: *reportGeneratePluginServerRootUrl, 302 | }, 303 | reportViewPlugin: &EPPlugins.ReportViewPlugin{ 304 | ReportDirectory: *reportViewPluginReportDirectory, 305 | ServeAtPort: *reportViewPluginServeAtPort, 306 | }, 307 | } 308 | } 309 | 310 | func setupGracefulExit(ep *Elasticpwn) { 311 | c := make(chan os.Signal) 312 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 313 | go func() { 314 | <-c 315 | EPUtils.EPLogger("Interrupted with Ctrl+C. Finalizing..") 316 | go func() { 317 | for { 318 | time.Sleep(time.Duration(2 * time.Second)) 319 | EPUtils.EPLogger("Interrupted with Ctrl+C. Finalizing..") 320 | } 321 | }() 322 | switch { 323 | case ep.mode == "kibana" && ep.kibanaPlugin.OutputMode == "json": 324 | EPUtils.ConvertJSONObjectsToJSONArray(ep.kibanaPlugin.OutputFilePath) 325 | case ep.mode == "elasticsearch" && ep.kibanaPlugin.OutputMode == "json": 326 | EPUtils.ConvertJSONObjectsToJSONArray(ep.elasticSearchPlugin.OutputFilePath) 327 | } 328 | os.Exit(0) 329 | }() 330 | } 331 | 332 | func main() { 333 | defer os.Exit(0) 334 | 335 | elasticpwn := initializeElasticpwn() 336 | setupGracefulExit(elasticpwn) 337 | 338 | switch elasticpwn.mode { 339 | case "elasticsearch": 340 | { 341 | urls, elasticSearchCollection := EPPlugins.Prepare( 342 | elasticpwn.elasticSearchPlugin.InputFilePath, 343 | elasticpwn.elasticSearchPlugin.MongoUrl, 344 | elasticpwn.elasticSearchPlugin.OutputMode, 345 | "elasticsearch", 346 | ) 347 | elasticpwn.elasticSearchPlugin.Run(urls, elasticSearchCollection) 348 | elasticpwn.elasticSearchPlugin.PostProcess() 349 | break 350 | } 351 | case "kibana": 352 | { 353 | urls, kibanaCollection := EPPlugins.Prepare( 354 | elasticpwn.kibanaPlugin.InputFilePath, 355 | elasticpwn.kibanaPlugin.MongoUrl, 356 | elasticpwn.kibanaPlugin.OutputMode, 357 | "kibana", 358 | ) 359 | elasticpwn.kibanaPlugin.Run(urls, kibanaCollection) 360 | elasticpwn.kibanaPlugin.PostProcess() 361 | break 362 | } 363 | case "report": 364 | { 365 | if len(os.Args) <= 2 { 366 | log.Fatal("report mode is not defined. it should be either \"generate\" or \"view\".") 367 | os.Exit(1) 368 | } 369 | EPReportMode := os.Args[2] 370 | switch EPReportMode { 371 | case "generate": 372 | elasticpwn.reportGeneratePlugin.Run() 373 | case "view": 374 | elasticpwn.reportViewPlugin.Run() 375 | } 376 | } 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /elasticpwn/plugins/elastic-util.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 13 | 14 | "go.mongodb.org/mongo-driver/bson" 15 | "go.mongodb.org/mongo-driver/mongo" 16 | ) 17 | 18 | // elastic-util contains common stuffs relevant to elastic products, 19 | // i.e. elasticsearch and kibana. 20 | 21 | // { 22 | // "health" : "yellow", 23 | // "status" : "open", 24 | // "index" : ".kibana", 25 | // "uuid" : "J399dVXVTDKTOShf6Q1DyA", 26 | // "pri" : "1", 27 | // "rep" : "1", 28 | // "docs.count" : "3", 29 | // "docs.deleted" : "0", 30 | // "store.size" : "54.1kb", 31 | // "pri.store.size" : "54.1kb" 32 | // }, 33 | type IndexInfo struct { 34 | InterestingIndexInfo 35 | Health string `bson:"health,omitempty" json:"health"` 36 | Status string `bson:"status,omitempty" json:"status"` 37 | Uuid string `bson:"uuid,omitempty" json:"uuid"` 38 | Pri string `bson:"pri,omitempty" json:"pri"` 39 | Rep string `bson:"rep,omitempty" json:"rep"` 40 | } 41 | 42 | type InterestingIndexInfo struct { 43 | Index string `bson:"index,omitempty" json:"index"` 44 | DocsCount string `bson:"docs.count,omitempty" json:"docs.count"` 45 | DocsDeleted string `bson:"docs.deleted,omitempty" json:"docs.deleted"` 46 | StoreSize string `bson:"store.size,omitempty" json:"store.size"` 47 | PriStoreSize string `bson:"pri.store.size,omitempty" json:"pri.store.size"` 48 | } 49 | 50 | func filterInterestingIndexFields(indexInfo IndexInfo) InterestingIndexInfo { 51 | var interestingIndexInfo InterestingIndexInfo 52 | 53 | interestingIndexInfo.DocsCount = indexInfo.DocsCount 54 | interestingIndexInfo.DocsDeleted = indexInfo.DocsDeleted 55 | interestingIndexInfo.StoreSize = indexInfo.StoreSize 56 | interestingIndexInfo.PriStoreSize = indexInfo.PriStoreSize 57 | interestingIndexInfo.Index = indexInfo.Index 58 | 59 | return interestingIndexInfo 60 | } 61 | 62 | func ProcessInterestingIndices(indices []IndexInfo) []InterestingIndexInfo { 63 | var interestingIndices []InterestingIndexInfo 64 | 65 | for _, indexInfo := range indices { 66 | if !IsUninterestingIndex(indexInfo.Index) { 67 | interestingIndices = append(interestingIndices, filterInterestingIndexFields(indexInfo)) 68 | } 69 | } 70 | 71 | return interestingIndices 72 | } 73 | 74 | func NewInterstingInfoFromIndexSearch() *InterestingInfoFromIndexSearch { 75 | return &InterestingInfoFromIndexSearch{ 76 | Emails: []string{}, 77 | Urls: []string{}, 78 | PublicIPs: []string{}, 79 | MoreThanTwoDotsInName: []string{}, 80 | } 81 | } 82 | 83 | func appendObjectsOfInterest( 84 | interestingInfoFromIndexSearch *InterestingInfoFromIndexSearch, 85 | interestingWordsFromIndexSearch []string, 86 | interestingInfo *InterestingInfoFromIndexSearch, 87 | interestingWords *[]string, 88 | ) { 89 | if interestingInfoFromIndexSearch != nil { 90 | interestingInfo.Emails = append(interestingInfo.Emails, interestingInfoFromIndexSearch.Emails...) 91 | interestingInfo.Urls = append(interestingInfo.Urls, interestingInfoFromIndexSearch.Urls...) 92 | interestingInfo.PublicIPs = append(interestingInfo.PublicIPs, interestingInfoFromIndexSearch.PublicIPs...) 93 | interestingInfo.MoreThanTwoDotsInName = append(interestingInfo.MoreThanTwoDotsInName, interestingInfoFromIndexSearch.MoreThanTwoDotsInName...) 94 | } 95 | if interestingWordsFromIndexSearch != nil { 96 | // modify the reference 97 | *interestingWords = append(*interestingWords, interestingWordsFromIndexSearch...) 98 | } 99 | } 100 | 101 | func ProcessInterestingInfoAndWordsThreadSafely( 102 | mu *sync.Mutex, 103 | singleInstanceScanResult interface{}, 104 | indexInfoObjectInJsonString string, 105 | ) { 106 | interestingInfoFromIndexSearch := ProcessInterestingInfoFromIndexSearch(indexInfoObjectInJsonString) 107 | interestingWordsFromIndexSearch := ProcessInterestingWordsFromIndexSearch(indexInfoObjectInJsonString) 108 | 109 | // just bear with me, gopls does not officially support generics yet 110 | mu.Lock() 111 | defer mu.Unlock() 112 | switch scanResult := singleInstanceScanResult.(type) { 113 | case *SingleElasticsearchInstanceScanResult: 114 | if scanResult.InterestingInfo == nil { 115 | scanResult.InterestingInfo = NewInterstingInfoFromIndexSearch() 116 | } 117 | appendObjectsOfInterest( 118 | interestingInfoFromIndexSearch, 119 | interestingWordsFromIndexSearch, 120 | scanResult.InterestingInfo, 121 | &scanResult.InterestingWords, 122 | ) 123 | case *SingleKibanaInstanceScanResult: 124 | if scanResult.InterestingInfo == nil { 125 | scanResult.InterestingInfo = NewInterstingInfoFromIndexSearch() 126 | } 127 | appendObjectsOfInterest( 128 | interestingInfoFromIndexSearch, 129 | interestingWordsFromIndexSearch, 130 | scanResult.InterestingInfo, 131 | &scanResult.InterestingWords, 132 | ) 133 | default: 134 | panic(fmt.Sprintf("unrecognized type of scan result detected: %v\n", scanResult)) 135 | } 136 | } 137 | 138 | func CheckOverGBIndexExistence(interestingIndices []InterestingIndexInfo) bool { 139 | hasOverGBIndex := false 140 | 141 | for _, indexInfo := range interestingIndices { 142 | hasOverGBIndex = strings.HasSuffix(indexInfo.StoreSize, "gb") || 143 | strings.HasSuffix(indexInfo.StoreSize, "tb") || 144 | strings.HasSuffix(indexInfo.StoreSize, "pb") || 145 | strings.HasSuffix(indexInfo.PriStoreSize, "gb") || 146 | strings.HasSuffix(indexInfo.PriStoreSize, "tb") || 147 | strings.HasSuffix(indexInfo.PriStoreSize, "pb") 148 | 149 | if hasOverGBIndex { 150 | break 151 | } 152 | } 153 | 154 | return hasOverGBIndex 155 | } 156 | 157 | // @todo read from a dictionary file instead 158 | var InterestingWordsLowercase = []string{ 159 | // "appid", 160 | // "appname", 161 | "app", 162 | "domain", 163 | "referrer", 164 | "referer", 165 | "url", 166 | "phone", 167 | "address", 168 | "email", 169 | "transfer", 170 | "balance", 171 | "payment", 172 | "user", 173 | "username", 174 | "password", 175 | "token", 176 | "chat", 177 | "message", 178 | } 179 | 180 | var UninterstingWordsLowercase = []string{ 181 | "example.com", 182 | "abc.com", 183 | } 184 | 185 | // if an index name is any one of these, it will very likely contain no useful information 186 | var UninterstingIndexNamesLowercase = []string{ 187 | // already hacked by 'meow' bot. see https://www.elastic.co/blog/protect-your-elasticsearch-deployments-against-meow-bot-attacks-for-free 188 | "meow", 189 | // elasticsearch internal index. useless. 190 | ".geoip_databases", 191 | // hackers create an index like this when they find an open elasticsearch instance. therefore useless. 192 | "readme", 193 | "read_me", 194 | "read__me", 195 | // magento store deployment. usually nothing interesting 196 | "magento", 197 | "market.kline", 198 | "waveland-datas", 199 | "resources_index", 200 | "company-datas", 201 | "movies", 202 | // these indices come from frameworks and usually contain very useless info 203 | "zend3", 204 | "casa", 205 | "kkrp", 206 | "actuator", 207 | "m.api", 208 | "solr", 209 | "minio", 210 | "daman", 211 | "index.php", 212 | "index.js", 213 | "index.jsp", 214 | "index.py", 215 | "index.do", 216 | "index.htm", 217 | "index.html", 218 | "index.cfm", 219 | "index.aspx", 220 | "index.cgi", 221 | "index.pl", 222 | "index.asp", 223 | "index.action", 224 | "yz.jsp", 225 | "ilm-history", 226 | "seismic", 227 | "result-logs", 228 | // sometimes users just wanna test it out. useless 229 | "demo", 230 | // elasticsearch internal 231 | ".async-search", 232 | // shopify plugins 233 | "vue_storefront", 234 | "produtos", 235 | } 236 | 237 | // must be coming from a framework. ditch all data if detected 238 | var UninterstingIfAllOfTheseInIndices = []string{ 239 | "service", 240 | "actuator", 241 | "casa", 242 | "auth", 243 | } 244 | 245 | var UninterestingIfExactlyMatches = []string{ 246 | "service", 247 | "auth", 248 | "actions", 249 | "casa", 250 | "website", 251 | "api", 252 | "login", 253 | "config", 254 | "oauth", 255 | "connect", 256 | "v1", 257 | "v2", 258 | "km.asmx", 259 | "biz", 260 | // edx open source 261 | "wap", 262 | // edx open source 263 | "courseware_index", 264 | // some weird chinese data 265 | "v3", 266 | // some weird chinese data 267 | "video_info", 268 | // elasticsearch internal 269 | ".elastichq", 270 | // trading related 271 | "btc.bitfinex.ticker", 272 | "eth.bitfinex.ticker", 273 | "btc.bitmex.ticker", 274 | } 275 | 276 | // if an index name starts with this, the index info is likely to be quite useless 277 | var UninterstingIndexNamesStartingWith = []string{ 278 | // elasticsearch internal 279 | ".kibana", 280 | // elasticserach internal 281 | ".apm", 282 | // elasticsearch index lifecycle 283 | "ilm-history", 284 | } 285 | 286 | // returns true if an index name is uninteresting, therefore useless to be inspected or stored 287 | func IsUninterestingIndex(indexName string) bool { 288 | return EPUtils.ContainsStartsWith(indexName, UninterstingIndexNamesStartingWith) != -1 || 289 | EPUtils.Contains(indexName, UninterstingIndexNamesLowercase) != -1 || 290 | EPUtils.ContainsExactlyMatchesWith(indexName, UninterestingIfExactlyMatches) != -1 291 | } 292 | 293 | type InterestingInfoFromIndexSearch struct { 294 | Emails []string `bson:"emails,omitempty" json:"emails"` 295 | Urls []string `bson:"urls,omitempty" json:"urls"` 296 | PublicIPs []string `bson:"publicIps,omitempty" json:"publicIps"` 297 | MoreThanTwoDotsInName []string `bson:"moreThanTwoDotsInName,omitempty" json:"moreThanTwoDotsInName"` 298 | } 299 | 300 | func ProcessInterestingInfoFromIndexSearch(rawIndexSearchStringResult string) *InterestingInfoFromIndexSearch { 301 | 302 | return &InterestingInfoFromIndexSearch{ 303 | Urls: EPUtils.FindAllUniqueUrls(rawIndexSearchStringResult), 304 | PublicIPs: EPUtils.FindAllUniquePublicIps(rawIndexSearchStringResult), 305 | Emails: EPUtils.Unique(EPUtils.EmailRegex.FindAllString(rawIndexSearchStringResult, -1)), 306 | MoreThanTwoDotsInName: EPUtils.Unique((EPUtils.AtLeastTwoDotsInSentenceRegex.FindAllString(rawIndexSearchStringResult, -1))), 307 | } 308 | } 309 | 310 | type InterestingWordRegex struct { 311 | regex *regexp.Regexp 312 | word string 313 | } 314 | 315 | type InterestingWordRegexMatches struct { 316 | Matches []string `bson:"matches,omitempty" json:"matches"` 317 | Word string `bson:"word,omitempty" json:"word"` 318 | } 319 | 320 | var InterestingWordsRegexes = func() []*InterestingWordRegex { 321 | var regexes []*InterestingWordRegex 322 | 323 | for _, interestingWordLowerCase := range InterestingWordsLowercase { 324 | // https://stackoverflow.com/questions/15326421/how-do-i-do-a-case-insensitive-regular-expression-in-go 325 | // intended to match things like "transfer: 100 USD" 326 | // (?i) = ignore case 327 | // ^ = start of the word 328 | // (.) = anything except line break 329 | // {1,30} = between 1 and 10 times 330 | regex := regexp.MustCompile(fmt.Sprintf("(?i)(%s)(.){1,30}", interestingWordLowerCase)) 331 | regexes = append(regexes, &InterestingWordRegex{ 332 | regex: regex, 333 | word: interestingWordLowerCase, 334 | }) 335 | } 336 | 337 | return regexes 338 | }() 339 | 340 | func ProcessInterestingWordsFromIndexSearch(rawIndexSearchStringResult string) []string { 341 | var matches []string 342 | for _, interestingWordRegex := range InterestingWordsRegexes { 343 | matchesForSingleRegex := EPUtils.Unique(interestingWordRegex.regex.FindAllString(rawIndexSearchStringResult, -1)) 344 | matches = append(matches, matchesForSingleRegex...) 345 | } 346 | 347 | return matches 348 | } 349 | 350 | func AppendScanResultToJSONFileWithNewline(outputFilePath string, marshalledScanResult []byte, mu *sync.Mutex) { 351 | // I/O should be thread-safe already, but just be extra safe 352 | mu.Lock() 353 | defer mu.Unlock() 354 | f, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 355 | if err != nil { 356 | EPUtils.EPLogger(fmt.Sprintf("Failed to create/open %v for output.\n", outputFilePath)) 357 | } 358 | 359 | singleKibanaInstanceScanResultMarshalledWithCommaAndNewline := append(marshalledScanResult, ",\n"...) 360 | if _, err := f.Write(singleKibanaInstanceScanResultMarshalledWithCommaAndNewline); err != nil { 361 | EPUtils.EPLogger(fmt.Sprintf("Failed to write to %v.\n", outputFilePath)) 362 | } 363 | if err := f.Close(); err != nil { 364 | EPUtils.EPLogger(fmt.Sprintf("Failed to close i/o from %v.\n", outputFilePath)) 365 | } 366 | } 367 | 368 | func Prepare( 369 | inputFilePath string, 370 | mongoUrl string, 371 | outputMode string, 372 | mongoCollectionName string, 373 | ) (urls []string, elasticSearchCollection *mongo.Collection) { 374 | urls = strings.Split(EPUtils.ReadUrlsFromFile(inputFilePath), "\n") 375 | EPUtils.EPLogger(fmt.Sprintf("Total %d lines of URLs detected\n", len(urls))) 376 | if outputMode == "mongo" { 377 | mongoClient := EPUtils.InitMongoConnnection(mongoUrl) 378 | // @todo customize db/collection name 379 | elasticSearchCollection = mongoClient.Database("ep").Collection(mongoCollectionName) 380 | } 381 | 382 | return urls, elasticSearchCollection 383 | } 384 | 385 | func InsertSingleScanResultToMongo( 386 | collection *mongo.Collection, 387 | singleScanResult interface{}, 388 | rootUrl string, 389 | ) { 390 | insertContext, cancelInsert := context.WithTimeout(context.Background(), 391 | 10*time.Second) 392 | insertResult, insertResultErr := collection.InsertOne(insertContext, bson.M{"scanResult": singleScanResult}) 393 | 394 | defer cancelInsert() 395 | if insertResultErr != nil { 396 | EPUtils.EPLogger(fmt.Sprintf("Error while inserting info about %s into MongoDB.", rootUrl)) 397 | fmt.Println(insertResultErr) 398 | } else { 399 | EPUtils.EPLogger(fmt.Sprintf("Inserted info about %s into MongoDB Collection. ID: %s", rootUrl, insertResult.InsertedID)) 400 | } 401 | } 402 | -------------------------------------------------------------------------------- /elasticpwn/plugins/elasticsearch-plugin.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 11 | 12 | "go.mongodb.org/mongo-driver/bson/primitive" 13 | "go.mongodb.org/mongo-driver/mongo" 14 | ) 15 | 16 | type ElasticSearchPlugin struct { 17 | InputFilePath string 18 | ThreadsNum int 19 | OutputMode string 20 | OutputFilePath string 21 | MongoUrl string 22 | EsPluginMaxIndices int 23 | EsPluginMaxIndexSize int 24 | } 25 | 26 | // all jsons but in a stringified form 27 | type SingleElasticsearchInstanceScanResult struct { 28 | Id primitive.ObjectID `bson:"_id,omitempty" json:"_id"` 29 | RootUrl string `bson:"rootUrl,omitempty" json:"rootUrl"` 30 | Indices []InterestingIndexInfo `bson:"indices,omitempty" json:"indices"` 31 | IndicesInfo sync.Map `bson:"indicesInfo,omitempty" json:"indicesInfo"` 32 | // sync.Map can't be (un)marshalled 33 | IndicesInfoInJson map[string]interface{} `bson:"indicesInfoInJson,omitempty" json:"indicesInfoInJson"` 34 | InterestingWords []string `bson:"interestingWords,omitempty" json:"interestingWords"` 35 | InterestingInfo *InterestingInfoFromIndexSearch `bson:"interestingInfo,omitempty" json:"interestingInfo"` 36 | HasAtLeastOneIndexSizeOverGB bool `bson:"hasAtLeastOneIndexSizeOverGB,omitempty" json:"hasAtLeastOneIndexSizeOverGB"` 37 | Aliases []interface{} `bson:"aliases,omitempty" json:"aliases"` 38 | Allocations []interface{} `bson:"allocations,omitempty" json:"allocations"` 39 | Nodes []interface{} `bson:"nodes,omitempty" json:"nodes"` 40 | CreatedAt time.Time `bson:"created_at" json:"created_at"` 41 | IsInitialized bool `bson:"isInitialized,omitempty" json:"isInitialized"` 42 | } 43 | 44 | const ( 45 | API_INDICES = "/_cat/indices" 46 | API_ALIASES = "/_cat/aliases" 47 | API_ALLOCATIONS = "/_cat/allocation" 48 | API_NODES = "/_cat/nodes" 49 | API_SEARCH = "/_search" 50 | 51 | // unused for now because they usually contain quite useless info from a security perspective 52 | API_COUNT = "/_cat/count" 53 | API_MASTER = "/_cat/master" 54 | API_NODE_ATTRS = "/_cat/nodeattrs" 55 | API_PENDING_TASKS = "/_cat/pending_tasks" 56 | API_PLUGINS = "/_cat/plugins" 57 | API_TASKS = "/_cat/tasks" 58 | API_TRAINED_MODELS = "/_cat/ml/trained_models" 59 | API_TRANSFORMS = "/_cat/transforms" 60 | ) 61 | 62 | const ( 63 | Q_FORMAT_JSON = "format=json" 64 | Q_SIZE_X = "size={INDEX_SIZE}" 65 | ) 66 | 67 | var Q_SIZE_X_FORMAT_JSON = fmt.Sprintf(strings.Join([]string{ 68 | Q_FORMAT_JSON, 69 | Q_SIZE_X, 70 | }, "&")) 71 | 72 | // any better way? 73 | func elasticSearchResultSwitch(singleElasticsearchInstanceScanResult *SingleElasticsearchInstanceScanResult, endpoint string, resp string) { 74 | if strings.Contains(endpoint, API_INDICES) { 75 | var jsonArrayResponse []IndexInfo 76 | jsonUnmarshalErr := json.Unmarshal([]byte(resp), &jsonArrayResponse) 77 | 78 | if jsonUnmarshalErr != nil { 79 | EPUtils.EPLogger(fmt.Sprintf("Error while unmarshalling %s", resp)) 80 | return 81 | } 82 | singleElasticsearchInstanceScanResult.Indices = ProcessInterestingIndices(jsonArrayResponse) 83 | 84 | return 85 | } 86 | 87 | var jsonArrayResponse []interface{} 88 | jsonUnmarshalErr := json.Unmarshal([]byte(resp), &jsonArrayResponse) 89 | 90 | if jsonUnmarshalErr != nil { 91 | EPUtils.EPLogger(fmt.Sprintf("Error while unmarshalling %s", resp)) 92 | } 93 | 94 | if jsonUnmarshalErr != nil { 95 | return 96 | } 97 | // []interface{} 98 | 99 | switch { 100 | case strings.Contains(endpoint, API_ALIASES): 101 | { 102 | singleElasticsearchInstanceScanResult.Aliases = jsonArrayResponse 103 | } 104 | case strings.Contains(endpoint, API_ALLOCATIONS): 105 | { 106 | singleElasticsearchInstanceScanResult.Allocations = jsonArrayResponse 107 | } 108 | case strings.Contains(endpoint, API_NODES): 109 | { 110 | singleElasticsearchInstanceScanResult.Nodes = jsonArrayResponse 111 | } 112 | } 113 | } 114 | 115 | func (elasticSearchPlugin *ElasticSearchPlugin) requestAllAPIs(url string, singleElasticsearchInstanceScanResult *SingleElasticsearchInstanceScanResult) { 116 | endpoints := []string{ 117 | // just make sure you requeset all indices 118 | fmt.Sprintf("%s?%s&size=1000", API_INDICES, Q_FORMAT_JSON), 119 | fmt.Sprintf("%s?%s", API_ALIASES, Q_FORMAT_JSON), 120 | fmt.Sprintf("%s?%s", API_ALLOCATIONS, Q_FORMAT_JSON), 121 | fmt.Sprintf("%s?%s", API_NODES, Q_FORMAT_JSON), 122 | } 123 | wg := sync.WaitGroup{} 124 | var validHTTPRequestCount EPUtils.Count32 = 0 125 | for _, endpoint := range endpoints { 126 | wg.Add(1) 127 | go func(endpoint string) { 128 | defer wg.Done() 129 | var ( 130 | resp string 131 | err error 132 | ) 133 | var finalUrl = fmt.Sprintf("%s%s", url, endpoint) 134 | EPUtils.EPLogger(fmt.Sprintf("Requesting %s\n", finalUrl)) 135 | resp, _, err = EPUtils.SendFailSafeHTTPRequest(finalUrl, 15, false, map[string]string{}, "GET") 136 | if err != nil { 137 | return 138 | } 139 | elasticSearchResultSwitch(singleElasticsearchInstanceScanResult, endpoint, resp) 140 | validHTTPRequestCount.Inc() 141 | }(endpoint) 142 | 143 | } 144 | wg.Wait() 145 | singleElasticsearchInstanceScanResult.IsInitialized = validHTTPRequestCount > 0 146 | } 147 | 148 | // outputs something like 123.123.123.123:9200/index_name/_search?format=json&size=1000 149 | func (elasticSearchPlugin *ElasticSearchPlugin) buildElasticSearchIndexSearchAPI(rootUrl string, indexName string) string { 150 | queryString := strings.ReplaceAll(Q_SIZE_X_FORMAT_JSON, `{INDEX_SIZE}`, fmt.Sprintf("%v", elasticSearchPlugin.EsPluginMaxIndexSize)) 151 | return fmt.Sprintf( 152 | "%s/%s%s?%s", 153 | rootUrl, 154 | indexName, 155 | API_SEARCH, 156 | queryString, 157 | ) 158 | } 159 | 160 | func (elasticSearchPlugin *ElasticSearchPlugin) searchSingleIndexInfo( 161 | indexName string, 162 | singleElasticsearchInstanceScanResult *SingleElasticsearchInstanceScanResult, 163 | ) string { 164 | getIndexEndpoint := elasticSearchPlugin.buildElasticSearchIndexSearchAPI(singleElasticsearchInstanceScanResult.RootUrl, indexName) 165 | EPUtils.EPLogger(fmt.Sprintf("Requesting %s", getIndexEndpoint)) 166 | searchIndexResult, _, searchIndexResultErr := EPUtils.SendFailSafeHTTPRequest(getIndexEndpoint, 30, false, map[string]string{}, "GET") 167 | if searchIndexResultErr != nil { 168 | EPUtils.EPLogger(fmt.Sprintf("Error in requesting index %s from %s", indexName, singleElasticsearchInstanceScanResult.RootUrl)) 169 | 170 | return "" 171 | } 172 | var jsonArrayResponse interface{} 173 | jsonUnmarshalErr := json.Unmarshal([]byte(searchIndexResult), &jsonArrayResponse) 174 | if jsonUnmarshalErr != nil { 175 | EPUtils.EPLogger(fmt.Sprintf("Error while unmarshalling result from %s", getIndexEndpoint)) 176 | fmt.Println(searchIndexResultErr) 177 | return "" 178 | } 179 | 180 | // each index is unique 181 | // https://stackoverflow.com/questions/45585589/golang-fatal-error-concurrent-map-read-and-map-write/45585833 182 | singleElasticsearchInstanceScanResult.IndicesInfo.Store(indexName, jsonArrayResponse) 183 | return searchIndexResult 184 | } 185 | 186 | func (elasticSearchPlugin *ElasticSearchPlugin) scanInterestingIndices( 187 | singleElasticsearchInstanceScanResult *SingleElasticsearchInstanceScanResult, 188 | ) { 189 | EPUtils.EPLogger(fmt.Sprintf("%s has valid indices. Will query them all", singleElasticsearchInstanceScanResult.RootUrl)) 190 | 191 | mu := &sync.Mutex{} 192 | getIndicesWg := sync.WaitGroup{} 193 | concurrentGoroutines := make(chan struct{}, 5) 194 | var maxIndicesFromSingleElasticsearchInstanceScanResult []InterestingIndexInfo 195 | maxIndicesNum := elasticSearchPlugin.EsPluginMaxIndices 196 | if len(singleElasticsearchInstanceScanResult.Indices) > maxIndicesNum { 197 | EPUtils.EPLogger(fmt.Sprintf("%s has number of indices more than %d. Will only request first %d indices as specified in -max-i option", singleElasticsearchInstanceScanResult.RootUrl, maxIndicesNum, maxIndicesNum)) 198 | maxIndicesFromSingleElasticsearchInstanceScanResult = singleElasticsearchInstanceScanResult.Indices[:maxIndicesNum] 199 | } else { 200 | maxIndicesFromSingleElasticsearchInstanceScanResult = singleElasticsearchInstanceScanResult.Indices 201 | } 202 | 203 | for _, indexInfo := range maxIndicesFromSingleElasticsearchInstanceScanResult { 204 | getIndicesWg.Add(1) 205 | go func(indexInfo InterestingIndexInfo) { 206 | defer getIndicesWg.Done() 207 | concurrentGoroutines <- struct{}{} 208 | // 123.123.123.123/example-index/_search?format=json&size=1000&pretty=true 209 | // don't store uninteresting index names 210 | searchIndexResult := elasticSearchPlugin.searchSingleIndexInfo( 211 | indexInfo.Index, 212 | singleElasticsearchInstanceScanResult, 213 | ) 214 | 215 | if searchIndexResult != "" { 216 | ProcessInterestingInfoAndWordsThreadSafely(mu, singleElasticsearchInstanceScanResult, searchIndexResult) 217 | } 218 | <-concurrentGoroutines 219 | }(indexInfo) 220 | } 221 | getIndicesWg.Wait() 222 | EPUtils.EPLogger(fmt.Sprintf("Finished querying indices information for %s", singleElasticsearchInstanceScanResult.RootUrl)) 223 | } 224 | 225 | func (elasticSearchPlugin *ElasticSearchPlugin) scanSingleElasticsearchInstance(url string) *SingleElasticsearchInstanceScanResult { 226 | _, _, errFromRootUrl := EPUtils.SendFailSafeHTTPRequest(url, 10, true, map[string]string{}, "GET") 227 | 228 | if errFromRootUrl != nil { 229 | return &SingleElasticsearchInstanceScanResult{ 230 | Id: primitive.NewObjectID(), 231 | CreatedAt: time.Now(), 232 | IsInitialized: false, 233 | RootUrl: url, 234 | } 235 | } else { 236 | EPUtils.EPLogger(fmt.Sprintf("%s is a working elasticSearch instance\n", url)) 237 | } 238 | singleElasticsearchInstanceScanResult := &SingleElasticsearchInstanceScanResult{ 239 | Id: primitive.NewObjectID(), 240 | CreatedAt: time.Now(), 241 | IndicesInfo: sync.Map{}, 242 | RootUrl: url, 243 | } 244 | elasticSearchPlugin.requestAllAPIs(url, singleElasticsearchInstanceScanResult) 245 | 246 | if singleElasticsearchInstanceScanResult.Indices == nil { 247 | EPUtils.EPLogger(fmt.Sprintf("Failed to get indices from %v\n", url)) 248 | return singleElasticsearchInstanceScanResult 249 | } 250 | singleElasticsearchInstanceScanResult.HasAtLeastOneIndexSizeOverGB = CheckOverGBIndexExistence(singleElasticsearchInstanceScanResult.Indices) 251 | elasticSearchPlugin.scanInterestingIndices(singleElasticsearchInstanceScanResult) 252 | 253 | return singleElasticsearchInstanceScanResult 254 | } 255 | 256 | var elasticSearchPluginFileWriteMutex sync.Mutex 257 | 258 | func (elasticSearchPlugin *ElasticSearchPlugin) outputSingleElasticsearchInstanceScanResult(singleElasticsearchInstanceScanResult *SingleElasticsearchInstanceScanResult, elasticSearchCollection *mongo.Collection) { 259 | // sync.Map can't be (un)marshalled 260 | singleElasticsearchInstanceScanResult.IndicesInfoInJson = EPUtils.ConvertSyncMapToMap(singleElasticsearchInstanceScanResult.IndicesInfo) 261 | singleElasticsearchInstanceScanResultMarshalled, singleElasticsearchInstanceScanResultMarshalErr := json.Marshal(singleElasticsearchInstanceScanResult) 262 | 263 | if singleElasticsearchInstanceScanResultMarshalErr != nil { 264 | EPUtils.EPLogger(fmt.Sprintf("Error while marshalling info about %s\n", singleElasticsearchInstanceScanResult.RootUrl)) 265 | fmt.Println(singleElasticsearchInstanceScanResultMarshalErr) 266 | } 267 | 268 | switch elasticSearchPlugin.OutputMode { 269 | case "json", "plain": 270 | { 271 | AppendScanResultToJSONFileWithNewline(elasticSearchPlugin.OutputFilePath, singleElasticsearchInstanceScanResultMarshalled, &elasticSearchPluginFileWriteMutex) 272 | } 273 | case "mongo": 274 | { 275 | if elasticSearchCollection == nil { 276 | panic("mongo option was specified but elasticSearchCollection is nil") 277 | } 278 | InsertSingleScanResultToMongo(elasticSearchCollection, singleElasticsearchInstanceScanResult, singleElasticsearchInstanceScanResult.RootUrl) 279 | 280 | } 281 | } 282 | } 283 | 284 | func (elasticSearchPlugin *ElasticSearchPlugin) Run(urls []string, elasticSearchCollection *mongo.Collection) { 285 | var finishedGoRoutineCount EPUtils.Count32 286 | concurrentGoroutines := make(chan struct{}, elasticSearchPlugin.ThreadsNum) 287 | var wg sync.WaitGroup 288 | for _, url := range urls { 289 | wg.Add(1) 290 | go func(url string, elasticSearchCollection *mongo.Collection) { 291 | defer wg.Done() 292 | concurrentGoroutines <- struct{}{} 293 | singleElasticsearchInstanceScanResult := elasticSearchPlugin.scanSingleElasticsearchInstance(url) 294 | elasticSearchPlugin.outputSingleElasticsearchInstanceScanResult(singleElasticsearchInstanceScanResult, elasticSearchCollection) 295 | finishedGoRoutineCount.Inc() 296 | <-concurrentGoroutines 297 | }(url, elasticSearchCollection) 298 | } 299 | go func() { 300 | for { 301 | time.Sleep(time.Duration(2 * time.Second)) 302 | EPUtils.EPLogger(fmt.Sprintf("%d/%d of URLs completed\n", finishedGoRoutineCount.Get(), len(urls))) 303 | } 304 | }() 305 | wg.Wait() 306 | EPUtils.EPLogger(fmt.Sprintf("%d/%d of URLs completed\n", len(urls), len(urls))) 307 | EPUtils.EPLogger("Scan finished") 308 | } 309 | 310 | func (elasticSearchPlugin *ElasticSearchPlugin) PostProcess() { 311 | if elasticSearchPlugin.OutputMode != "json" { 312 | return 313 | } 314 | EPUtils.ConvertJSONObjectsToJSONArray(elasticSearchPlugin.OutputFilePath) 315 | } 316 | -------------------------------------------------------------------------------- /elasticpwn/plugins/kibana-plugin.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | EPLookup_addrs "github.com/9oelM/elasticpwn/elasticpwn/lookup-addrs" 11 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 12 | 13 | "go.mongodb.org/mongo-driver/bson/primitive" 14 | "go.mongodb.org/mongo-driver/mongo" 15 | ) 16 | 17 | // Scraps data from an open Kibana instance using webdriver 18 | type KibanaPlugin struct { 19 | InputFilePath string 20 | ThreadsNum int 21 | OutputMode string 22 | OutputFilePath string 23 | MongoUrl string 24 | MaxIndices int 25 | MaxIndexSize int 26 | } 27 | 28 | type IpInfo struct { 29 | SubjectUrls string `bson:"subjectUrls,omitempty" json:"subjectUrls"` 30 | Organizations string `bson:"organizations,omitempty" json:"organizations"` 31 | // ex. "amazonaws.com." 32 | CloudHostingProvider string `bson:"cloudHostingProvider,omitempty" json:"cloudHostingProvider"` 33 | Cname string `bson:"cname,omitempty" json:"cname"` 34 | } 35 | 36 | // The interesting data that was stored in a single Kibana instance 37 | type SingleKibanaInstanceScanResult struct { 38 | CreatedAt time.Time `bson:"created_at" json:"created_at"` 39 | Id primitive.ObjectID `bson:"_id,omitempty" json:"_id"` 40 | 41 | // indices string 42 | IsInitialized bool `bson:"isInitialized,omitempty" json:"isInitialized"` 43 | // example: 123.123.123.123:5000 44 | RootUrl string `bson:"rootUrl,omitempty" json:"rootUrl"` 45 | IpInfo *IpInfo 46 | HasAtLeastOneIndexSizeOverGB bool `bson:"hasAtLeastOneIndexSizeOverGB,omitempty" json:"hasAtLeastOneIndexSizeOverGB"` 47 | // only stores indices of interesting names 48 | Indices []InterestingIndexInfo `bson:"indices,omitempty" json:"indices"` 49 | IndicesInfo sync.Map 50 | // sync.Map can't be (un)marshalled 51 | IndicesInfoInJson map[string]interface{} `bson:"indicesInfoInJson,omitempty" json:"indicesInfoInJson"` 52 | InterestingWords []string `bson:"interestingWords,omitempty" json:"interestingWords"` 53 | InterestingInfo *InterestingInfoFromIndexSearch `bson:"interestingInfo,omitempty" json:"interestingInfo"` 54 | } 55 | 56 | type KibanaRequests struct { 57 | GET_root string 58 | GET_indices string 59 | } 60 | 61 | const CONSOLE_PATH = "app/kibana#/dev_tools/console?_g=()" 62 | 63 | type KibanaGetRequests struct { 64 | indices string 65 | indexSearch string 66 | } 67 | 68 | // some versions have slightly different APIs 69 | type KibanaAPI struct { 70 | get *KibanaGetRequests 71 | } 72 | 73 | // Working versions: 74 | // 5.2.1 75 | var kibanaVer5_2_1 = &KibanaAPI{ 76 | get: &KibanaGetRequests{ 77 | indices: "api/console/proxy?uri=_cat%2Findices%3Fformat%3Djson", 78 | indexSearch: "api/console/proxy?uri={INDEX_NAME}%2F_search%3Fformat%3Djson%26size%3D{INDEX_SIZE}", 79 | }, 80 | } 81 | 82 | // Working versions: 83 | // 5.6.16 84 | // 6.2.4 85 | // 6.2.2 86 | // 7.15.0 87 | var kibanaVer7_15_0 = &KibanaAPI{ 88 | get: &KibanaGetRequests{ 89 | // "api/console/proxy?path=%2F_cat%2Findices%3Fformat%3Djson&method=GET" 90 | indices: "api/console/proxy?path=%2F_cat%2Findices%3Fformat%3Djson&method=GET", 91 | indexSearch: "api/console/proxy?path=%2F{INDEX_NAME}%2F_search%3Fformat%3Djson%26size%3D{INDEX_SIZE}&method=GET", 92 | }, 93 | } 94 | 95 | func (kpAPI *KibanaAPI) buildKibanaIndexSearchAPI(rootUrl string, indexName string, indexSize int) string { 96 | builtAPI := strings.Replace(kpAPI.get.indexSearch, `{INDEX_NAME}`, indexName, 1) 97 | builtAPI = strings.Replace(builtAPI, `{INDEX_SIZE}`, fmt.Sprintf("%v", indexSize), 1) 98 | 99 | return fmt.Sprintf("%s/%s", rootUrl, builtAPI) 100 | } 101 | 102 | var kibanaHeader = map[string]string{ 103 | // kibana requires this useless header to be set: https://discuss.elastic.co/t/where-can-i-get-the-correct-kbn-xsrf-value-for-my-plugin-http-requests/158725 104 | "kbn-xsrf": "_", 105 | "Content-Type": "application/json", 106 | "User-Agent": "curl/7.0.0", 107 | "Accept": "*/*", 108 | } 109 | 110 | // to see if we need to abort early because the instance is not up at all 111 | // returns true if unhealthy 112 | func (kp *KibanaPlugin) checkIsInstanceDown(rootUrl string) bool { 113 | // you need to insert kibana headers even for the index page 114 | anything, statusCode, _ := EPUtils.SendFailSafeHTTPRequest(rootUrl, 15, false, kibanaHeader, "GET") 115 | 116 | return anything == "" && statusCode != 200 117 | } 118 | 119 | // returns nil if could not get indices 120 | func (kp *KibanaPlugin) getIndices(rootUrl string) []IndexInfo { 121 | EPUtils.EPLogger(fmt.Sprintf("%v has a working Kibana frontend", rootUrl)) 122 | 123 | var indicesArray []IndexInfo 124 | allPossibleGetIndicesRequests := []string{ 125 | fmt.Sprintf("%s/%s", rootUrl, kibanaVer5_2_1.get.indices), 126 | fmt.Sprintf("%s/%s", rootUrl, kibanaVer7_15_0.get.indices), 127 | } 128 | for _, req := range allPossibleGetIndicesRequests { 129 | var indicesArrayInJsonString string = "" 130 | var statusCode int 131 | switch { 132 | case strings.HasSuffix(req, kibanaVer7_15_0.get.indices): 133 | { 134 | // recent versions of kibana has this weird system where you need to POST in order to GET through proxy 135 | indicesArrayInJsonString, statusCode, _ = EPUtils.SendFailSafeHTTPRequest(req, 15, false, kibanaHeader, "POST") 136 | break 137 | } 138 | case strings.HasSuffix(req, kibanaVer5_2_1.get.indices): 139 | { 140 | indicesArrayInJsonString, statusCode, _ = EPUtils.SendFailSafeHTTPRequest(req, 15, false, kibanaHeader, "GET") 141 | break 142 | } 143 | } 144 | 145 | jsonUnmarshalErr := json.Unmarshal([]byte(indicesArrayInJsonString), &indicesArray) 146 | 147 | if jsonUnmarshalErr == nil && statusCode != 404 { 148 | break 149 | } else { 150 | indicesArray = nil 151 | } 152 | } 153 | 154 | return indicesArray 155 | } 156 | 157 | func (kp *KibanaPlugin) scanInterestingIndices( 158 | singleKibanaInstanceScanResult *SingleKibanaInstanceScanResult, 159 | ) { 160 | wg := sync.WaitGroup{} 161 | // setting this number high will likely cause a panic (too many files open) and high memory usage 162 | // because there could be many indices 163 | concurrentGoroutines := make(chan struct{}, 5) 164 | mu := &sync.Mutex{} 165 | 166 | maxIndicesNum := kp.MaxIndices 167 | if len(singleKibanaInstanceScanResult.Indices) <= maxIndicesNum { 168 | maxIndicesNum = len(singleKibanaInstanceScanResult.Indices) 169 | } 170 | 171 | // some low versions of kibana do not support /mget method (request multiple indices with one call), so just request each index respectively 172 | for _, indexInfo := range singleKibanaInstanceScanResult.Indices[:maxIndicesNum] { 173 | wg.Add(1) 174 | 175 | go func(indexInfo InterestingIndexInfo) { 176 | defer wg.Done() 177 | concurrentGoroutines <- struct{}{} 178 | 179 | var indexInfoObject map[string]interface{} 180 | allPossibleGetIndexSearchRequests := []string{ 181 | kibanaVer7_15_0.buildKibanaIndexSearchAPI(singleKibanaInstanceScanResult.RootUrl, indexInfo.Index, kp.MaxIndexSize), 182 | kibanaVer5_2_1.buildKibanaIndexSearchAPI(singleKibanaInstanceScanResult.RootUrl, indexInfo.Index, kp.MaxIndexSize), 183 | } 184 | for _, req := range allPossibleGetIndexSearchRequests { 185 | var indexInfoObjectInJsonString string 186 | var statusCode int 187 | switch req { 188 | case allPossibleGetIndexSearchRequests[0]: 189 | { 190 | // recent versions of kibana has this weird system where you need to POST in order to GET through proxy 191 | // keep timeout reasonably low, otherwise will cause memory usage spike in low-end machines 192 | indexInfoObjectInJsonString, statusCode, _ = EPUtils.SendFailSafeHTTPRequest(req, 30, false, kibanaHeader, "POST") 193 | 194 | break 195 | } 196 | case allPossibleGetIndexSearchRequests[1]: 197 | { 198 | indexInfoObjectInJsonString, statusCode, _ = EPUtils.SendFailSafeHTTPRequest(req, 30, false, kibanaHeader, "GET") 199 | 200 | break 201 | } 202 | } 203 | 204 | jsonUnmarshalErr := json.Unmarshal([]byte(indexInfoObjectInJsonString), &indexInfoObject) 205 | 206 | if jsonUnmarshalErr == nil && strings.TrimSpace(indexInfoObjectInJsonString) != "" && statusCode != 404 { 207 | singleKibanaInstanceScanResult.IndicesInfo.Store(indexInfo.Index, indexInfoObject) 208 | 209 | ProcessInterestingInfoAndWordsThreadSafely(mu, singleKibanaInstanceScanResult, indexInfoObjectInJsonString) 210 | break 211 | } else { 212 | indexInfoObject = nil 213 | } 214 | } 215 | 216 | <-concurrentGoroutines 217 | }(indexInfo) 218 | } 219 | 220 | wg.Wait() 221 | } 222 | 223 | // rootUrl example: 123.123.123.123:5601 224 | func (kp *KibanaPlugin) scanSingleKibanaInstance(rootUrl string) *SingleKibanaInstanceScanResult { 225 | singleKibanaInstanceScanResult := &SingleKibanaInstanceScanResult{ 226 | IsInitialized: false, 227 | RootUrl: rootUrl, 228 | Id: primitive.NewObjectID(), 229 | CreatedAt: time.Now(), 230 | IndicesInfo: sync.Map{}, 231 | } 232 | 233 | isInstanceDown := kp.checkIsInstanceDown(rootUrl) 234 | if isInstanceDown { 235 | EPUtils.EPLogger(fmt.Sprintf("%v is down\n", rootUrl)) 236 | 237 | return singleKibanaInstanceScanResult 238 | } 239 | 240 | allIndices := kp.getIndices(rootUrl) 241 | if allIndices == nil { 242 | EPUtils.EPLogger(fmt.Sprintf("Failed to get indices from %v\n", rootUrl)) 243 | return singleKibanaInstanceScanResult 244 | } 245 | 246 | interestingIndices := ProcessInterestingIndices(allIndices) 247 | if interestingIndices == nil { 248 | EPUtils.EPLogger(fmt.Sprintf("Failed to get interesting indices from %v\n", rootUrl)) 249 | return singleKibanaInstanceScanResult 250 | } 251 | singleKibanaInstanceScanResult.HasAtLeastOneIndexSizeOverGB = CheckOverGBIndexExistence(interestingIndices) 252 | singleKibanaInstanceScanResult.IsInitialized = true 253 | 254 | singleKibanaInstanceScanResult.Indices = interestingIndices 255 | 256 | kp.scanInterestingIndices(singleKibanaInstanceScanResult) 257 | 258 | return singleKibanaInstanceScanResult 259 | } 260 | 261 | var kibanaPluginFileWriteMutex sync.Mutex 262 | 263 | func (kp *KibanaPlugin) outputSingleKibanaInstanceScanResult( 264 | singleKibanaInstanceScanResult *SingleKibanaInstanceScanResult, 265 | kibanaCollection *mongo.Collection, 266 | ) { 267 | indicesInfoMap := EPUtils.ConvertSyncMapToMap(singleKibanaInstanceScanResult.IndicesInfo) 268 | singleKibanaInstanceScanResult.IndicesInfoInJson = indicesInfoMap 269 | 270 | singleKibanaInstanceScanResultMarshalled, singleKibanaInstanceScanResultMarshalErr := json.Marshal(singleKibanaInstanceScanResult) 271 | 272 | if singleKibanaInstanceScanResultMarshalErr != nil { 273 | EPUtils.EPLogger(fmt.Sprintf("Failed to marshall scan result for %v", singleKibanaInstanceScanResult.RootUrl)) 274 | 275 | return 276 | } 277 | 278 | switch kp.OutputMode { 279 | case "json", "plain": 280 | { 281 | AppendScanResultToJSONFileWithNewline(kp.OutputFilePath, singleKibanaInstanceScanResultMarshalled, &kibanaPluginFileWriteMutex) 282 | } 283 | case "mongo": 284 | { 285 | if kibanaCollection == nil { 286 | panic("mongo option was given, but collection is nil") 287 | } 288 | InsertSingleScanResultToMongo(kibanaCollection, singleKibanaInstanceScanResult, singleKibanaInstanceScanResult.RootUrl) 289 | } 290 | default: 291 | { 292 | panic(fmt.Sprintf("Unrecognized output mode: %v", kp.OutputMode)) 293 | } 294 | } 295 | } 296 | 297 | func (kp *KibanaPlugin) scanKibanaInstanceAndIpInfo(url string) *SingleKibanaInstanceScanResult { 298 | singleKibanaInstanceScanResultChan := make(chan *SingleKibanaInstanceScanResult) 299 | ipInfoChan := make(chan *IpInfo) 300 | wg := sync.WaitGroup{} 301 | wg.Add(1) 302 | go func(url string) { 303 | singleKibanaInstanceScanResult := kp.scanSingleKibanaInstance(url) 304 | singleKibanaInstanceScanResultChan <- singleKibanaInstanceScanResult 305 | }(url) 306 | wg.Add(1) 307 | go func(url string) { 308 | cloudHostingProvider, subjectUrls, organizations, cname := EPLookup_addrs.GetIpInfo(url) 309 | ipInfoChan <- &IpInfo{ 310 | CloudHostingProvider: cloudHostingProvider, 311 | SubjectUrls: subjectUrls, 312 | Organizations: organizations, 313 | Cname: cname, 314 | } 315 | }(url) 316 | singleKibanaInstanceScanResult, ipInfo := <-singleKibanaInstanceScanResultChan, <-ipInfoChan 317 | singleKibanaInstanceScanResult.IpInfo = ipInfo 318 | 319 | return singleKibanaInstanceScanResult 320 | } 321 | 322 | func (kp *KibanaPlugin) Run(urls []string, kibanaCollection *mongo.Collection) { 323 | wg := sync.WaitGroup{} 324 | concurrentGoroutines := make(chan struct{}, kp.ThreadsNum) 325 | var finishedGoRoutineCount EPUtils.Count32 326 | 327 | for _, url := range urls { 328 | wg.Add(1) 329 | go func(url string) { 330 | defer wg.Done() 331 | concurrentGoroutines <- struct{}{} 332 | singleKibanaInstanceScanResult := kp.scanKibanaInstanceAndIpInfo(url) 333 | 334 | kp.outputSingleKibanaInstanceScanResult(singleKibanaInstanceScanResult, kibanaCollection) 335 | finishedGoRoutineCount.Inc() 336 | <-concurrentGoroutines 337 | }(url) 338 | } 339 | go func() { 340 | for { 341 | time.Sleep(time.Duration(1 * time.Second)) 342 | EPUtils.EPLogger(fmt.Sprintf("%d/%d of URLs completed\n", finishedGoRoutineCount.Get(), len(urls))) 343 | } 344 | }() 345 | wg.Wait() 346 | } 347 | 348 | func (kp *KibanaPlugin) PostProcess() { 349 | if kp.OutputMode != "json" { 350 | return 351 | } 352 | EPUtils.ConvertJSONObjectsToJSONArray(kp.OutputFilePath) 353 | } 354 | -------------------------------------------------------------------------------- /elasticpwn/plugins/plugin.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | type ElasticpwnPlugin interface { 4 | Run() interface{} 5 | } 6 | -------------------------------------------------------------------------------- /elasticpwn/plugins/report-generate-plugin.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | 12 | EPUtils "github.com/9oelM/elasticpwn/elasticpwn/util" 13 | ) 14 | 15 | // Keep the behavior of this plugin as idempotent as possible 16 | // to avoid any weird errors 17 | type ReportGeneratePlugin struct { 18 | MongoUrl string 19 | CollectionName string 20 | ServerRootUrl string 21 | } 22 | 23 | func (rp *ReportGeneratePlugin) cmdInDir(cwd string, rootCommand string, arg ...string) { 24 | cmd := exec.Command(rootCommand, arg...) 25 | cmd.Dir = filepath.FromSlash(cwd) 26 | output, err := cmd.CombinedOutput() 27 | if err != nil { 28 | log.Println(string(output)) 29 | log.Fatal(err) 30 | } 31 | log.Println(string(output)) 32 | } 33 | 34 | func (rp *ReportGeneratePlugin) Run() { 35 | unixHomePath, unixHomePathOk := os.LookupEnv("HOME") 36 | windowsHomePath, windowsHomePathOk := os.LookupEnv("HOMEPATH") 37 | 38 | finalHomePath := "" 39 | if unixHomePathOk { 40 | finalHomePath = unixHomePath 41 | } else if windowsHomePathOk { 42 | finalHomePath = windowsHomePath 43 | } else { 44 | log.Fatal("Either of environment variables \"HOME\" or \"HOMEPATH\" is not defined. Define your ~ directory as that and try again.") 45 | } 46 | 47 | elasticpwnRootDir := filepath.FromSlash(fmt.Sprintf("%v/%v", finalHomePath, ".elasticpwn")) 48 | elasticpwnRepoCloneDir := filepath.FromSlash(fmt.Sprintf("%v/%v", elasticpwnRootDir, "elasticpwn")) 49 | elasticpwnReportFrontendDir := filepath.FromSlash(fmt.Sprintf("%v/report/frontend", elasticpwnRepoCloneDir)) 50 | 51 | log.Println(fmt.Sprintf("Trying to create a directory at %v", elasticpwnRootDir)) 52 | err := os.Mkdir(elasticpwnRootDir, 0755) 53 | 54 | if !errors.Is(os.ErrExist, err) { 55 | log.Println(fmt.Sprintf("%v already exists. Continuing.", elasticpwnRootDir)) 56 | } else if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | err = os.RemoveAll(elasticpwnRepoCloneDir) 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | output, err := exec.Command("git", "clone", "https://github.com/9oelM/elasticpwn.git", elasticpwnRepoCloneDir).CombinedOutput() 65 | if err != nil { 66 | log.Fatal(err) 67 | } 68 | log.Println(string(output)) 69 | 70 | nvmrcContent := string(EPUtils.ReadFile(filepath.FromSlash(fmt.Sprintf("%v/.nvmrc", elasticpwnReportFrontendDir)))) 71 | nodeVOutput, err := exec.Command("node", "-v").CombinedOutput() 72 | if err != nil { 73 | log.Fatal(err) 74 | } 75 | currentNodeVersion := strings.ReplaceAll(string(nodeVOutput), "\n", "") 76 | 77 | if strings.TrimSpace(nvmrcContent) != strings.TrimSpace(currentNodeVersion) { 78 | log.Println(fmt.Sprintf("elasticpwn recommends node version of %v, but the current node version is %v. Continuing regardless. If you are seeing unexpected things, switch to node version of %v and try again.", nvmrcContent, currentNodeVersion, nvmrcContent)) 79 | } 80 | 81 | rp.cmdInDir(elasticpwnReportFrontendDir, "npm", "i") 82 | 83 | dotEnvLocalContent := fmt.Sprintf(`MONGODB_URI=%v 84 | COLLECTION_NAME=%v 85 | DB_NAME=ep 86 | SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH=%v 87 | `, rp.MongoUrl, rp.CollectionName, rp.ServerRootUrl) 88 | EPUtils.OverwriteFile(filepath.FromSlash(fmt.Sprintf("%v/.env.local", elasticpwnReportFrontendDir)), dotEnvLocalContent) 89 | 90 | nextConfigJsContent := ` 91 | /** @type {import('next').NextConfig} */ 92 | module.exports = { 93 | reactStrictMode: true, 94 | distDir: 'build', 95 | // https://stackoverflow.com/questions/66137368/next-js-environment-variables-are-undefined-next-js-10-0-5 96 | env: { 97 | SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH: process.env.SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH 98 | }, 99 | // https://github.com/vercel/next.js/discussions/13578 100 | // this needs to be uncommented when npm run build 101 | assetPrefix: '.', 102 | } 103 | ` 104 | EPUtils.OverwriteFile(filepath.FromSlash(fmt.Sprintf("%v/next.config.js", elasticpwnReportFrontendDir)), nextConfigJsContent) 105 | 106 | rp.cmdInDir(elasticpwnReportFrontendDir, "npm", "run", "build") 107 | rp.cmdInDir(elasticpwnReportFrontendDir, "npm", "run", "export") 108 | cwd, err := os.Getwd() 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | fmt.Println(cwd) 113 | err = os.Rename(filepath.FromSlash(fmt.Sprintf("%v/out", elasticpwnReportFrontendDir)), filepath.FromSlash(fmt.Sprintf("%v/report", cwd))) 114 | if err != nil { 115 | log.Fatal(err) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /elasticpwn/plugins/report-view-plugin.go: -------------------------------------------------------------------------------- 1 | package EPPlugins 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | ) 8 | 9 | type ReportViewPlugin struct { 10 | ReportDirectory string 11 | ServeAtPort string 12 | } 13 | 14 | func (rvp *ReportViewPlugin) Run() { 15 | log.Println(fmt.Sprintf("Report being served at http://localhost:%v", rvp.ServeAtPort)) 16 | err := http.ListenAndServe(fmt.Sprintf(":%v", rvp.ServeAtPort), http.FileServer(http.Dir(rvp.ReportDirectory))) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /elasticpwn/util/concurrency.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import "sync/atomic" 4 | 5 | type Count32 int32 6 | 7 | func (c *Count32) Inc() int32 { 8 | return atomic.AddInt32((*int32)(c), 1) 9 | } 10 | 11 | func (c *Count32) Dec() int32 { 12 | return atomic.AddInt32((*int32)(c), -1) 13 | } 14 | 15 | func (c *Count32) Get() int32 { 16 | return atomic.LoadInt32((*int32)(c)) 17 | } 18 | -------------------------------------------------------------------------------- /elasticpwn/util/file.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | ) 12 | 13 | func ReadUrlsFromFile(path_to_url_text_file string) string { 14 | EPLogger(fmt.Sprintf("Reading URLs from %s\n", path_to_url_text_file)) 15 | b := ReadFile(path_to_url_text_file) 16 | urlsArr := strings.Split(strings.TrimSpace(string(b)), "\n") 17 | // @todo trim whitespaces and empty lines 18 | for index, url := range urlsArr { 19 | urlsArr[index] = strings.TrimSuffix(url, "/") 20 | } 21 | return strings.Join(urlsArr, "\n") 22 | } 23 | 24 | // reads a file that's structured like 25 | // {"something": 1 }, 26 | // {"something": 2 }, 27 | // {"something": 3 }, 28 | // and turns it into 29 | // [{"something": 1 }, 30 | // {"something": 2 }, 31 | // {"something": 3 }] 32 | func ConvertJSONObjectsToJSONArray(pathToJsonFile string) { 33 | output, err := ioutil.ReadFile(pathToJsonFile) 34 | ExitOnError(err) 35 | 36 | // this is because we can't take care of the last comma when we finish writing json 37 | withoutLastTrailingComma := strings.TrimSuffix(string(output), ",\n") 38 | // right square bracket of the JSON array 39 | output = append([]byte(withoutLastTrailingComma), "]"...) 40 | 41 | tmpFile := fmt.Sprintf("%s.tmp", pathToJsonFile) 42 | f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) 43 | defer func() { 44 | err := f.Close() 45 | if err != nil { 46 | EPLogger(fmt.Sprintf("Failed to close %v while finalizing\n", pathToJsonFile)) 47 | return 48 | } 49 | }() 50 | 51 | if err != nil { 52 | EPLogger(fmt.Sprintf("Failed to open %v while finalizing\n", pathToJsonFile)) 53 | return 54 | } 55 | 56 | // left square bracket of the JSON array 57 | _, err = f.WriteString("[") 58 | ExitOnError(err) 59 | 60 | _, err = f.Write(output) 61 | ExitOnError(err) 62 | 63 | err = os.Remove(pathToJsonFile) 64 | ExitOnError(err) 65 | 66 | err = os.Rename(tmpFile, pathToJsonFile) 67 | ExitOnError(err) 68 | } 69 | 70 | func ReadFile(path string) []byte { 71 | file, err := os.Open(filepath.FromSlash(path)) 72 | if err != nil { 73 | log.Fatal(err) 74 | } 75 | defer func() { 76 | if err = file.Close(); err != nil { 77 | log.Fatal(err) 78 | } 79 | }() 80 | 81 | b, err := ioutil.ReadAll(file) 82 | if err != nil { 83 | log.Fatal(err) 84 | } 85 | 86 | return b 87 | } 88 | 89 | func OverwriteFile(path string, content string) { 90 | err := os.Remove(filepath.FromSlash(path)) 91 | if !errors.Is(err, os.ErrNotExist) && err != nil { 92 | log.Fatal(err) 93 | } 94 | f, err := os.Create(filepath.FromSlash(path)) 95 | defer func() { 96 | err := f.Close() 97 | if err != nil { 98 | log.Fatal(err) 99 | } 100 | }() 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | f.WriteString(content) 105 | } 106 | -------------------------------------------------------------------------------- /elasticpwn/util/http.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "regexp" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | type RetryReason int8 14 | 15 | const ( 16 | None RetryReason = 0 17 | SHOULD_TRY_HTTPS RetryReason = 1 18 | SHOULD_TRY_HTTP RetryReason = 2 19 | ) 20 | const ( 21 | HTTPS = "https" 22 | HTTP = "http" 23 | ) 24 | 25 | // reuse http client. 26 | // if declared inside the function, 27 | // the memory usage will spike, leading to a forceful exit 28 | var httpClient = http.Client{Transport: &http.Transport{DisableKeepAlives: true}} 29 | 30 | func SendFailSafeHTTPRequest(endpoint string, timeoutsecs int, disableRetries bool, headers map[string]string, method string) (string, int, error) { 31 | var ( 32 | err error 33 | response *http.Response 34 | retries int = 1 35 | retryReason RetryReason = 0 36 | cancelFuncs []context.CancelFunc 37 | ) 38 | for retries > 0 { 39 | var maybeFixedEndpoint = func(retryMode RetryReason, endpoint string) string { 40 | switch retryReason { 41 | case SHOULD_TRY_HTTPS: 42 | EPLogger(fmt.Sprintf("Retrying with https because http did not work: %v", endpoint)) 43 | firstHttpStringOccurenceRegex := regexp.MustCompile("^(.*?)http(.*)$") 44 | return firstHttpStringOccurenceRegex.ReplaceAllString(endpoint, HTTPS) 45 | case SHOULD_TRY_HTTP: 46 | EPLogger(fmt.Sprintf("Retrying with http because https did not work: %v", endpoint)) 47 | firstHttpsStringOccurenceRegex := regexp.MustCompile("^(.*?)https(.*)$") 48 | return firstHttpsStringOccurenceRegex.ReplaceAllString(endpoint, HTTPS) 49 | default: 50 | return endpoint 51 | } 52 | }(retryReason, endpoint) 53 | var maybeFixedEndpointWithPrefix = func(endpoint string) string { 54 | if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { 55 | return fmt.Sprintf("http://%s", endpoint) 56 | } 57 | 58 | return endpoint 59 | }(maybeFixedEndpoint) 60 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutsecs)*time.Second) 61 | cancelFuncs = append(cancelFuncs, cancel) 62 | var req *http.Request 63 | 64 | if method != "GET" && method != "POST" { 65 | panic(fmt.Sprintf("%v is not an accepted http method", method)) 66 | } 67 | req, err = http.NewRequestWithContext(ctx, method, maybeFixedEndpointWithPrefix, nil) 68 | if err != nil { 69 | EPLogger(fmt.Sprintf("Failed to send request to %v\n", maybeFixedEndpointWithPrefix)) 70 | 71 | if disableRetries { 72 | break 73 | } else { 74 | retries -= 1 75 | continue 76 | } 77 | } 78 | for key, header := range headers { 79 | req.Header.Set(key, header) 80 | } 81 | response, err = httpClient.Do(req) 82 | 83 | if err != nil { 84 | EPLogger(fmt.Sprintf("Failed to fetch from %s\n", maybeFixedEndpointWithPrefix)) 85 | if !disableRetries { 86 | EPLogger(fmt.Sprintf("Remaining retries for %s: %d times\n", maybeFixedEndpointWithPrefix, retries)) 87 | } 88 | var errorMessage = fmt.Sprint(err) 89 | if strings.Contains(errorMessage, "server gave HTTP response to HTTPS client") { 90 | EPLogger("Retrying with HTTP instead of HTTPS") 91 | retryReason = SHOULD_TRY_HTTP 92 | } else if strings.Contains(errorMessage, "server gave HTTPS response to HTTP client") { 93 | EPLogger("Retrying with HTTPS instead of HTTP") 94 | retryReason = SHOULD_TRY_HTTPS 95 | } 96 | if disableRetries { 97 | break 98 | } else { 99 | retries -= 1 100 | } 101 | } else { 102 | break 103 | } 104 | } 105 | 106 | if response != nil { 107 | data, err := ioutil.ReadAll(response.Body) 108 | response.Body.Close() 109 | for _, c := range cancelFuncs { 110 | c() 111 | } 112 | 113 | if err != nil { 114 | return "", response.StatusCode, err 115 | } 116 | 117 | return string(data), response.StatusCode, nil 118 | } 119 | for _, c := range cancelFuncs { 120 | c() 121 | } 122 | 123 | return "", -1, err 124 | } 125 | -------------------------------------------------------------------------------- /elasticpwn/util/log.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | ) 8 | 9 | func timeIn(name string) time.Time { 10 | loc, err := time.LoadLocation(name) 11 | if err != nil { 12 | panic(err) 13 | } 14 | return time.Now().In(loc) 15 | } 16 | 17 | func EPLog(l *log.Logger, msg string) { 18 | l.SetPrefix(timeIn("Asia/Seoul").Format("2006-01-02 15:04:05") + " [EP] ") 19 | l.Print(msg) 20 | } 21 | 22 | var L = log.New(os.Stdout, "", 0) 23 | 24 | func EPLogger(message string) { 25 | EPLog(L, message) 26 | } 27 | -------------------------------------------------------------------------------- /elasticpwn/util/mongo.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "go.mongodb.org/mongo-driver/mongo" 8 | "go.mongodb.org/mongo-driver/mongo/options" 9 | "go.mongodb.org/mongo-driver/mongo/readpref" 10 | ) 11 | 12 | // This is a user defined method that returns mongo.Client, 13 | // context.Context, context.CancelFunc and error. 14 | // mongo.Client will be used for further database operation. 15 | // context.Context will be used set deadlines for process. 16 | // context.CancelFunc will be used to cancel context and 17 | // resource associtated with it. 18 | 19 | func connectToMongodb(url string) (*mongo.Client, context.Context, 20 | context.CancelFunc, error) { 21 | // ctx will be used to set deadline for process, here 22 | // deadline will of 30 seconds. 23 | ctx, cancel := context.WithTimeout(context.Background(), 24 | 10*time.Second) 25 | 26 | // mongo.Connect return mongo.Client method 27 | client, err := mongo.Connect(ctx, options.Client().ApplyURI(url)) 28 | return client, ctx, cancel, err 29 | } 30 | 31 | // This is a user defined method that accepts 32 | // mongo.Client and context.Context 33 | // This method used to ping the mongoDB, return error if any. 34 | func ping(client *mongo.Client, ctx context.Context) error { 35 | 36 | // mongo.Client has Ping to ping mongoDB, deadline of 37 | // the Ping method will be determined by cxt 38 | // Ping method return error if any occored, then 39 | // the error can be handled. 40 | if err := client.Ping(ctx, readpref.Primary()); err != nil { 41 | return err 42 | } 43 | EPLogger("Connected to MongoDB") 44 | return nil 45 | } 46 | 47 | func InitMongoConnnection(mongoUrl string) *mongo.Client { 48 | mongoClient, _, cancelConnectToMongodb, mongodbConnectErr := connectToMongodb(mongoUrl) 49 | if mongodbConnectErr != nil { 50 | panic(mongodbConnectErr) 51 | } 52 | defer cancelConnectToMongodb() 53 | pingContext, cancelPing := context.WithTimeout(context.Background(), 54 | 10*time.Second) 55 | defer cancelPing() 56 | pingError := ping(mongoClient, pingContext) 57 | 58 | if pingError != nil { 59 | EPLogger("Failed to connect to mongodb. Check your mongodb config.") 60 | panic(pingError) 61 | } 62 | 63 | return mongoClient 64 | } 65 | -------------------------------------------------------------------------------- /elasticpwn/util/regex.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | var UrlRegex = regexp.MustCompile(`[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)?`) 8 | 9 | // it will match a package name with a hash too, like org.springframework.security.web.authentication.logout.LogoutFilter@7b1e5e55 10 | // although the purpose was to match an email, it is still a good info, so leave it as it is 11 | var EmailRegex = regexp.MustCompile("[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*") 12 | var IpWithOptionalPortRegex = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(:[0-9]+)?`) 13 | var PrivateIpRegexWithOptionalPortRegex = regexp.MustCompile(`^(127\.)|(192\.168\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(::1$)|([fF][cCdD])(:[0-9]+)?`) 14 | 15 | // things like 3.0.0 or org.sonatype.sisu (possibly package names) 16 | var AtLeastTwoDotsInSentenceRegex = regexp.MustCompile(`(\w+)\.(\w+)\.\w*`) 17 | 18 | func FindAllUniquePublicIps(rawString string) []string { 19 | allIps := IpWithOptionalPortRegex.FindAllString(rawString, -1) 20 | var allPublicIps []string 21 | 22 | for _, ip := range allIps { 23 | if len(PrivateIpRegexWithOptionalPortRegex.FindAllString(ip, -1)) == 0 { 24 | allPublicIps = append(allPublicIps, ip) 25 | } 26 | } 27 | 28 | return Unique(allPublicIps) 29 | } 30 | 31 | func FindAllUniqueUrls(rawString string) []string { 32 | allUrls := UrlRegex.FindAllString(rawString, -1) 33 | var allUniqueUrls []string 34 | 35 | for _, url := range allUrls { 36 | // because email addr also gets mistakenly considered as url 37 | if len(EmailRegex.FindAllString(url, -1)) == 0 { 38 | allUniqueUrls = append(allUniqueUrls, url) 39 | } 40 | } 41 | 42 | return Unique(allUniqueUrls) 43 | } 44 | -------------------------------------------------------------------------------- /elasticpwn/util/regex_test.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "log" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | // @todo: add actual/expected vars 10 | func TestEmailRegex(t *testing.T) { 11 | testString := ` 12 | testtestasfewasfaewf@gmail.com 13 | url.url.url.com 14 | a1241242.141241241241249012j40129j 12421poj12p9j12pj412poj4 1op2j4po12 j4po12j po12j 09234 15 | 16 | \\\214j092yth0q234ht80q2h340fh24f0h23fwfhi@gmail.comf243oif24iof 17 | flkawejkfjewkfnklwenfi3o2nfi 3f3nm ifn 3nmm10p92m90fn90 n0n nwalknmvklnsdsfsfasd fdsafsdalkf jlaskdjfklasd jklwelkfklefalkweflkwe jlkawlkj lkj test@hotmail.com test+01@hotmail.com 1231212312312 3213123 18 | ` 19 | 20 | allEmails := EmailRegex.FindAllString(testString, -1) 21 | 22 | if len(allEmails) < 1 { 23 | t.Errorf("Did not find any emails") 24 | } else { 25 | log.Printf("Found these emails: %v\n", strings.Join(allEmails, ",")) 26 | } 27 | } 28 | 29 | // @todo: add actual/expected vars 30 | func TestUrlRegex(t *testing.T) { 31 | testString := ` 32 | testtestasfewasfaewf@gmail.com 33 | url.url.url.com 34 | a1241242.141241241241249012j40129j 12421poj12p9j12pj412poj4 1op2j4po12 j4po12j po12j 09234 35 | 36 | https://example.com 37 | 38 | example.com 39 | 40 | \\\214j092yth0q234ht80q2https://2222.comh340fh24f0h23fwfhi@gmail.comf243oif24iof 41 | flkawejkfje example.com https://qqqqqqq.com wkfnklwenfi3o2nfi 3f3nm ifn 3nmm10p92m90fn90 n0n nwalknmvklnsdsfsfasd fdsafsdalkf jlaskdjfklasd jklwelkfklefalkweflkwe jlkawlkj lkj test@hotmail.com test+01@hotmail.com 1231212312312 3213123 https://qqqqqqq.com https://qqqqqqq.com 42 | ` 43 | 44 | allUrls := FindAllUniqueUrls(testString) 45 | 46 | if len(allUrls) < 1 { 47 | t.Errorf("Did not find any urls") 48 | } else { 49 | log.Printf("Found these urls: %v\n", strings.Join(allUrls, ",")) 50 | } 51 | } 52 | 53 | // only ipv4 for now 54 | func TestFindAllUniquePublicIps(t *testing.T) { 55 | expectedPublicIps := []string{ 56 | "123.123.123.123:500", 57 | "123.123.124.14", 58 | "200.200.200.200:12414", 59 | "200.200.200.200", 60 | } 61 | privateIps := []string{ 62 | "192.168.2.1", 63 | "192.168.10.2", 64 | "10.0.0.0", 65 | "10.255.255.255", 66 | "172.16.0.0", 67 | "172.30.222.222", 68 | } 69 | 70 | rawString := `testtestasfewasfaewf@gmail.com 71 | url.url.url.com 72 | a1241242.141241241241249012j40129j 12421poj12p9j12pj412poj4 1op2j4po12 j4po12j po12j 09234 73 | 74 | https://example.com 75 | 76 | example.com 77 | 78 | \\\214j092yth0q234ht80q2https://2222.comh340fh24f0h23fwfhi@gmail.comf243oif24iof 79 | flkawejkfje example.com https://qqqqqqq.com wkfnklwenfi3o2nfi 3f3nm ifn 3nmm10p92m90fn90 n0n nwalknmvklnsdsfsfasd fdsafsdalkf jlaskdjfklasd jklwelkfklefalkweflkwe jlkawlkj lkj test@hotmail.com test+01@hotmail.com 1231212312312 3213123 80 | 81 | 82 | 83 | 84 | ` + expectedPublicIps[0] + ` 85 | 999.999.999.999 86 | asdfasdf999.999.999.999 87 | 100101001101010.100.0.10asdfasdf 88 | 89 | 90 | 100101001101010.100.0.10 91 | ` + expectedPublicIps[1] + ` 92 | "WEF" 12412094571209571.12512.5125091u012740284091280941.325.1254123.5213.36624624.74,324p4350234850923841234023842343oirh132109h 0912hr9012h sadilkvcdsalkvnsakvl19 1890384019284091 490902 1029209aweljfkweajlk wajlkej 93 | ` + expectedPublicIps[2] + ` 94 | sakldjflawejiofjaewiofjweiofjwejfoiaweawefoiawejfoiawheoifh 95 | ` + expectedPublicIps[3] + " " + strings.Join(privateIps, " aaaa ") 96 | 97 | actualPublicIps := FindAllUniquePublicIps(rawString) 98 | 99 | for _, ip := range actualPublicIps { 100 | if Contains(ip, privateIps) != -1 { 101 | t.Errorf("%s is not a public ip but was considered to be so", ip) 102 | } else if Contains(ip, expectedPublicIps) == -1 { 103 | t.Errorf("%s is a public ip but was not considered to be so", ip) 104 | } 105 | } 106 | log.Printf("Found these public ipv4 urls: %s", strings.Join(actualPublicIps, ",")) 107 | } 108 | -------------------------------------------------------------------------------- /elasticpwn/util/util.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | ) 7 | 8 | func ConvertSyncMapToMap(sMap sync.Map) map[string]interface{} { 9 | tmpMap := make(map[string]interface{}) 10 | sMap.Range(func(k, v interface{}) bool { 11 | tmpMap[k.(string)] = v 12 | return true 13 | }) 14 | return tmpMap 15 | } 16 | 17 | func containsXWith(cb func(a string, b string) bool) func(x string, wordlist []string) (idx int) { 18 | return func(x string, wordlist []string) (idx int) { 19 | for i, v := range wordlist { 20 | if cb(x, v) { 21 | return i 22 | } 23 | } 24 | return -1 25 | } 26 | } 27 | 28 | func Unique(slice []string) []string { 29 | keys := make(map[string]bool) 30 | list := []string{} 31 | for _, entry := range slice { 32 | if _, value := keys[entry]; !value { 33 | keys[entry] = true 34 | list = append(list, entry) 35 | } 36 | } 37 | return list 38 | } 39 | 40 | func equals(x string, y string) bool { 41 | return x == y 42 | } 43 | 44 | // returns -1 if a string does not end with any word in wordlist 45 | var ContainsEndsWith = containsXWith(strings.HasSuffix) 46 | 47 | // returns -1 if a string does not start with any word in wordlist 48 | var ContainsStartsWith = containsXWith(strings.HasPrefix) 49 | 50 | // returns -1 if none of the strings exactly matches any word in wordlist 51 | var ContainsExactlyMatchesWith = containsXWith(equals) 52 | 53 | // returns -1 if a string does not match any word in wordlist 54 | var Contains = containsXWith(strings.Contains) 55 | 56 | func ExitOnError(e error) { 57 | if e != nil { 58 | panic(e) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /elasticpwn/util/validators.go: -------------------------------------------------------------------------------- 1 | package EPUtils 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func ValidateStringFlag(flagValue interface{}, defaultFlagValue interface{}, flagName string) bool { 8 | hasError := false 9 | switch flagValue.(type) { 10 | case string: 11 | { 12 | defaultFlagValueStr, ok := defaultFlagValue.(string) 13 | if flagValue == defaultFlagValueStr && ok { 14 | fmt.Printf("%v option is not set. Please try again.\n", flagName) 15 | hasError = true 16 | } 17 | } 18 | } 19 | 20 | return hasError 21 | } 22 | 23 | func ValidatePositiveInt(flagValue int, flagName string) bool { 24 | hasError := false 25 | if flagValue < 1 { 26 | fmt.Printf("%v option can't be less than 1. Input a positive number.\n", flagName) 27 | hasError = true 28 | } 29 | 30 | return hasError 31 | } 32 | -------------------------------------------------------------------------------- /report/build-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo " 3 | ############################################ 4 | 5 | before running this script, 6 | 1. you should have configured .env.local in /frontend properly 7 | 2. you should have a running mongodb server containing scan results 8 | 9 | ############################################ 10 | " 11 | 12 | cd frontend 13 | 14 | original_nextjs_config=$(cat next.config.js) 15 | 16 | function cleanup { 17 | echo "${original_nextjs_config}" > next.config.js 18 | } 19 | 20 | trap cleanup EXIT 21 | 22 | # https://unix.stackexchange.com/questions/356312/sed-insert-something-after-the-second-last-line 23 | sed -i '$i ,assetPrefix: ".",' next.config.js 24 | 25 | npm run build 26 | 27 | npm run "export" 28 | 29 | echo "${original_nextjs_config}" > next.config.js 30 | -------------------------------------------------------------------------------- /report/dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd frontend 3 | 4 | nvm use 5 | 6 | npm run dev & 7 | 8 | cd ../backend 9 | 10 | go run . & -------------------------------------------------------------------------------- /report/elasticpwn-backend/.env.example: -------------------------------------------------------------------------------- 1 | MONGODB_URI= 2 | DATABASE_NAME=elasticpwn 3 | DATABASE_COLLECTION_NAME=elasticsearch_reviewed_urls 4 | PORT=9292 -------------------------------------------------------------------------------- /report/elasticpwn-backend/.gitignore: -------------------------------------------------------------------------------- 1 | .env.local 2 | elasticpwn-backend -------------------------------------------------------------------------------- /report/elasticpwn-backend/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/9oelM/elasticpwn/report/elasticpwn-backend 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/gin-gonic/gin v1.7.4 7 | github.com/joho/godotenv v1.4.0 8 | go.mongodb.org/mongo-driver v1.7.3 9 | ) 10 | 11 | require ( 12 | github.com/gin-contrib/sse v0.1.0 // indirect 13 | github.com/go-playground/locales v0.14.0 // indirect 14 | github.com/go-playground/universal-translator v0.18.0 // indirect 15 | github.com/go-playground/validator/v10 v10.9.0 // indirect 16 | github.com/go-stack/stack v1.8.0 // indirect 17 | github.com/golang/protobuf v1.5.2 // indirect 18 | github.com/golang/snappy v0.0.1 // indirect 19 | github.com/json-iterator/go v1.1.12 // indirect 20 | github.com/klauspost/compress v1.13.6 // indirect 21 | github.com/leodido/go-urn v1.2.1 // indirect 22 | github.com/mattn/go-isatty v0.0.14 // indirect 23 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 24 | github.com/modern-go/reflect2 v1.0.2 // indirect 25 | github.com/pkg/errors v0.9.1 // indirect 26 | github.com/ugorji/go/codec v1.2.6 // indirect 27 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 28 | github.com/xdg-go/scram v1.0.2 // indirect 29 | github.com/xdg-go/stringprep v1.0.2 // indirect 30 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 31 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect 32 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect 33 | golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c // indirect 34 | golang.org/x/text v0.3.7 // indirect 35 | google.golang.org/protobuf v1.27.1 // indirect 36 | gopkg.in/yaml.v2 v2.4.0 // indirect 37 | ) 38 | -------------------------------------------------------------------------------- /report/elasticpwn-backend/go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 7 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 8 | github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM= 9 | github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= 10 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 11 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 12 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 13 | github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= 14 | github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= 15 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 16 | github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= 17 | github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= 18 | github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= 19 | github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A= 20 | github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= 21 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 22 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 23 | github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= 24 | github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= 25 | github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= 26 | github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 27 | github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 28 | github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= 29 | github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 30 | github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 31 | github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= 32 | github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= 33 | github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= 34 | github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= 35 | github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= 36 | github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= 37 | github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= 38 | github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= 39 | github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= 40 | github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 41 | github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 42 | github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 43 | github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 44 | github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= 45 | github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= 46 | github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= 47 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 48 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 49 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 50 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 51 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 52 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 53 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 54 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 55 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 56 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 57 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 58 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 59 | github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= 60 | github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 61 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 62 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 63 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 64 | github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= 65 | github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= 66 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 67 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 68 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 69 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 70 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 71 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 72 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 73 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 74 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 75 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 76 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 77 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 78 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 79 | github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= 80 | github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= 81 | github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= 82 | github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= 83 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 84 | github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= 85 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 86 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 87 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 88 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 89 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 90 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 91 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 92 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 93 | github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= 94 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 95 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 96 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 97 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 98 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 99 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 100 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 101 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 102 | github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 103 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 104 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 105 | github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= 106 | github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= 107 | github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 108 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 109 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 110 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 111 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 112 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 113 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 114 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 115 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 116 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 117 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 118 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 119 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 120 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= 121 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 122 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 123 | github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= 124 | github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= 125 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 126 | github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= 127 | github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= 128 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 129 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 130 | github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= 131 | github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= 132 | github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= 133 | github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= 134 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= 135 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 136 | go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= 137 | go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= 138 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 139 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 140 | golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= 141 | golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 142 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 143 | golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 144 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= 145 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 146 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 147 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 148 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 149 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 150 | golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 151 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 152 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 153 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 154 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 155 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 156 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 158 | golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 159 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 160 | golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 161 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 162 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 163 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 164 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 165 | golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 166 | golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c h1:QOfDMdrf/UwlVR0UBq2Mpr58UzNtvgJRXA4BgPfFACs= 167 | golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 168 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 169 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 170 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 171 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 172 | golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 173 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 174 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 175 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 176 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 177 | golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 178 | golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 179 | golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 180 | golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 181 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 182 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 183 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 184 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 185 | google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= 186 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 187 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 188 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 189 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 190 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 191 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 192 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 193 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 194 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 195 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 196 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 197 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 198 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 199 | -------------------------------------------------------------------------------- /report/elasticpwn-backend/handlers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "regexp" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "go.mongodb.org/mongo-driver/bson" 14 | "go.mongodb.org/mongo-driver/mongo" 15 | "go.mongodb.org/mongo-driver/mongo/options" 16 | ) 17 | 18 | func isValidUrl(rawUrl string) bool { 19 | _, err := url.ParseRequestURI(rawUrl) 20 | 21 | foundIp := IpWithOptionalPortRegex.FindAllString(rawUrl, -1) 22 | foundIpv6 := Ipv6WithOptionalPortRegex.FindAllString(rawUrl, -1) 23 | return err == nil || foundIp != nil || foundIpv6 != nil 24 | } 25 | 26 | var DefaultOrdered = false 27 | 28 | func PingHandler(c *gin.Context) { 29 | c.JSON(200, gin.H{ 30 | "message": "pong", 31 | }) 32 | } 33 | 34 | func ErrorHandler(c *gin.Context, err error) { 35 | c.JSON(500, gin.H{ 36 | "error": fmt.Sprintf("%v", err), 37 | }) 38 | } 39 | 40 | var IpWithOptionalPortRegex = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(:[0-9]+)?`) 41 | 42 | var Ipv6WithOptionalPortRegex = regexp.MustCompile(`(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(:[0-9]+)?`) 43 | 44 | type PostUrlsHandlerPayload struct { 45 | Urls []string `form:"urls" json:"urls" xml:"urls" binding:"required"` 46 | } 47 | 48 | func PostUrlsHandlerGenerator(urlsCollection *mongo.Collection) func(c *gin.Context) { 49 | return func(c *gin.Context) { 50 | var postUrlsHandlerPayload PostUrlsHandlerPayload 51 | 52 | if err := c.ShouldBindJSON(&postUrlsHandlerPayload); err != nil { 53 | c.JSON(http.StatusBadRequest, gin.H{"error": "should include urls"}) 54 | } 55 | 56 | log.Println(fmt.Sprintf("received %v", postUrlsHandlerPayload.Urls)) 57 | 58 | var urlsBsonDArray []interface{} 59 | for _, url := range postUrlsHandlerPayload.Urls { 60 | if !isValidUrl(url) { 61 | c.JSON(http.StatusNotAcceptable, gin.H{ 62 | "error": fmt.Sprintf("%v is not a valid url", url), 63 | }) 64 | return 65 | } else { 66 | urlsBsonDArray = append(urlsBsonDArray, bson.M{"url": url}) 67 | } 68 | } 69 | 70 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 71 | defer cancel() 72 | _, err := urlsCollection.InsertMany( 73 | ctx, 74 | urlsBsonDArray, 75 | // just make it idempotent by ignoring duplicate key error 76 | &options.InsertManyOptions{Ordered: &DefaultOrdered}, 77 | ) 78 | 79 | // just make it idempotent by ignoring duplicate key error 80 | if err != nil && !mongo.IsDuplicateKeyError(err) { 81 | ErrorHandler(c, err) 82 | return 83 | } 84 | 85 | c.JSON(200, gin.H{ 86 | "error": nil, 87 | }) 88 | } 89 | } 90 | 91 | func GetUrlsHandlerGenerator(urlsCollection *mongo.Collection) func(c *gin.Context) { 92 | return func(c *gin.Context) { 93 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 94 | defer cancel() 95 | distinct, err := urlsCollection.Distinct(ctx, "url", bson.M{}) 96 | if err != nil { 97 | ErrorHandler(c, err) 98 | return 99 | } 100 | 101 | c.JSON(200, gin.H{ 102 | "urls": distinct, 103 | }) 104 | } 105 | } 106 | 107 | type DeleteUrlsHandlerPayload struct { 108 | Collection string `form:"collection" json:"collection" xml:"collection" binding:"required"` 109 | } 110 | 111 | // only for the case where you delete all urls in a collection (for cleaning up test data, etc) 112 | func DeleteUrlsHandlerGenerator(mongoClient *mongo.Client, databaseName string) func(c *gin.Context) { 113 | return func(c *gin.Context) { 114 | var deleteUrlsHandlerPayload DeleteUrlsHandlerPayload 115 | 116 | if err := c.ShouldBindJSON(&deleteUrlsHandlerPayload); err != nil { 117 | c.JSON(http.StatusBadRequest, gin.H{"error": "should include collection"}) 118 | return 119 | } 120 | 121 | ctx0, cancel := context.WithTimeout(context.Background(), 10*time.Second) 122 | defer cancel() 123 | db := mongoClient.Database(databaseName) 124 | collectionNames, err := db.ListCollectionNames(ctx0, bson.D{}) 125 | if err != nil { 126 | ErrorHandler(c, err) 127 | return 128 | } 129 | 130 | existsCollectionName := false 131 | for _, collectionName := range collectionNames { 132 | if collectionName == deleteUrlsHandlerPayload.Collection { 133 | existsCollectionName = true 134 | break 135 | } 136 | } 137 | 138 | if !existsCollectionName { 139 | c.JSON(http.StatusBadRequest, gin.H{ 140 | "error": fmt.Sprintf("collection name should be any of: %v", collectionNames), 141 | }) 142 | return 143 | } 144 | 145 | ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second) 146 | defer cancel() 147 | err = db.Collection(deleteUrlsHandlerPayload.Collection).Drop(ctx1) 148 | 149 | if err != nil { 150 | ErrorHandler(c, err) 151 | return 152 | } 153 | 154 | c.JSON(200, gin.H{ 155 | "error": nil, 156 | }) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /report/elasticpwn-backend/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "runtime" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/joho/godotenv" 14 | "go.mongodb.org/mongo-driver/bson" 15 | "go.mongodb.org/mongo-driver/mongo" 16 | "go.mongodb.org/mongo-driver/mongo/options" 17 | ) 18 | 19 | func insertTestUrl(urlsCollection *mongo.Collection) { 20 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 21 | log.Println("Inserting") 22 | _, err := urlsCollection.InsertOne(ctx, bson.M{"url": "https://test.com"}) 23 | 24 | if err != nil { 25 | if mongo.IsDuplicateKeyError(err) { 26 | log.Println("Test url already created before") 27 | } else { 28 | log.Println(err) 29 | log.Fatal("Error in inserting") 30 | } 31 | } 32 | 33 | defer cancel() 34 | } 35 | 36 | func createUniqueUrlIndex(urlsCollection *mongo.Collection) { 37 | _, err := urlsCollection.Indexes().CreateOne( 38 | context.Background(), 39 | mongo.IndexModel{ 40 | Keys: bson.D{{Key: "url", Value: 1}}, 41 | Options: options.Index().SetUnique(true), 42 | }, 43 | ) 44 | 45 | if err != nil { 46 | log.Println(err) 47 | log.Fatal("Failed to create unique index") 48 | } 49 | } 50 | func CORSMiddleware() gin.HandlerFunc { 51 | return func(c *gin.Context) { 52 | c.Header("Access-Control-Allow-Origin", "*") 53 | c.Header("Access-Control-Allow-Credentials", "true") 54 | c.Header("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") 55 | c.Header("Access-Control-Allow-Methods", "POST,HEAD,PATCH,OPTIONS,GET,PUT") 56 | 57 | if c.Request.Method == "OPTIONS" { 58 | c.AbortWithStatus(204) 59 | return 60 | } 61 | 62 | c.Next() 63 | } 64 | } 65 | 66 | func main() { 67 | godotenv.Load(".env.local") 68 | MONGODB_URI, isMongoDbURISuppliedFromEnvFile := os.LookupEnv("MONGODB_URI") 69 | DATABASE_NAME, isDbNameSuppliedFromEnvFile := os.LookupEnv("DATABASE_NAME") 70 | DATABASE_COLLECTION_NAME, isCollectionNameSuppliedFromEnvFile := os.LookupEnv("DATABASE_COLLECTION_NAME") 71 | PORT, isPortSuppliedFromEnvFile := os.LookupEnv("PORT") 72 | 73 | reportBackendFs := flag.NewFlagSet("elasticpwn-bakcend", flag.ContinueOnError) 74 | mongodbURI := reportBackendFs.String("mongodbUri", "", "mongodb URI. Example: mongodb+srv://username:pw@somewhere.mongodb.net/default-collection-name?retryWrites=true&w=majority") 75 | databaseName := reportBackendFs.String("databaseName", "", "mongodb db name") 76 | databasecollectionName := reportBackendFs.String("databaseCollectionName", "", "mongodb collection name. Should be any one of: elasticsearch_reviewed_urls|elasticsearch_reviewed_urls_dev|kibana_reviewed_urls|kibana_reviewed_urls_dev") 77 | port := reportBackendFs.String("port", "9292", "port at which the server will run.") 78 | 79 | if !isMongoDbURISuppliedFromEnvFile || !isDbNameSuppliedFromEnvFile || !isCollectionNameSuppliedFromEnvFile || !isPortSuppliedFromEnvFile { 80 | if err := reportBackendFs.Parse(os.Args[1:]); err != nil { 81 | log.Fatal("At least one env var is nil. Check .env.local file. Otherwise, supply correct flags. Precedence: .env.local over flags") 82 | } 83 | } 84 | 85 | if reportBackendFs.Parsed() { 86 | if !isMongoDbURISuppliedFromEnvFile && *mongodbURI == "" { 87 | reportBackendFs.PrintDefaults() 88 | log.Fatal("MONGODB_URI was not found in either .env.local file or flag input") 89 | } else if !isDbNameSuppliedFromEnvFile && *databaseName == "" { 90 | reportBackendFs.PrintDefaults() 91 | log.Fatal("DATABASE_NAME was not found in either .env.local file or flag input") 92 | } else if !isCollectionNameSuppliedFromEnvFile && *databasecollectionName == "" { 93 | reportBackendFs.PrintDefaults() 94 | log.Fatal("DATABASE_COLLECTION_NAME was not found in either .env.local file or flag input") 95 | } else if !isPortSuppliedFromEnvFile && *port == "" { 96 | reportBackendFs.PrintDefaults() 97 | log.Fatal("DATABASE_COLLECTION_NAME was not found in either .env.local file or flag input") 98 | } 99 | } 100 | 101 | finalMongodbURI := func() string { 102 | if isMongoDbURISuppliedFromEnvFile { 103 | return MONGODB_URI 104 | } else { 105 | return *mongodbURI 106 | } 107 | }() 108 | finalDatabaseName := func() string { 109 | if isDbNameSuppliedFromEnvFile { 110 | return DATABASE_NAME 111 | } else { 112 | return *databaseName 113 | } 114 | }() 115 | finalCollectionName := func() string { 116 | if isCollectionNameSuppliedFromEnvFile { 117 | return DATABASE_COLLECTION_NAME 118 | } else { 119 | return *databasecollectionName 120 | } 121 | }() 122 | finalPort := func() string { 123 | if isPortSuppliedFromEnvFile { 124 | return PORT 125 | } else { 126 | return *port 127 | } 128 | }() 129 | if finalCollectionName != "elasticsearch_reviewed_urls" && 130 | finalCollectionName != "elasticsearch_reviewed_urls_dev" && 131 | finalCollectionName != "kibana_reviewed_urls" && 132 | finalCollectionName != "kibana_reviewed_urls_dev" { 133 | reportBackendFs.PrintDefaults() 134 | log.Fatal("DATABASE_COLLECTION_NAME (-databaseCollectionName) should be any one of: elasticsearch_reviewed_urls|elasticsearch_reviewed_urls_dev|kibana_reviewed_urls|kibana_reviewed_urls_dev") 135 | } 136 | 137 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 138 | defer cancel() 139 | mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(finalMongodbURI)) 140 | log.Println("Connected to database") 141 | defer func() { 142 | if err = mongoClient.Disconnect(ctx); err != nil { 143 | log.Fatal(err) 144 | } 145 | }() 146 | 147 | if err != nil { 148 | log.Fatal(err) 149 | } 150 | 151 | collection := mongoClient.Database(finalDatabaseName).Collection(finalCollectionName) 152 | insertTestUrl(collection) 153 | createUniqueUrlIndex(collection) 154 | 155 | r := gin.Default() 156 | r.Use(CORSMiddleware()) 157 | r.GET("/ping", PingHandler) 158 | r.GET("/urls", GetUrlsHandlerGenerator(collection)) 159 | r.POST("/urls", PostUrlsHandlerGenerator(collection)) 160 | r.DELETE("/urls", DeleteUrlsHandlerGenerator(mongoClient, finalDatabaseName)) 161 | 162 | serverHost := "" 163 | if runtime.GOOS == "windows" { 164 | serverHost = "localhost" 165 | } else { 166 | serverHost = "0.0.0.0" 167 | } 168 | r.Run(fmt.Sprintf("%v:%v", serverHost, finalPort)) 169 | } 170 | -------------------------------------------------------------------------------- /report/frontend/.env.local.example: -------------------------------------------------------------------------------- 1 | MONGODB_URI=mongodb://root:example@172.17.0.1:27017/ 2 | COLLECTION_NAME=elasticsearch # you need to change this to kibana based on your mode 3 | DB_NAME=ep # this is the db name that is going to be used 4 | SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH=http://localhost:9292 # elasticpwn-backend hosting url. not configurable as of now, it should be exactly this. -------------------------------------------------------------------------------- /report/frontend/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /report/frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | build 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env.local 29 | .env.development.local 30 | .env.test.local 31 | .env.production.local 32 | 33 | # vercel 34 | .vercel 35 | 36 | # typescript 37 | *.tsbuildinfo 38 | -------------------------------------------------------------------------------- /report/frontend/.nvmrc: -------------------------------------------------------------------------------- 1 | v16.12.0 -------------------------------------------------------------------------------- /report/frontend/README.md: -------------------------------------------------------------------------------- 1 | https://github.com/vercel/next.js/issues/4068#issuecomment-382298624 -------------------------------------------------------------------------------- /report/frontend/components/layout/layout.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react' 2 | import { 3 | defaultTheme, 4 | ThemeProvider, 5 | Preflight, 6 | } from '@xstyled/styled-components' 7 | 8 | const Layout: FC = ({ children }) => { 9 | return 12 | 13 | {children} 14 | 15 | } 16 | 17 | export default Layout -------------------------------------------------------------------------------- /report/frontend/components/quickActionButtons/quickActionButtons.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { x } from "@xstyled/styled-components"; 3 | import { enhance } from "../../util/react-essentials"; 4 | import { SF } from '../../styles/fragments'; 5 | import { usePersistentDatabaseServerAvailableStatus } from '../../hooks/usePersistentDatabaseServerAvailableStatus'; 6 | 7 | export type QuickActionButtonsProps = { 8 | onReviewed: VoidFunction 9 | onPass: VoidFunction 10 | } 11 | 12 | export const QuickActionButtons = enhance(({ 13 | onReviewed, 14 | onPass 15 | }) => { 16 | return 26 | 30 | Reviewed 31 | Alt+R 32 | 33 | 37 | Come back later 38 | Alt+C 39 | 40 | 41 | })() -------------------------------------------------------------------------------- /report/frontend/components/spacer/spacer.tsx: -------------------------------------------------------------------------------- 1 | import { x } from '@xstyled/styled-components' 2 | import React, { FC } from 'react' 3 | 4 | const Spacer: FC<{ 5 | vertical?: boolean 6 | size: string 7 | }> = ({ 8 | vertical = true, 9 | size 10 | }) => { 11 | return 16 | } 17 | 18 | export default Spacer -------------------------------------------------------------------------------- /report/frontend/config/env.ts: -------------------------------------------------------------------------------- 1 | export const Config = { 2 | DB_NAME: process.env.DB_NAME as string, 3 | COLLECTION_NAME: process.env.COLLECTION_NAME as string, 4 | SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH: process.env.SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH as string | undefined 5 | } 6 | 7 | Object.keys(Config).forEach((key) => { 8 | console.log(`Found env key: ${key}`) 9 | if (key === `SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH`) return 10 | 11 | if (key === undefined || key === null) { 12 | throw new Error(`Config['${key}'] is undefined or null. Please check .env.local file.`) 13 | } 14 | }) -------------------------------------------------------------------------------- /report/frontend/global.d.ts: -------------------------------------------------------------------------------- 1 | // types/globals.d.ts 2 | import type { MongoClient } from 'mongodb' 3 | 4 | declare global { 5 | namespace NodeJS { 6 | export interface Global { 7 | _mongoClientPromise: Promise 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /report/frontend/hooks/useOnReviewedAndOnPass.ts: -------------------------------------------------------------------------------- 1 | import { useRouter } from "next/router"; 2 | import React from "react"; 3 | import { ScanResult } from "../types/elastic"; 4 | import { API, APIStatus } from "../util/api"; 5 | import { LocalStorageKeys, LocalStorageManager } from "../util/localStorage"; 6 | import { tcAsync } from "../util/react-essentials"; 7 | 8 | export function useOnReviewedAndOnPass( 9 | scanResult: ScanResult['scanResult'], 10 | nextScanResultRaw: ScanResult | null, 11 | ) { 12 | 13 | const router = useRouter() 14 | const browseToNextReport = React.useCallback(() => { 15 | if (!nextScanResultRaw) return; 16 | 17 | router.push(`/reports/${nextScanResultRaw._id}?rootUrl=${nextScanResultRaw.scanResult.rootUrl}`) 18 | }, [nextScanResultRaw]) 19 | const onReviewed = React.useCallback(async () => { 20 | const [err, postUrlsAsReviewedResult] = await tcAsync(API.postUrlsAsReviewed([scanResult.rootUrl])) 21 | LocalStorageManager.addStringToDict(LocalStorageKeys.REVIEWED, scanResult.rootUrl) 22 | 23 | browseToNextReport() 24 | if (err) { 25 | console.log(err) 26 | return; 27 | } 28 | const hasAnotherError = postUrlsAsReviewedResult?.error !== null 29 | if (hasAnotherError) { 30 | console.log(postUrlsAsReviewedResult) 31 | return; 32 | } 33 | }, [browseToNextReport, scanResult.rootUrl]) 34 | const onPass = React.useCallback(() => { 35 | browseToNextReport() 36 | }, [browseToNextReport]) 37 | 38 | React.useEffect(() => { 39 | const listener = (ev: KeyboardEvent) => { 40 | if (!ev.altKey) return 41 | 42 | switch (ev.key) { 43 | case `r`: { 44 | onReviewed() 45 | break 46 | } 47 | case `c`: { 48 | onPass() 49 | break 50 | } 51 | } 52 | } 53 | document.addEventListener(`keydown`, listener) 54 | 55 | return () => { 56 | document.removeEventListener(`keydown`, listener) 57 | } 58 | }, [onReviewed, onPass]) 59 | 60 | return { 61 | onReviewed, 62 | onPass 63 | } 64 | } -------------------------------------------------------------------------------- /report/frontend/hooks/usePersistentDatabaseServerAvailableStatus.ts: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from "react" 2 | import { APIStatus, API } from "../util/api" 3 | import { tcAsync } from "../util/react-essentials" 4 | 5 | // just call this everywhere, without using a global state 6 | // to just make things very simple 7 | export function usePersistentDatabaseServerAvailableStatus() { 8 | const [ 9 | persistentDatbaseServerAvailableStatus, 10 | setPersistentDatbaseServerAvailable 11 | ] = useState(APIStatus.LOADING) 12 | 13 | useEffect(() => { 14 | async function checkPersistentDatabaseServerAvailable() { 15 | const [err] = await tcAsync(API.ping()) 16 | setPersistentDatbaseServerAvailable(err === null ? APIStatus.SUCCESSFUL : APIStatus.FAILED) 17 | } 18 | checkPersistentDatabaseServerAvailable() 19 | }, []) 20 | 21 | return persistentDatbaseServerAvailableStatus 22 | } -------------------------------------------------------------------------------- /report/frontend/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | /// 4 | 5 | // NOTE: This file should not be edited 6 | // see https://nextjs.org/docs/basic-features/typescript for more information. 7 | -------------------------------------------------------------------------------- /report/frontend/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | module.exports = { 3 | reactStrictMode: true, 4 | distDir: 'build', 5 | // https://stackoverflow.com/questions/66137368/next-js-environment-variables-are-undefined-next-js-10-0-5 6 | env: { 7 | SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH: process.env.SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH 8 | } 9 | // https://github.com/vercel/next.js/discussions/13578 10 | // this needs to be uncommented when npm run build 11 | // assetPrefix: '.', 12 | } 13 | -------------------------------------------------------------------------------- /report/frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint", 10 | "export": "next export" 11 | }, 12 | "dependencies": { 13 | "@xstyled/styled-components": "^3.1.1", 14 | "lodash.flow": "^3.5.0", 15 | "mongodb": "^4.1.3", 16 | "next": "12.0.1", 17 | "react": "17.0.2", 18 | "react-dom": "17.0.2", 19 | "styled-components": "^5.3.3" 20 | }, 21 | "devDependencies": { 22 | "@types/lodash.flow": "^3.5.6", 23 | "@types/node": "16.11.6", 24 | "@types/react": "17.0.33", 25 | "eslint": "7.32.0", 26 | "eslint-config-next": "12.0.1", 27 | "typescript": "4.4.4" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /report/frontend/pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import '../styles/globals.css' 2 | import type { AppProps } from 'next/app' 3 | 4 | function MyApp({ Component, pageProps }: AppProps) { 5 | return 6 | } 7 | 8 | export default MyApp 9 | -------------------------------------------------------------------------------- /report/frontend/pages/index.tsx: -------------------------------------------------------------------------------- 1 | import { x } from '@xstyled/styled-components' 2 | import { Filter } from 'mongodb' 3 | import type { GetStaticProps, NextPage } from 'next' 4 | import Link from 'next/link' 5 | import React, { useCallback, useEffect, useMemo, useState } from 'react' 6 | import Layout from '../components/layout/layout' 7 | import { Config } from '../config/env' 8 | import { usePersistentDatabaseServerAvailableStatus } from '../hooks/usePersistentDatabaseServerAvailableStatus' 9 | import { SF } from '../styles/fragments' 10 | import TableInfo from '../templates/localFragments/Indices' 11 | import { ElasticProductInfo, ScanResult } from '../types/elastic' 12 | import { API, APIStatus } from '../util/api' 13 | import { LocalStorageInternal, LocalStorageKeys, LocalStorageManager } from '../util/localStorage' 14 | import mongoClientPromise, { usefulScanResultFilter } from '../util/mongo' 15 | import { tcAsync } from '../util/react-essentials' 16 | 17 | const Home: NextPage<{ 18 | count: number, 19 | allInterestingScanResults: { rootUrl: string; id: string; }[] 20 | allUninterestingScanResultsUrls: { rootUrl: string }[] 21 | }> = ({ 22 | count, 23 | allInterestingScanResults, 24 | allUninterestingScanResultsUrls, 25 | }) => { 26 | const [ 27 | newInterstingScanResults, 28 | setNewInterestingScanResults 29 | ] = useState<{ rootUrl: string; id: string; }[]>([]) 30 | const [ 31 | newUninterestingUrls, 32 | setNewUninterestingUrls 33 | ] = useState([]) 34 | 35 | const [ 36 | reviewedUrlsFromPersistentDatabase, 37 | setReviewedUrlsFromPersistentDatabase 38 | ] = useState<{ rootUrl: string; id: string; }[]>([]) 39 | 40 | const [reviewedUrlsInCurrentSession, setReviewedUrlsInCurrentSession] = useState(null) 41 | // https://github.com/vercel/next.js/discussions/19911 42 | useEffect(() => { 43 | const reviewedUrls = LocalStorageManager.getValue(LocalStorageKeys.REVIEWED) 44 | setReviewedUrlsInCurrentSession(reviewedUrls) 45 | }, []) 46 | 47 | const persistentDatbaseServerAvailableStatus = usePersistentDatabaseServerAvailableStatus() 48 | 49 | useEffect(() => { 50 | async function processUrls() { 51 | const [err, reviewedUrlsFromPersistentDatabase] = await tcAsync(API.getReviewedUrls()) 52 | 53 | if (err || !reviewedUrlsFromPersistentDatabase || !reviewedUrlsFromPersistentDatabase.urls) { 54 | console.error(`could not get reviewed urls`) 55 | setNewUninterestingUrls(allUninterestingScanResultsUrls.map(({ rootUrl }) => rootUrl)) 56 | return 57 | } 58 | 59 | // @ts-ignore 60 | setReviewedUrlsFromPersistentDatabase(reviewedUrlsFromPersistentDatabase.urls) 61 | setNewInterestingScanResults( 62 | allInterestingScanResults.filter(({ rootUrl }) => { 63 | return !reviewedUrlsFromPersistentDatabase.urls?.find((url) => rootUrl === url) 64 | }) 65 | ) 66 | // for the case where you've previously reviewed the same urls already 67 | setNewUninterestingUrls( 68 | allUninterestingScanResultsUrls.filter(({ rootUrl }) => { 69 | return !reviewedUrlsFromPersistentDatabase.urls?.find((url) => rootUrl === url) 70 | }).map(({ rootUrl }) => rootUrl) 71 | ) 72 | } 73 | 74 | if (persistentDatbaseServerAvailableStatus !== APIStatus.LOADING) { 75 | processUrls() 76 | } 77 | }, [allInterestingScanResults, allUninterestingScanResultsUrls, persistentDatbaseServerAvailableStatus]) 78 | 79 | const onSaveUninterestingUrlsAsReviewed = useCallback(async () => { 80 | const [err, result] = await tcAsync(API.postUrlsAsReviewed(newUninterestingUrls)) 81 | if (err || result?.error !== null) { 82 | console.error(`failed to save uninteresting urls`) 83 | return 84 | } 85 | }, [newUninterestingUrls]) 86 | 87 | const uninterestingUrlsInstruction = React.useMemo(() => { 88 | return newUninterestingUrls.length > 0 ? 89 | 90 | 91 | If you are using a mongodb backend to store your reviewed data, you should click the button below to save all uninteresting urls (total of {newUninterestingUrls.length}) collected for this report as reviewed in your mongodb database. 92 | 93 | 94 | If you don‘t, you will face these urls again in your report for the next recon (if you are running the scan with the corresponding option). 95 | {` `} 96 | 97 | If you think something went wrong while scanning, re-run the scanning and generate the report again instead of clicking the button, because the change is irreversible. 98 | 99 | 100 | 109 | Save all {newUninterestingUrls.length} uninteresing URLs as reviewed 110 | 111 | : null 112 | }, [newUninterestingUrls, onSaveUninterestingUrlsAsReviewed]) 113 | 114 | return ( 115 | 116 | 117 | 118 | elasticpwn report 119 | { 120 | persistentDatbaseServerAvailableStatus === APIStatus.SUCCESSFUL ? 121 | 125 | 126 | {count} 127 | {newInterstingScanResults.length} 128 | {reviewedUrlsFromPersistentDatabase.length} 129 | {newUninterestingUrls.length} 130 | 131 | : 135 | 136 | {count} 137 | {newInterstingScanResults.length} 138 | {newUninterestingUrls.length} 139 | 140 | 141 | } 142 | 143 | 144 | {persistentDatbaseServerAvailableStatus === APIStatus.SUCCESSFUL ? uninterestingUrlsInstruction : null} 145 | 150 | 151 | 152 | Reviewed in current session (based on localStorage data): 153 | 154 | 155 | {allInterestingScanResults.map(({ rootUrl, id }) => { 156 | if (reviewedUrlsInCurrentSession?.[rootUrl]) { 157 | return ( 158 | 159 | 160 | {rootUrl} 161 | 162 | 163 | ) 164 | } 165 | return null 166 | }) 167 | } 168 | 169 | 170 | 171 | 172 | To be reviewed in current session (based on localStorage data): 173 | 174 | 175 | {(persistentDatbaseServerAvailableStatus === APIStatus.SUCCESSFUL ? newInterstingScanResults : allInterestingScanResults).map(({ rootUrl, id }) => { 176 | if (reviewedUrlsInCurrentSession?.[rootUrl]) { 177 | return null 178 | } 179 | return ( 180 | 181 | 182 | {rootUrl} 183 | 184 | 185 | ) 186 | }) 187 | } 188 | 189 | 190 | 191 | 192 | 193 | ) 194 | } 195 | 196 | export default Home 197 | 198 | export const getStaticProps: GetStaticProps = async (context) => { 199 | const client = await mongoClientPromise 200 | const elasticpwnDb = await client.db(Config.DB_NAME) 201 | const collection = elasticpwnDb.collection(Config.COLLECTION_NAME) 202 | const count = await collection.count() 203 | /** 204 | * @todo cache results in development 205 | */ 206 | const allInterestingScanResults = 207 | await collection 208 | .find(usefulScanResultFilter) 209 | .map(({ 210 | scanResult, 211 | // for findById in child pages 212 | _id 213 | }) => ({ 214 | rootUrl: scanResult.rootUrl, 215 | id: _id.toString() 216 | })) 217 | .toArray() 218 | const allUninterestingScanResultsUrls = 219 | await collection 220 | .find({ $nor: [usefulScanResultFilter] }) 221 | .map(({ 222 | scanResult, 223 | }) => ({ 224 | rootUrl: scanResult.rootUrl, 225 | })) 226 | .toArray() 227 | 228 | return { 229 | props: { 230 | count, 231 | allInterestingScanResults, 232 | allUninterestingScanResultsUrls, 233 | }, 234 | // revalidate every 30 mins to speed up build speed while developing 235 | revalidate: 60 * 30 236 | } 237 | } -------------------------------------------------------------------------------- /report/frontend/pages/reports/[id].tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { useRouter } from 'next/router' 3 | import { ScanResult } from "../../types/elastic" 4 | import { x } from "@xstyled/styled-components"; 5 | import { SF } from "../../styles/fragments"; 6 | import { getOnlyNumber, getOnlyString } from "../../util/string"; 7 | import PreInfo from "../../templates/localFragments/PreInfo"; 8 | import { getUnitScore, SizeUnit } from "../../util/filesize"; 9 | import TableInfo from "../../templates/localFragments/Indices"; 10 | import Layout from "../../components/layout/layout"; 11 | import { GetStaticPaths, GetStaticProps, NextPage } from "next"; 12 | 13 | import mongoClientPromise, { usefulScanResultFilter } from '../../util/mongo' 14 | import { ObjectId } from "bson"; 15 | import { QuickActionButtons } from "../../components/quickActionButtons/quickActionButtons"; 16 | import { Config } from "../../config/env"; 17 | import { enhance, tcAsync } from "../../util/react-essentials"; 18 | import { API } from "../../util/api"; 19 | import { useOnReviewedAndOnPass } from "../../hooks/useOnReviewedAndOnPass"; 20 | import { usePersistentDatabaseServerAvailableStatus } from "../../hooks/usePersistentDatabaseServerAvailableStatus"; 21 | import Link from "next/link"; 22 | import { LocalStorageKeys, LocalStorageManager } from "../../util/localStorage"; 23 | 24 | type ReportProps = { 25 | id: string 26 | scanResult: ScanResult, 27 | nextScanResult: ScanResult | null 28 | } 29 | 30 | const Report: NextPage = enhance(({ 31 | scanResult: scanResultRaw, 32 | nextScanResult: nextScanResultRaw, 33 | }) => { 34 | const { scanResult } = scanResultRaw 35 | 36 | const [wasCurrentRootUrlReviewed, setCurrentRootUrlReviewed] = React.useState(false) 37 | React.useEffect(() => { 38 | const allReviewedUrls = LocalStorageManager.getValue(LocalStorageKeys.REVIEWED) 39 | 40 | if (allReviewedUrls === null) { 41 | return 42 | } 43 | setCurrentRootUrlReviewed(Boolean(allReviewedUrls[scanResult.rootUrl])) 44 | }, [scanResult.rootUrl]) 45 | 46 | const indicesSortedBySize = React.useMemo(() => { 47 | if (!scanResult || !scanResult.indices) return null 48 | 49 | return [...scanResult.indices].sort((a, b) => { 50 | if (!a["store.size"]) { 51 | return 1 52 | } else if (!b["store.size"]) { 53 | return 0 54 | } 55 | 56 | const aSizeUnitScore = getUnitScore(getOnlyString(a["store.size"]) as SizeUnit) 57 | const bSizeUnitScore = getUnitScore(getOnlyString(b["store.size"]) as SizeUnit) 58 | 59 | if (aSizeUnitScore !== bSizeUnitScore) { 60 | return bSizeUnitScore - aSizeUnitScore 61 | } 62 | 63 | const aSize = getOnlyNumber(a["store.size"]) 64 | const bSize = getOnlyNumber(b["store.size"]) 65 | 66 | if (aSize === null) { 67 | return 1 68 | } else if (bSize === null) { 69 | return 0 70 | } 71 | 72 | return bSize - aSize 73 | }) 74 | }, [scanResult]) 75 | 76 | const indices = React.useMemo(() => { 77 | return indicesSortedBySize ? 87 | { 88 | indicesSortedBySize.map((index) => { 89 | const isIndexSizeBiggerThanMB = 90 | index["store.size"] && getUnitScore(getOnlyString(index["store.size"]) as SizeUnit) > 2 91 | 92 | return 98 | {/* index.index (name) */} 99 | {index.index} 100 | {index["store.size"]} 105 | {index["docs.count"]} 106 | {index["docs.deleted"]} 107 | {index["pri.store.size"]} 108 | 109 | }) 110 | } 111 | : null 112 | }, [indicesSortedBySize]) 113 | 114 | 115 | console.log(scanResultRaw) 116 | console.log(nextScanResultRaw) 117 | const interestingInfo = React.useMemo(() => { 118 | if (!scanResult || !scanResult.interestingInfo) return null 119 | 120 | const moreThanTwoDotsInName = scanResult.interestingInfo.moreThanTwoDotsInName || [] 121 | const urls = scanResult.interestingInfo.moreThanTwoDotsInName || [] 122 | return [ 123 | { 124 | title: "Contains more than two dots", 125 | info: [ 126 | ...new Set([ 127 | ...moreThanTwoDotsInName, 128 | ...urls 129 | ])].filter((line) => getOnlyNumber(line) === null).sort() 130 | }, 131 | { 132 | title: "Contains '@'", 133 | info: scanResult.interestingInfo.emails 134 | }, 135 | { 136 | title: "Public IPs", 137 | info: scanResult.interestingInfo.publicIps, 138 | } 139 | ].map(({ title, info }) => { 140 | if (!info || info.length === 0) return null 141 | 142 | return 149 | }) 150 | }, [scanResult]) 151 | 152 | const allocations = React.useMemo(() => { 153 | if (!scanResult) return null 154 | 155 | return scanResult.allocations ? 169 | {scanResult.allocations.map((alloc, i) => { 170 | const isDiskTotalBiggerThanMB = alloc["disk.total"] !== null && 171 | getUnitScore(getOnlyString(alloc["disk.total"]) as SizeUnit) > 2 172 | 173 | return ( 174 | 175 | {alloc["disk.avail"]} 176 | {alloc["disk.indices"]} 177 | {alloc["disk.percent"]} 178 | {alloc["disk.total"]} 183 | {alloc["disk.used"]} 184 | {alloc["host"]} 185 | {alloc["ip"]} 186 | {alloc["node"]} 187 | {alloc["shards"]} 188 | 189 | ) 190 | })} 191 | : null 192 | }, [scanResult]) 193 | 194 | const aliases = React.useMemo(() => { 195 | if (!scanResult) return null 196 | 197 | if (!scanResult.aliases || scanResult.aliases.length === 0) return null 198 | 199 | return 207 | {scanResult.aliases.map((alias, i) => { 208 | return ( 209 | 210 | {alias["alias"]} 211 | {alias["filter"]} 212 | {alias["index"]} 213 | 214 | ) 215 | })} 216 | 217 | }, [scanResult]) 218 | 219 | const rawIndicesInfo = React.useMemo(() => { 220 | if (!scanResult || !scanResult.indicesInfoInJson) return null 221 | 222 | let prettyJSON: string | null = null 223 | try { 224 | prettyJSON = JSON.stringify(scanResult.indicesInfoInJson, null, 2) 225 | } catch (e) { 226 | console.log(e) 227 | } 228 | 229 | if (!prettyJSON || prettyJSON?.trim() === '{}') return null 230 | 231 | return 236 | }, [scanResult]) 237 | 238 | const { 239 | onReviewed, 240 | onPass 241 | } = useOnReviewedAndOnPass(scanResult, nextScanResultRaw) 242 | 243 | if (!scanResult) { 244 | return scan result is null 245 | } 246 | 247 | return ( 248 | 249 | 257 | 258 | 264 | ← back to main 265 | 266 | 267 | {`Report: `} 272 | 280 | {scanResult.rootUrl} 281 | 282 | 283 | {wasCurrentRootUrlReviewed ? 288 | This URL has been reviewed once by you. 289 | : null} 290 | 297 | {indices} 298 | {scanResult.interestingWords ? : null} 302 | {interestingInfo} 303 | {allocations} 304 | {aliases} 305 | {rawIndicesInfo} 306 | 307 | 308 | 309 | 313 | 314 | ) 315 | })() 316 | 317 | export default Report 318 | 319 | export const getStaticProps: GetStaticProps<{}, { id: string }> = async (context) => { 320 | if (!context.params) { 321 | throw new Error(`context.params is undefined or null`) 322 | } 323 | 324 | console.log(`context.params.id: ${context.params.id}`) 325 | 326 | const client = await mongoClientPromise 327 | const elasticpwnDb = await client.db(Config.DB_NAME) 328 | const collection = elasticpwnDb.collection(Config.COLLECTION_NAME) 329 | 330 | const currentAndNextScanResultsCursor = await collection.find({ 331 | $and: [ 332 | { 333 | _id: { 334 | /** 335 | * the documents are just sorted by id (insertion order) 336 | */ 337 | $gte: new ObjectId(context.params.id) 338 | }, 339 | }, 340 | usefulScanResultFilter, 341 | ] 342 | }).limit(2) 343 | 344 | const singleScanResult = await currentAndNextScanResultsCursor.next() 345 | const nextSingleScanResult = (await currentAndNextScanResultsCursor.hasNext()) ? await currentAndNextScanResultsCursor.next() : null 346 | 347 | // https://github.com/vercel/next.js/issues/11993#issuecomment-617375501 348 | const scanResult = JSON.parse(JSON.stringify(singleScanResult)) 349 | const nextScanResult = JSON.parse(JSON.stringify(nextSingleScanResult)) 350 | 351 | return { 352 | props: { 353 | // to enable destructuring in the component 354 | scanResult: scanResult === null ? { scanResult: null } : scanResult, 355 | nextScanResult: nextScanResult, 356 | } 357 | } 358 | } 359 | 360 | export const getStaticPaths: GetStaticPaths<{ id: string }> = async () => { 361 | const client = await mongoClientPromise 362 | const elasticpwnDb = await client.db(Config.DB_NAME) 363 | const collection = elasticpwnDb.collection(Config.COLLECTION_NAME) 364 | 365 | const allInterestingScanResultsIds = 366 | await collection 367 | .find(usefulScanResultFilter) 368 | .map(({ _id }) => ({ id: _id.toString() })).toArray() 369 | 370 | return { 371 | paths: allInterestingScanResultsIds.map(({ id }) => ({ params: { id }})), // all pages with interesting scan result need to be created at build time 372 | // this needs to be set as false to enable `next export` command 373 | fallback: false 374 | } 375 | } -------------------------------------------------------------------------------- /report/frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/9oelM/elasticpwn/31f45a432401f3e656999e2d3771951c9ff7fce9/report/frontend/public/favicon.ico -------------------------------------------------------------------------------- /report/frontend/public/vercel.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /report/frontend/styles/Home.module.css: -------------------------------------------------------------------------------- 1 | .container { 2 | padding: 0 2rem; 3 | } 4 | 5 | .main { 6 | min-height: 100vh; 7 | padding: 4rem 0; 8 | flex: 1; 9 | display: flex; 10 | flex-direction: column; 11 | justify-content: center; 12 | align-items: center; 13 | } 14 | 15 | .footer { 16 | display: flex; 17 | flex: 1; 18 | padding: 2rem 0; 19 | border-top: 1px solid #eaeaea; 20 | justify-content: center; 21 | align-items: center; 22 | } 23 | 24 | .footer a { 25 | display: flex; 26 | justify-content: center; 27 | align-items: center; 28 | flex-grow: 1; 29 | } 30 | 31 | .title a { 32 | color: #0070f3; 33 | text-decoration: none; 34 | } 35 | 36 | .title a:hover, 37 | .title a:focus, 38 | .title a:active { 39 | text-decoration: underline; 40 | } 41 | 42 | .title { 43 | margin: 0; 44 | line-height: 1.15; 45 | font-size: 4rem; 46 | } 47 | 48 | .title, 49 | .description { 50 | text-align: center; 51 | } 52 | 53 | .description { 54 | margin: 4rem 0; 55 | line-height: 1.5; 56 | font-size: 1.5rem; 57 | } 58 | 59 | .code { 60 | background: #fafafa; 61 | border-radius: 5px; 62 | padding: 0.75rem; 63 | font-size: 1.1rem; 64 | font-family: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, 65 | Bitstream Vera Sans Mono, Courier New, monospace; 66 | } 67 | 68 | .grid { 69 | display: flex; 70 | align-items: center; 71 | justify-content: center; 72 | flex-wrap: wrap; 73 | max-width: 800px; 74 | } 75 | 76 | .card { 77 | margin: 1rem; 78 | padding: 1.5rem; 79 | text-align: left; 80 | color: inherit; 81 | text-decoration: none; 82 | border: 1px solid #eaeaea; 83 | border-radius: 10px; 84 | transition: color 0.15s ease, border-color 0.15s ease; 85 | max-width: 300px; 86 | } 87 | 88 | .card:hover, 89 | .card:focus, 90 | .card:active { 91 | color: #0070f3; 92 | border-color: #0070f3; 93 | } 94 | 95 | .card h2 { 96 | margin: 0 0 1rem 0; 97 | font-size: 1.5rem; 98 | } 99 | 100 | .card p { 101 | margin: 0; 102 | font-size: 1.25rem; 103 | line-height: 1.5; 104 | } 105 | 106 | .logo { 107 | height: 1em; 108 | margin-left: 0.5rem; 109 | } 110 | 111 | @media (max-width: 600px) { 112 | .grid { 113 | width: 100%; 114 | flex-direction: column; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /report/frontend/styles/fragments.ts: -------------------------------------------------------------------------------- 1 | export const SF = { 2 | fullWH: { 3 | w: `100%`, 4 | h: `100%`, 5 | }, 6 | flex: { 7 | display: 'flex', 8 | }, 9 | wrapLines: { 10 | // xstyled does not support this 11 | wordWrap: "break-word", 12 | 'whiteSpace': "pre-wrap" 13 | }, 14 | cursorPointer: { 15 | cursor: 'pointer' 16 | }, 17 | standardButton: { 18 | w: 100, 19 | h: 100, 20 | bg: { hover: 'gray-500', _: 'gray-600' }, 21 | borderRadius: 6, 22 | display: `flex`, 23 | flexDirection: 'column', 24 | justifyContent: 'center', 25 | alignItems: 'center', 26 | pointerEvents: "auto", 27 | transition: true, 28 | transitionDuration: 200, 29 | boxShadow: "2xl", 30 | } 31 | } -------------------------------------------------------------------------------- /report/frontend/styles/globals.css: -------------------------------------------------------------------------------- 1 | html, 2 | body { 3 | padding: 0; 4 | margin: 0; 5 | font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen, 6 | Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif; 7 | } 8 | 9 | a { 10 | color: inherit; 11 | text-decoration: none; 12 | } 13 | 14 | * { 15 | box-sizing: border-box; 16 | } 17 | 18 | * { 19 | font-family: ui-monospace,SFMono-Regular,Consolas,'Liberation Mono',Menlo,monospace; 20 | } 21 | 22 | html,body,#__next { 23 | margin: 0; 24 | padding: 0; 25 | height: 100%; 26 | width: 100%; 27 | background-color: #18181bff; 28 | } 29 | -------------------------------------------------------------------------------- /report/frontend/templates/localFragments/Indices.tsx: -------------------------------------------------------------------------------- 1 | import { x } from '@xstyled/styled-components' 2 | import React, { FC } from 'react' 3 | import { SF } from '../../styles/fragments' 4 | import { ElasticProductInfo } from '../../types/elastic' 5 | 6 | const TableInfo: FC<{ 7 | title: string, 8 | headings: string[] 9 | }> = ({ 10 | children, 11 | title, 12 | headings 13 | }) => { 14 | return 20 | 24 | {title} 25 | 26 | 31 | 40 | 43 | 44 | {headings.map((heading, i) => { 45 | return {heading} 46 | })} 47 | 48 | 49 | 51 | {children} 52 | 53 | 54 | 55 | 56 | } 57 | 58 | export default TableInfo -------------------------------------------------------------------------------- /report/frontend/templates/localFragments/PreInfo.tsx: -------------------------------------------------------------------------------- 1 | import { x } from '@xstyled/styled-components' 2 | import React, { FC } from 'react' 3 | import { SF } from '../../styles/fragments' 4 | 5 | const normalHeight = { md: '500px', xs: '300px' } 6 | const bigHeight = { md: '900px', xs: '600px' } 7 | 8 | const PreInfo: FC<{ 9 | title: string 10 | info: string 11 | height?: 'normal'|'big' 12 | }> = ({ 13 | title, 14 | info, 15 | height = `normal` 16 | }) => { 17 | return 22 | 26 | {title} 27 | 28 | 33 | 41 | {info} 42 | 43 | 44 | 45 | } 46 | 47 | export default PreInfo -------------------------------------------------------------------------------- /report/frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "strict": true, 12 | "forceConsistentCasingInFileNames": true, 13 | "noEmit": true, 14 | "esModuleInterop": true, 15 | "module": "esnext", 16 | "moduleResolution": "node", 17 | "resolveJsonModule": true, 18 | "isolatedModules": true, 19 | "downlevelIteration": true, 20 | "jsx": "preserve", 21 | "incremental": true 22 | }, 23 | "include": [ 24 | "next-env.d.ts", 25 | "**/*.ts", 26 | "**/*.tsx" 27 | ], 28 | "exclude": [ 29 | "node_modules" 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /report/frontend/types/elastic.ts: -------------------------------------------------------------------------------- 1 | import { ObjectId } from "bson"; 2 | 3 | export interface ElasticProductInfo { 4 | _id: string 5 | rootUrl: string 6 | // {"index":"index_name","docs.count":"2355","docs.deleted":"0","store.size":"3.8mb","pri.store.size":"3.8mb"} 7 | indices: null | { 8 | index: string 9 | "docs.count": string 10 | "docs.deleted": string 11 | "store.size": string 12 | "pri.store.size": string 13 | }[] 14 | // this data is too complex. It will be presented in 
 tag, so just render it as string
15 |     indicesInfoInJson: string
16 |     interestingWords: string[]
17 |     interestingInfo: null | {
18 |         emails: string[]
19 |         urls: string[]
20 |         publicIps: string[]
21 |         moreThanTwoDotsInName: string[]
22 |     }
23 |     // {"alias":".kibana","filter":"-","index":".kibana_1","is_write_index":"-","routing.index":"-","routing.search":"-"}
24 |     aliases: null | {
25 |         alias: string
26 |         filter: string
27 |         index: string
28 |     }[]
29 |     // {"disk.avail":"133.8gb","disk.indices":"83.2mb","disk.percent":"13","disk.total":"154.8gb","disk.used":"21gb","host":"10.42.197.128","ip":"10.42.197.128","node":"fde99ab8806e","shards":"13"}
30 |     allocations: null | {
31 |         "disk.avail": string | null
32 |         "disk.indices": string | null
33 |         "disk.percent": string | null
34 |         "disk.total": string | null
35 |         "disk.used": string | null
36 |         "host": string | null 
37 |         "ip": string | null 
38 |         "node": string | null 
39 |         "shards": string | null
40 |     }[]
41 |     isInitialized: boolean
42 | 
43 |     // unused properties (for now)
44 | 
45 |     hasAtLeastOneIndexSizeOverGB: boolean
46 |     // {"action":"cluster:monitor/nodes/stats","ip":"10.42.197.128","node":"fde99ab8806e","parent_task_id":"-","running_time":"3.4ms","start_time":"1635165293843","task_id":"_-IU2jWcQsaT0XsJCga1mg:8439865","timestamp":"12:34:53","type":"transport"}
47 |     tasks: Record[]
48 |     created_at: string
49 |     trainedModels: string
50 |     transforms: string
51 |     // not really interested in these anyway
52 |     count: null
53 |     master: null
54 |     nodeaatrs: null
55 |     nodes: null
56 |     pendingTasks: null
57 |     plugins: null
58 | }
59 | 
60 | export interface ScanResult {
61 |     _id: ObjectId
62 |     scanResult: ElasticProductInfo
63 | }


--------------------------------------------------------------------------------
/report/frontend/util/api.ts:
--------------------------------------------------------------------------------
 1 | import { Config } from "../config/env"
 2 | import { tcAsync } from "./react-essentials"
 3 | 
 4 | console.log(Config.SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH)
 5 | 
 6 | const rootUrl = Config.SERVER_ROOT_URL_WITHOUT_TRAILING_SLASH
 7 | 
 8 | export enum APIStatus {
 9 |     LOADING = "LOADING",
10 |     FAILED = "FAILED",
11 |     SUCCESSFUL = "SUCCESSFUL"
12 | }
13 | 
14 | export class API {
15 |     static ERRORS = Object.freeze({
16 |         NO_ROOT_URL: `NO_ROOT_URL`,
17 |         PERSISTENT_SERVER_DOWN: `PERSISTENT_SERVER_DOWN` 
18 |     })
19 | 
20 |     static async ping(): Promise {
21 |         const rawResponse = await fetch(`${rootUrl}/ping`);
22 |         rawResponse.json()
23 |     }
24 | 
25 |     static async postUrlsAsReviewed(urls: string[]): Promise<{ error: null | string }> {
26 |         if (!rootUrl) return { error: API.ERRORS.NO_ROOT_URL }
27 |         const rawResponse = await fetch(`${rootUrl}/urls`, {
28 |             method: 'POST',
29 |             headers: {
30 |               'Accept': 'application/json',
31 |               'Content-Type': 'application/json'
32 |             },
33 |             body: JSON.stringify({ urls })
34 |         });
35 | 
36 |         const jsonResponse = await rawResponse.json()
37 |         return jsonResponse
38 |     }
39 | 
40 |     static async getReviewedUrls(): Promise<{ urls?: string[] }> {
41 |         if (!rootUrl) return { urls: undefined }
42 |         const rawResponse = await fetch(`${rootUrl}/urls`, {
43 |             headers: {
44 |               'Accept': 'application/json',
45 |               'Content-Type': 'application/json'
46 |             },
47 |         });
48 | 
49 |         const jsonResponse = await rawResponse.json()
50 |         return jsonResponse
51 |     }
52 | }


--------------------------------------------------------------------------------
/report/frontend/util/filesize.ts:
--------------------------------------------------------------------------------
 1 | export enum SizeUnit {
 2 |     b = "b",
 3 |     kb = "kb",
 4 |     mb = "mb",
 5 |     gb = "gb",
 6 |     tb = "tb",
 7 |     pb = "pb",
 8 |     eb = "eb",
 9 |     zb = "zb",
10 | }
11 | 
12 | export function getUnitScore(unit: SizeUnit) {
13 |     switch (unit) {
14 |         case SizeUnit.b:
15 |             return 0
16 |         case SizeUnit.kb:
17 |             return 1
18 |         case SizeUnit.mb:
19 |             return 2
20 |         case SizeUnit.gb:
21 |             return 3
22 |         case SizeUnit.tb:
23 |             return 4
24 |         case SizeUnit.pb:
25 |             return 5
26 |         case SizeUnit.eb:
27 |             return 6
28 |         case SizeUnit.zb:
29 |             return 7
30 |         // idk but this case happens
31 |         default:
32 |             return 0
33 |     }
34 | }


--------------------------------------------------------------------------------
/report/frontend/util/localStorage.ts:
--------------------------------------------------------------------------------
 1 | import { ScanResult } from "../types/elastic"
 2 | 
 3 | export enum LocalStorageKeys { 
 4 |     NOT_REVIEWED = `elasticpwn_NOT_REVIEWED`,
 5 |     REVIEWED = `elasticpwn_REVIEWED`,
 6 |     COME_BACK_LATER = `elasticpwn_COME_BACK_LATER`, 
 7 | }
 8 | 
 9 | export interface LocalStorageInternal {
10 |     [LocalStorageKeys.NOT_REVIEWED]: Record, boolean>
11 |     [LocalStorageKeys.REVIEWED]: Record, boolean>
12 |     [LocalStorageKeys.COME_BACK_LATER]: Record, boolean>
13 | }
14 | 
15 | export class LocalStorageManager {
16 |     static getValue(key: LocalStorageKeys): LocalStorageInternal[T] | null {
17 |         const item = localStorage.getItem(key)
18 |         
19 |         if (item === null) return null
20 | 
21 |         let parsed: null | any = null
22 | 
23 |         try {
24 |             parsed = JSON.parse(item)
25 |         } catch {
26 |             return null
27 |         }
28 | 
29 |         return parsed
30 |     }
31 | 
32 |     static setValue(key: LocalStorageKeys, value: Value): boolean {
33 |         let valueStr: string | null = null 
34 |         
35 |         try {
36 |             valueStr = JSON.stringify(value)
37 |         } catch {
38 |             return false
39 |         }
40 | 
41 |         localStorage.setItem(key, valueStr)
42 | 
43 |         return true
44 |     }
45 | 
46 |     static addStringToDict(localStorageKey: LocalStorageKeys, key: string): boolean {
47 |         const existingValue = LocalStorageManager.getValue(localStorageKey) ?? {}
48 | 
49 |         existingValue[key] = true
50 | 
51 |         return LocalStorageManager.setValue(localStorageKey, existingValue)
52 |     }
53 | }


--------------------------------------------------------------------------------
/report/frontend/util/mongo.ts:
--------------------------------------------------------------------------------
 1 | import { Filter, MongoClient, MongoClientOptions } from 'mongodb'
 2 | import { ScanResult } from '../types/elastic'
 3 | 
 4 | const uri = process.env.MONGODB_URI as string
 5 | 
 6 | let client
 7 | let clientPromise: Promise
 8 | 
 9 | if (!process.env.MONGODB_URI) {
10 |   throw new Error('Please add your Mongo URI to .env.local')
11 | }
12 | 
13 | if (process.env.NODE_ENV === 'development') {
14 |   // In development mode, use a global variable so that the value
15 |   // is preserved across module reloads caused by HMR (Hot Module Replacement).
16 |   // @ts-ignore
17 |   if (!global!._mongoClientPromise) {
18 |     client = new MongoClient(uri)
19 |     // @ts-ignore
20 |     global._mongoClientPromise = client.connect()
21 |   }
22 |   // @ts-ignore
23 |   clientPromise = global._mongoClientPromise
24 | } else {
25 |   // In production mode, it's best to not use a global variable.
26 |   client = new MongoClient(uri)
27 |   clientPromise = client.connect()
28 | }
29 | 
30 | // Export a module-scoped MongoClient promise. By doing this in a
31 | // separate module, the client can be shared across functions.
32 | export default clientPromise
33 | 
34 | export const usefulScanResultFilter: Filter = {
35 |     $or: [{
36 |       'scanResult.aliases.0': { $exists: true }
37 |     }, {
38 |       'scanResult.allocations.0': { $exists: true }
39 |     }, {
40 |       'scanResult.indices.0': { $exists: true }
41 |     }, {
42 |       'scanResult.indicesInfoInJson': { $ne: null }
43 |     }]
44 | }
45 | 


--------------------------------------------------------------------------------
/report/frontend/util/react-essentials.tsx:
--------------------------------------------------------------------------------
 1 | import { ComponentType, ErrorInfo, FC, memo, PureComponent, ReactNode } from "react"
 2 | import flow from "lodash.flow"
 3 | 
 4 | export const NullFallback: FC = () => null
 5 | 
 6 | export type ErrorBoundaryProps = {
 7 |   Fallback: ReactNode
 8 | }
 9 | 
10 | export type ErrorBoundaryState = {
11 |   error?: Error
12 |   errorInfo?: ErrorInfo
13 | }
14 | 
15 | export class ErrorBoundary extends PureComponent<
16 |   ErrorBoundaryProps,
17 |   ErrorBoundaryState
18 | > {
19 |   constructor(props: ErrorBoundaryProps) {
20 |     super(props)
21 |     this.state = { error: undefined, errorInfo: undefined }
22 |   }
23 | 
24 |   public componentDidCatch(error: Error, errorInfo: ErrorInfo): void {
25 |     this.setState({
26 |       error: error,
27 |       errorInfo: errorInfo,
28 |     })
29 |     /**
30 |      * @todo log Sentry here
31 |      */
32 |   }
33 | 
34 |   public render(): ReactNode {
35 |     if (this.state.error) return this.props.Fallback
36 |     return this.props.children
37 |   }
38 | }
39 | 
40 | export function withErrorBoundary(Component: ComponentType) {
41 |   return (Fallback = NullFallback) => {
42 |     // eslint-disable-next-line react/display-name
43 |     return memo(({ ...props }: Props) => {
44 |       return (
45 |         }>
46 |           
47 |         
48 |       )
49 |     })
50 |   }
51 | }
52 | 
53 | export const enhance: (
54 |   Component: FC
55 | ) => (
56 |   Fallback?: FC
57 | ) => React.MemoExoticComponent<({ ...props }: Props) => JSX.Element> = flow(
58 |   memo,
59 |   withErrorBoundary
60 | )
61 | 
62 | export type TcResult = [null, Data] | [Throws]
63 | 
64 | export async function tcAsync(
65 |   promise: Promise
66 | ): Promise> {
67 |   try {
68 |     const response: T = await promise
69 | 
70 |     return [null, response]
71 |   } catch (error) {
72 |     return [error] as [Throws]
73 |   }
74 | }
75 | 
76 | export function tcSync<
77 |   ArrType,
78 |   Params extends Array,
79 |   Returns,
80 |   Throws = Error
81 | >(
82 |   fn: (...params: Params) => Returns,
83 |   ...deps: Params
84 | ): TcResult {
85 |   try {
86 |     const data: Returns = fn(...deps)
87 | 
88 |     return [null, data]
89 |   } catch (e) {
90 |     return [e] as [Throws]
91 |   }
92 | }
93 | 
94 | export function exhaustiveCheck(x: never): void {
95 |   throw new Error(`${x} should be unreachable`)
96 | }


--------------------------------------------------------------------------------
/report/frontend/util/string.ts:
--------------------------------------------------------------------------------
 1 | export function getOnlyString(alphaNumericWord: string): string {
 2 |     return alphaNumericWord.replace(/[^a-z]/gi, '');
 3 | }
 4 | 
 5 | export function getOnlyNumber(alphaNumericWord: string): number | null {
 6 |     const floatingNumberMatches = alphaNumericWord.match(/[+-]?\d+(\.\d+)?/g)
 7 |     if (floatingNumberMatches === null) return null
 8 | 
 9 |     return parseFloat(floatingNumberMatches[0])
10 | }
11 | 


--------------------------------------------------------------------------------
/screenshots/REPORT_ROOT_SCREENSHOT.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/9oelM/elasticpwn/31f45a432401f3e656999e2d3771951c9ff7fce9/screenshots/REPORT_ROOT_SCREENSHOT.png


--------------------------------------------------------------------------------
/screenshots/REPORT_SCREENSHOT.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/9oelM/elasticpwn/31f45a432401f3e656999e2d3771951c9ff7fce9/screenshots/REPORT_SCREENSHOT.png


--------------------------------------------------------------------------------
/screenshots/WORKSPACE_INSTRUCTION.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/9oelM/elasticpwn/31f45a432401f3e656999e2d3771951c9ff7fce9/screenshots/WORKSPACE_INSTRUCTION.png


--------------------------------------------------------------------------------