├── .gitignore ├── src ├── index.js ├── style.scss └── app.jsx ├── lib ├── http-request.js ├── backends │ ├── json.js │ ├── javascript-xhr.js │ ├── python-urllib.js │ └── python-requests.js ├── frontends │ ├── json.js │ ├── http.js │ └── curl.js └── transforms │ ├── drop-content-length-header.js │ ├── drop-host-header.js │ ├── split-form-data.js │ └── parse-json-data.js ├── vite.config.js ├── .github └── workflows │ └── nodejs.yml ├── release.sh ├── package.json ├── LICENSE ├── README.md └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | dist/ 4 | node_modules/ 5 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import './app.jsx' 2 | import './style.scss' 3 | -------------------------------------------------------------------------------- /lib/http-request.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | export class Request { 4 | constructor () { 5 | this.uri = null 6 | this.method = null 7 | this.headers = {} 8 | this.cookies = [] 9 | this.body = null 10 | } 11 | 12 | hasHeader(header) { 13 | return (header in this.headers) 14 | } 15 | } 16 | 17 | -------------------------------------------------------------------------------- /lib/backends/json.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | function generateJSON(request) { 4 | return JSON.stringify(request, null, 2) 5 | } 6 | 7 | 8 | export const name = 'JSON' 9 | export const generate = generateJSON 10 | export const highlighter = 'javascript' 11 | 12 | export default { 13 | name, 14 | generate, 15 | highlighter, 16 | } 17 | -------------------------------------------------------------------------------- /lib/frontends/json.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | import { Request } from '../http-request' 4 | 5 | function parseJSON(input) { 6 | return Object.assign(new Request(), JSON.parse(input)) 7 | } 8 | 9 | export const name = 'JSON' 10 | export const parse = parseJSON 11 | export const highlighter = { name: 'javascript', json: true } 12 | 13 | export default { 14 | name, 15 | parse, 16 | highlighter, 17 | } 18 | -------------------------------------------------------------------------------- /lib/transforms/drop-content-length-header.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | function dropContentLength(request, logger=console) { 4 | if (!request.hasHeader('Content-Length')) { return } 5 | 6 | const contentLength = parseInt(request.headers['Content-Length']) 7 | if (typeof request.body === 'string' && request.body.length === contentLength) { 8 | logger.log(`The Content-Length header can be re-computed, so I dropped it`) 9 | delete request.headers['Content-Length'] 10 | } 11 | } 12 | 13 | 14 | export const name = 'Drop unnecessary Content-Length header' 15 | export const transform = dropContentLength 16 | 17 | export default { 18 | name, 19 | transform, 20 | } 21 | -------------------------------------------------------------------------------- /vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import preact from '@preact/preset-vite' 3 | import { nodePolyfills } from 'vite-plugin-node-polyfills' 4 | 5 | // TODO 6 | // Our argparse dependency relies on Node-only modules (assert, fs, path, etc.). 7 | // To make it work in the browser, we need polyfills for these modules. 8 | // It is also a CommonJS module, so we need to convert to ESM. 9 | 10 | // https://vite.dev/config/ 11 | export default defineConfig({ 12 | base: './', 13 | plugins: [ 14 | preact(), 15 | nodePolyfills({ 16 | include: ['fs', 'path', 'process', 'querystring', 'stream', 'util'], 17 | globals: { global: true, process: true }, 18 | }), 19 | ], 20 | }) 21 | -------------------------------------------------------------------------------- /.github/workflows/nodejs.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: [push, pull_request] 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | strategy: 13 | matrix: 14 | node-version: [20.x, 22.x, 24.x] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@v1 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | - run: npm ci 23 | - run: npm run build --if-present 24 | #- run: npm test 25 | # env: 26 | # CI: true 27 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | S3_BUCKET=ryan.govost.es 4 | # Should start with a slash, but not end with one 5 | S3_BUCKET_SUBDIR=/http-translator 6 | CF_DISTRIBUTION=E1PTO4RG80K1VB 7 | 8 | rm -Rf dist 9 | npm install 10 | npm run build 11 | 12 | SYNC_COMMAND=( 13 | aws s3 sync 14 | --delete 15 | --exclude .DS_Store 16 | --acl public-read 17 | dist "s3://${S3_BUCKET}${S3_BUCKET_SUBDIR}" 18 | ) 19 | 20 | "${SYNC_COMMAND[@]}" --dryrun 21 | 22 | read -p "Are you sure? " -n 1 -r 23 | echo 24 | if [[ ! $REPLY =~ ^[Yy]$ ]]; then 25 | exit 1 26 | fi 27 | 28 | # Actually run the sync command 29 | SYNC_OUTPUT=$("${SYNC_COMMAND[@]}") 30 | 31 | # Invalidate the Cloudfront cache 32 | echo "$SYNC_OUTPUT" \ 33 | | sed -Ee 's,^.*'"$S3_BUCKET"'('"$S3_BUCKET_SUBDIR"'/.*)$,\1,g' \ 34 | | xargs aws cloudfront create-invalidation \ 35 | --distribution-id "$CF_DISTRIBUTION" \ 36 | --paths 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "http-translator", 3 | "private": true, 4 | "version": "1.0.1", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "@codemirror/lang-javascript": "^6.2.4", 13 | "@codemirror/lang-json": "^6.0.2", 14 | "@codemirror/lang-python": "^6.2.1", 15 | "@uiw/react-codemirror": "^4.23.14", 16 | "argparse": "^2.0.1", 17 | "bootstrap": "^5.3.7", 18 | "http-parser-js": "^0.5.10", 19 | "preact": "^10.26.9", 20 | "shlex": "^3.0.0" 21 | }, 22 | "devDependencies": { 23 | "@preact/preset-vite": "^2.10.1", 24 | "@rollup/plugin-commonjs": "^28.0.6", 25 | "@rollup/plugin-node-resolve": "^16.0.1", 26 | "browser-stdout": "^1.3.1", 27 | "sass": "^1.89.2", 28 | "vite": "^6.3.6", 29 | "vite-plugin-node-polyfills": "^0.23.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lib/transforms/drop-host-header.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | 4 | 5 | function dropHost(request, logger=console) { 6 | if (!request.hasHeader('Host')) { return } 7 | 8 | let parsedURI = new URL(request.uri) 9 | var uselessHosts = [] 10 | if (parsedURI.port === '') { 11 | uselessHosts.push(parsedURI.hostname) 12 | if (parsedURI.protocol === 'http:') { uselessHosts.push(`${parsedURI.hostname}:80`) } 13 | else if (parsedURI.protocol === 'https:') { uselessHosts.push(`${parsedURI.hostname}:443`) } 14 | } else { 15 | uselessHosts.push(parsedURI.hostname + ':' + parsedURI.port) 16 | } 17 | 18 | if (uselessHosts.includes(request.headers['Host'])) { 19 | logger.log(`The Host header "${request.headers['Host']}" is implied by the URL, so I dropped it`) 20 | delete request.headers['Host'] 21 | } 22 | } 23 | 24 | 25 | export const name = 'Drop unnecessary Host header' 26 | export const transform = dropHost 27 | 28 | export default { 29 | name, 30 | transform, 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Ryan Govostes 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /lib/transforms/split-form-data.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | import querystring from 'querystring' 4 | 5 | 6 | // Regular expression for matching a `token`, from RFC 2616 7 | const tokenRegex = /[^()<>@,;:\\"/\[\]?={} \t]+/ 8 | const mediaTypeRegex = new RegExp( 9 | '^(' + tokenRegex.source + ')/(' + tokenRegex.source + ')' 10 | ) 11 | 12 | 13 | export function splitFormData(request, logger=console) { 14 | if (!request.hasHeader('Content-Type')) { return } 15 | 16 | // Extract the type/subtype from the Content-Type header 17 | var match = request.headers['Content-Type'].match(mediaTypeRegex) 18 | if (!match) { 19 | logger.log('I can\'t parse this Content-Type:', 20 | request.headers['Content-Type']) 21 | return 22 | } 23 | 24 | // Stop if not application/x-www-form-urlencoded 25 | if (match[1] !== 'application' || match[2] !== 'x-www-form-urlencoded') { 26 | return 27 | } 28 | 29 | // If there's a trailer afterwards, for instance the charset, we ignore it 30 | if (match[0].length != request.headers['Content-Type'].length) { 31 | let trailer = request.headers['Content-Type'].substring(match[0].length) 32 | logger.log('Ignoring trailer after Content-Type:', trailer) 33 | } 34 | 35 | request.formData = querystring.parse(request.body || '') 36 | // delete request.body 37 | } 38 | 39 | 40 | export const name = 'Split x-www-form-urlencoded content' 41 | export const transform = splitFormData 42 | 43 | export default { 44 | name, 45 | transform, 46 | } 47 | -------------------------------------------------------------------------------- /lib/transforms/parse-json-data.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | 4 | // Regular expression for matching a `token`, from RFC 2616 5 | const tokenRegex = /[^()<>@,;:\\"/\[\]?={} \t]+/ 6 | const mediaTypeRegex = new RegExp( 7 | '^(' + tokenRegex.source + ')/(' + tokenRegex.source + ')' 8 | ) 9 | 10 | 11 | export function parseJsonData(request, logger=console) { 12 | if (!request.hasHeader('Content-Type')) { return } 13 | 14 | // Extract the type/subtype from the Content-Type header 15 | var match = request.headers['Content-Type'].match(mediaTypeRegex) 16 | if (!match) { 17 | logger.log('I can\'t parse this Content-Type:', 18 | request.headers['Content-Type']) 19 | return 20 | } 21 | 22 | // Stop if not application/json 23 | if (match[1] !== 'application' || match[2] !== 'json') { 24 | return 25 | } 26 | 27 | // If there's a trailer afterwards, for instance the charset, we ignore it 28 | if (match[0].length != request.headers['Content-Type'].length) { 29 | let trailer = request.headers['Content-Type'].substring(match[0].length) 30 | logger.log('Ignoring trailer after Content-Type:', trailer) 31 | } 32 | 33 | // Try to parse 34 | try { 35 | request.jsonData = JSON.parse(request.body || '') 36 | } catch (err) { 37 | logger.log(err.message) 38 | return 39 | } 40 | 41 | // If successful, drop the Content-Type header 42 | delete request.headers['Content-Type'] 43 | } 44 | 45 | 46 | export const name = 'Parse JSON content' 47 | export const transform = parseJsonData 48 | 49 | export default { 50 | name, 51 | transform, 52 | } 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Build](https://github.com/rgov/http-translator/workflows/Node.js%20CI/badge.svg) 2 | 3 | # HTTP Request Translator 4 | 5 | This tool translates HTTP requests from a curl command-line invocation or raw HTTP request into Python or JavaScript code. 6 | 7 | You can generate curl commands from several tools: 8 | 9 | * **[Burp Suite](https://portswigger.net/burp):** Right-click on a request and select "Copy as curl command". 10 | 11 | * **[Charles](https://www.charlesproxy.com):** Right-click on a request and select "Copy cURL Request". 12 | 13 | * **Safari Web Inspector:** Right-click on a resource in the Network or Resources tab and select "Copy as cURL". 14 | 15 | Thee tool is not complete; it doesn’t know about some basic things right now (such as cookies) or less common curl options such as `--proxy-header`. Contributions are welcome. 16 | 17 | See also Matt Holt's [curl-to-Go](https://mholt.github.io/curl-to-go/), if that fits your use case better. 18 | 19 | 20 | ## Building from Source 21 | 22 | npm install && npm run build 23 | 24 | Build results will be put in `/dist`. You can also run a development webserver with `npm run dev`. 25 | 26 | 27 | ## Implementation 28 | 29 | This project is written in React. Please keep in mind this was an exercise in learning React, so the code may not be the best. 30 | 31 | **Frontends** (in `lib/frontends`) parse the input and populate a Request object (in `lib/http-request.js`). 32 | 33 | **Transforms** (in `lib/transforms`) perform some modification to the Request object, such as breaking up `application/x-www-form-urlencoded` content. 34 | 35 | **Backends** (in `lib/backends`) consume the Request and generate the output. 36 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | HTTP Request Translator 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 21 | 22 | 23 | 24 |
25 |
26 | 27 | 28 | 29 | 30 |
31 | 32 |

HTTP Request Translator

33 |

brought to you by Ryan Govostes

34 |
35 | 36 |
37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /lib/frontends/http.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | import { Request } from '../http-request' 4 | // This is a JavaScript reimplementation of Node's HTTP parser 5 | import * as httpParserJs from 'http-parser-js' 6 | const { HTTPParser } = httpParserJs 7 | 8 | 9 | function parseHTTP(input, logger=console) { 10 | // Since it may be hard to get CRLFs into the text area, we can try to be 11 | // smart about adding them back in. 12 | if (!input.includes('\r\n')) { 13 | let i = input.indexOf('\n\n') 14 | if (i !== -1) { 15 | let before = input.substring(0, i+2) 16 | let after = input.substring(i+2) 17 | input = before.replace(/\n/g, '\r\n') + after 18 | } 19 | } 20 | 21 | return new Promise((resolve, reject) => { 22 | var parser = new HTTPParser(HTTPParser.REQUEST) 23 | var request = new Request() 24 | 25 | parser[HTTPParser.kOnHeadersComplete] = (info) => { 26 | request.uri = info.url 27 | request.method = HTTPParser.methods[info.method] 28 | for (let i = 0; i < info.headers.length; i += 2) { 29 | request.headers[info.headers[i]] = info.headers[i+1] 30 | } 31 | } 32 | 33 | parser[HTTPParser.kOnBody] = (b, start, len) => { 34 | request.body = b.toString('utf8', start, start + len) 35 | } 36 | 37 | parser[HTTPParser.kOnMessageComplete] = () => { 38 | // If there is a Host header, we need to patch up the URL 39 | if (request.hasHeader('Host')) { 40 | request.uri = `http://${request.headers['Host']}${request.uri}` 41 | } else { 42 | request.uri = `http://localhost${request.uri}` 43 | } 44 | resolve(request) 45 | } 46 | 47 | // Execute the parser 48 | let buffer = Buffer.from(input) 49 | parser.execute(buffer, 0, buffer.length) 50 | parser.finish() 51 | }) 52 | } 53 | 54 | 55 | export const name = 'HTTP' 56 | export const parse = parseHTTP 57 | export const highlighter = 'http' 58 | 59 | export const example = ` 60 | POST / HTTP/1.1 61 | Host: example.com 62 | User-Agent: curl/7.54.0 63 | Accept: */* 64 | Content-Length: 11 65 | Content-Type: application/x-www-form-urlencoded 66 | 67 | hello=world 68 | `.trim() 69 | 70 | export default { 71 | name, 72 | parse, 73 | highlighter, 74 | example, 75 | } 76 | -------------------------------------------------------------------------------- /lib/backends/javascript-xhr.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | 4 | // https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name 5 | const xhrForbiddenHeaders = [ 6 | "Accept-Charset", 7 | "Accept-Encoding", 8 | "Access-Control-Request-Headers", 9 | "Access-Control-Request-Method", 10 | "Connection", 11 | "Content-Length", 12 | "Cookie", 13 | "Cookie2", 14 | "Date", 15 | "DNT", 16 | "Expect", 17 | "Host", 18 | "Keep-Alive", 19 | "Origin", 20 | /^Proxy-/i, 21 | /^Sec-/i, 22 | "Referer", 23 | "TE", 24 | "Trailer", 25 | "Transfer-Encoding", 26 | "Upgrade", 27 | "Via" 28 | ] 29 | 30 | function javascript_escape(value, quote='"') { 31 | return value.replace(new RegExp('([' + quote + '\\\\])', 'g'), '\\$1') 32 | .replace(/\n/g, '\\n') 33 | .replace(/\t/g, '\\t') 34 | .replace(/\r/g, '\\r') 35 | .replace(/[^ -~]+/g, function (match) { 36 | var hex = new Buffer(match).toString('hex') 37 | return hex.replace(/(..)/g, '\\x$1') 38 | }) 39 | } 40 | 41 | function generateJavascriptXHR(request, logger=console) { 42 | // Emit the preamble 43 | var code = [] 44 | code.push('var xhr = new XMLHttpRequest()') 45 | 46 | // Parse the URI 47 | var parsedURI = new URL(request.uri) 48 | 49 | // Remove the username and password from the URL 50 | var username = parsedURI.username 51 | var password = parsedURI.password 52 | parsedURI.username = parsedURI.password = '' 53 | 54 | // Output the open() method 55 | var open = `xhr.open("${request.method}", "${parsedURI.href}"` 56 | open += ", false" // asynchronous 57 | if (!!username && !!password) { 58 | open += `, "${username}", "${password}"` 59 | } else if (!!username && !password) { 60 | open += `, "${username}"` 61 | } else if (!username && !!password) { 62 | open += `, "", "${password}"` 63 | } 64 | open += ')' 65 | code.push(open) 66 | 67 | // Emit code for the headers 68 | if (Object.keys(request.headers).length > 0) { 69 | for (const name in request.headers) { 70 | // Scan the list of forbidden headers 71 | var verboten = false 72 | for (const pattern of xhrForbiddenHeaders) { 73 | console.log(pattern) 74 | if (typeof pattern === 'string') { 75 | if (name.toLowerCase() === pattern.toLowerCase()) { 76 | verboten = true 77 | break 78 | } 79 | } else if (name.match(pattern)) { 80 | verboten = true 81 | break 82 | } 83 | } 84 | 85 | if (verboten) { 86 | logger.error(`Omitting forbidden ${name} header`) 87 | continue 88 | } 89 | 90 | const value = request.headers[name] 91 | code.push(`xhr.setRequestHeader("${name}", "${value}")`) 92 | } 93 | } 94 | 95 | // Add some placeholders for handlers 96 | code.push('xhr.onload = function (e) { /* ... */ }') 97 | code.push('xhr.onerror = function (e) { /* ... */ }') 98 | 99 | // Add the send command 100 | if (request.jsonData !== undefined) { 101 | var json = JSON.stringify(request.jsonData, null, 2) 102 | code.push(`xhr.send(JSON.stringify(${json}))`) 103 | } else if (typeof request.body === 'string' && request.body != '') { 104 | code.push(`xhr.send("${javascript_escape(request.body)}")`) 105 | } else { 106 | code.push(`xhr.send(null)`) 107 | } 108 | 109 | 110 | return code.join('\n') 111 | } 112 | 113 | 114 | export const name = 'JavaScript (XMLHttpRequest)' 115 | export const generate = generateJavascriptXHR 116 | export const highlighter = 'javascript' 117 | 118 | export default { 119 | name, 120 | generate, 121 | highlighter, 122 | } 123 | -------------------------------------------------------------------------------- /lib/backends/python-urllib.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | function python_escape(value, quote='\'') { 4 | return value.replace(new RegExp('([' + quote + '\\\\])', 'g'), '\\$1') 5 | .replace(/\n/g, '\\n') 6 | .replace(/\t/g, '\\t') 7 | .replace(/\r/g, '\\r') 8 | .replace(/[^ -~]+/g, function (match) { 9 | var hex = Buffer.from(match).toString('hex') 10 | return hex.replace(/(..)/g, '\\x$1') 11 | }) 12 | } 13 | 14 | function pythonize(obj) { 15 | if (typeof obj === 'string') { 16 | return '\'' + python_escape(obj) + '\'' 17 | } else if (typeof obj === 'number') { 18 | return obj 19 | } else if (typeof obj === 'boolean') { 20 | return obj ? 'True' : 'False' 21 | } else if (obj === null || obj === undefined) { 22 | return 'None' 23 | } else if (Array.isArray(obj)) { 24 | return '[ ' + obj.map(pythonize).join(', ') + ' ]' 25 | } else if (obj.constructor === Object) { 26 | var rows = [] 27 | for (let k in obj) { 28 | rows.push(pythonize(k) + ': ' + pythonize(obj[k])) 29 | } 30 | return '{ ' + rows.join(', ') + ' }' 31 | } else { 32 | throw 'unknown type' 33 | } 34 | } 35 | 36 | function generatePythonUrllib(request, logger=console) { 37 | // Determine required imports 38 | const imports = new Set() 39 | imports.add('urllib.request') 40 | 41 | // Emit preamble 42 | const code = [] 43 | code.push('req = urllib.request.Request(') 44 | 45 | // Emit the URL 46 | const parsedURI = new URL(request.uri) 47 | const emitQueryParams = parsedURI.search !== '' 48 | if (emitQueryParams) { 49 | imports.add('urllib.parse') 50 | code.push(` ${pythonize(parsedURI.origin + parsedURI.pathname + '?')} + urllib.parse.urlencode({`) 51 | for (const [name, value] of parsedURI.searchParams) { 52 | code.push(` ${pythonize(name)}: ${pythonize(value)},`) 53 | } 54 | code.push(' }),') 55 | } else { 56 | code.push(` ${pythonize(request.uri)},`) 57 | } 58 | 59 | // Emit the headers 60 | if (Object.keys(request.headers).length > 0) { 61 | code.push(' headers={') 62 | for (const name in request.headers) { 63 | code.push(` ${pythonize(name)}: ${pythonize(request.headers[name])},`) 64 | } 65 | code.push(' },') 66 | } 67 | 68 | // Emit the body 69 | const hasFormData = request.formData && Object.keys(request.formData).length > 0 70 | if (request.jsonData !== undefined) { 71 | imports.add('json') 72 | code.push(` data=json.dumps(${pythonize(request.jsonData)}).encode(),`) 73 | } else if (hasFormData) { 74 | imports.add('urllib.parse') 75 | code.push(' data=urllib.parse.urlencode({') 76 | for (const name in request.formData) { 77 | code.push(` ${pythonize(name)}: ${pythonize(request.formData[name])},`) 78 | } 79 | code.push(` }).encode(),`) 80 | } else if (typeof request.body === 'string' && request.body !== '') { 81 | code.push(` data=b${pythonize(request.body)},`) 82 | } 83 | 84 | // Emit the method 85 | code.push(` method=${pythonize(request.method)}`) 86 | 87 | // Finalize the code 88 | code.push(')') 89 | code.push('') 90 | code.push('with urllib.request.urlopen(req) as response:') 91 | code.push(' body = response.read().decode()') 92 | 93 | // Insert the needed imports at the top 94 | code.splice(0, 0, '') // blank line before imports 95 | new Array(...imports) 96 | .sort((a, b) => b.localeCompare(a)) // reverse order 97 | .forEach(x => code.splice(0, 0, `import ${x}`)); 98 | 99 | 100 | return code.join('\n') 101 | } 102 | 103 | export const name = 'Python urllib' 104 | export const generate = generatePythonUrllib 105 | export const highlighter = 'python' 106 | 107 | export default { 108 | name, 109 | generate, 110 | highlighter 111 | } 112 | -------------------------------------------------------------------------------- /lib/frontends/curl.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | import * as argparse from 'argparse' 4 | import * as shlex from 'shlex' 5 | import BrowserStdout from 'browser-stdout' 6 | import { Request } from '../http-request' 7 | 8 | 9 | // Patch process.stdout to use browser-stdout 10 | process.stdout = BrowserStdout() 11 | 12 | 13 | // Patch argparse.ArgumentParser to log messages, rather than die 14 | class ArgumentParser extends argparse.ArgumentParser { 15 | _printMessage (message) { 16 | this.logger.log(message.replace(/\s+$/, '')) 17 | } 18 | 19 | exit (status, message) { 20 | if (message !== undefined) { 21 | (status ? this.logger.error : this.logger.log)(message.replace(/\s+$/, '')) 22 | } 23 | this.logger.log('The argument parser exited with status', status) 24 | this.exit_status = status 25 | } 26 | } 27 | 28 | 29 | function parseCurlCommandLine(command, logger=console) { 30 | var argv = shlex.split(command) 31 | argv.shift() // consume the leading `curl` 32 | 33 | var parser = new ArgumentParser({ prog: 'curl' }) 34 | parser.logger = logger // allow _printMessage patch to use our logger 35 | parser.add_argument('url') 36 | parser.add_argument('--request', '-X', { dest: 'method' }) 37 | parser.add_argument('--header', '-H', { dest: 'headers', action: 'append'}) 38 | parser.add_argument('--referer', '-e') 39 | parser.add_argument('--user-agent', '-A') 40 | parser.add_argument('--data', '--data-ascii', '--data-binary', '--data-raw', 41 | /*'--data-urlencode',*/ '-d', { action: 'append'}) 42 | var [args, extra] = (parser.parse_known_args(argv) || []) 43 | 44 | // If the parser died, don't continue 45 | if (parser.exit_status !== undefined) { return } 46 | 47 | // We don't parse all arguments. Be transparent about it. 48 | if (extra.length > 0) { 49 | logger.log('I skipped these unsupported arguments:', extra) 50 | } 51 | 52 | var req = new Request() 53 | req.uri = args.url 54 | 55 | for (const header of (args.headers || [])) { 56 | var match 57 | if (match = header.match(/^\s*([^:]+);\s*$/)) { 58 | req.headers[match[1]] = '' 59 | } else if (match = header.match(/^\s*([^:]+):\s*(.*?)\s*$/)) { 60 | req.headers[match[1]] = match[2] 61 | } else if (match = header.match(/^\s*([^:]+):\s*$/)) { 62 | // Tricky case, this syntax will unset a default header, but otherwise 63 | // won't unset a 64 | var name = match[1] 65 | logger.log('I don\'t support unsetting headers (like ' + name + ') yet.') 66 | } else { 67 | logger.log('I don\'t understand this header:', header) 68 | } 69 | } 70 | 71 | // TODO: curl supports many ways of specifying the body of the request. We 72 | // will need a custom argparse Action that keeps the data in order. 73 | // 74 | // * --data, --data-ascii, --data-raw -- possibly strip whitespace 75 | // * --data-binary -- do not strip whitespace 76 | // * --data-urlencode -- URL encode the *value* but not the key (if present) 77 | if (args.data) { 78 | if (!('Content-Type' in req.headers)) { 79 | req.headers['Content-Type'] = 'application/x-www-form-urlencoded' 80 | } 81 | 82 | req.method = 'POST' 83 | req.body = args.data.join('&') 84 | } 85 | 86 | // Allow overriding the method, or fall back to GET 87 | if (args.method) { 88 | req.method = args.method 89 | } else if (req.method === null) { 90 | req.method = 'GET' 91 | } 92 | 93 | return req 94 | } 95 | 96 | 97 | export const name = 'curl' 98 | export const parse = parseCurlCommandLine 99 | export const highlighter = 'shell' 100 | 101 | export const example = ` 102 | curl 'http://example.com/api/v2/defragment' \\ 103 | -H 'Host: example.com' \\ 104 | -H 'Content-Length: 19' \\ 105 | -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \\ 106 | -H 'Cookie: session=291419e390a3b67a3946e0854cc9e33e' \\ 107 | -H 'Referer: http://example.com/' \\ 108 | -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15' \\ 109 | -H 'X-Requested-With: XMLHttpRequest' \\ 110 | --data 'drive=C&confirm=yes' 111 | `.trim() 112 | 113 | export default { 114 | name, 115 | parse, 116 | highlighter, 117 | example, 118 | } 119 | -------------------------------------------------------------------------------- /lib/backends/python-requests.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | 4 | 5 | function python_escape(value, quote='\'') { 6 | return value.replace(new RegExp('([' + quote + '\\\\])', 'g'), '\\$1') 7 | .replace(/\n/g, '\\n') 8 | .replace(/\t/g, '\\t') 9 | .replace(/\r/g, '\\r') 10 | .replace(/[^ -~]+/g, function (match) { 11 | var hex = new Buffer(match).toString('hex') 12 | return hex.replace(/(..)/g, '\\x$1') 13 | }) 14 | } 15 | 16 | function pythonize(obj) { 17 | // TODO: Pretty printing 18 | if (typeof obj === 'string') { 19 | return '\'' + python_escape(obj) + '\'' 20 | } else if (typeof obj === 'number') { 21 | return obj 22 | } else if (typeof obj === 'boolean') { 23 | return obj ? 'True' : 'False' 24 | } else if (obj === null || obj === undefined) { 25 | return 'None' 26 | } else if (Array.isArray(obj)) { 27 | return '[ ' + obj.map(pythonize).join(', ') + ' ]' 28 | } else if (obj.constructor === Object) { 29 | var rows = [] 30 | for (let k in obj) { 31 | rows.push(pythonize(k) + ': ' + pythonize(obj[k])) 32 | } 33 | return '{ ' + rows.join(', ') + ' }' 34 | } else { 35 | throw 'unknown type' 36 | } 37 | } 38 | 39 | function generatePythonRequests(request, logger=console) { 40 | // Emit the preamble 41 | var code = [] 42 | code.push('import requests') 43 | code.push('') 44 | 45 | // Map the HTTP method to the function 46 | const methods = { 47 | DELETE: 'requests.delete', 48 | GET: 'requests.get', 49 | HEAD: 'requests.head', 50 | OPTIONS: 'requests.options', 51 | POST: 'requests.post', 52 | PUT: 'requests.put', 53 | } 54 | var func = methods[request.method] || 'requests.request' 55 | var isCustomMethod = !methods.hasOwnProperty(request.method) 56 | 57 | // Parse the URI 58 | var parsedURI = new URL(request.uri) 59 | 60 | // Try to break out URL parameters from the query string, if we can 61 | var emitQueryParams = (parsedURI.search !== '') 62 | if (emitQueryParams) { 63 | // Modify the URI we'll eventually emit so that it doesn't include the 64 | // query string 65 | request.uri = `${parsedURI.origin}${parsedURI.pathname}` 66 | } 67 | 68 | // Emit the function call, URL, and method 69 | code.push(`${func}(`) 70 | code.push(` ${pythonize(request.uri)},`) 71 | if (isCustomMethod) { code.push(` method=${pythonize(request.method)},`) } 72 | 73 | // Emit the URL params 74 | if (emitQueryParams) { 75 | // FIXME: We don't handle multiple parameters with the same name correctly 76 | code.push(' params={') 77 | for (const [name, value] of parsedURI.searchParams) { 78 | code.push(` ${pythonize(name)}: ${pythonize(value)},`) 79 | } 80 | code.push(' },') 81 | } 82 | 83 | // Emit code for the headers 84 | if (Object.keys(request.headers).length > 0) { 85 | code.push(' headers={') 86 | for (const name in request.headers) { 87 | const value = request.headers[name] 88 | code.push(` ${pythonize(name)}: ${pythonize(value)},`) 89 | } 90 | code.push(' },') 91 | } 92 | 93 | // Try to convert JSON to a Python object 94 | let json = undefined 95 | if (request.jsonData !== undefined) { 96 | try { 97 | json = pythonize(request.jsonData) 98 | } catch (err) { 99 | logger.log('Failed to represent JSON as Python') 100 | } 101 | } 102 | 103 | // Emit code for the data 104 | if (json) { 105 | code.push(` json=${json},`) 106 | } else if (request.formData !== undefined && Object.keys(request.formData).length > 0) { 107 | code.push(' data={') 108 | for (const name in request.formData) { 109 | const value = request.formData[name] 110 | code.push(` ${pythonize(name)}: ${pythonize(value)},`) 111 | } 112 | code.push(' },') 113 | } else if (typeof request.body === 'string' && request.body != '') { 114 | code.push(` data=${pythonize(request.body)},`) 115 | } 116 | 117 | // Finish the code and return it 118 | code.push(')') 119 | return code.join('\n') 120 | } 121 | 122 | 123 | export const name = 'Python Requests' 124 | export const generate = generatePythonRequests 125 | export const highlighter = 'python' 126 | 127 | export default { 128 | name, 129 | generate, 130 | highlighter, 131 | } 132 | -------------------------------------------------------------------------------- /src/style.scss: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css?family=Crete+Round'); 2 | 3 | %red-triangle { 4 | content: ""; 5 | 6 | /* useful: http://apps.eky.hk/css-triangle-generator/ */ 7 | width: 0; 8 | height: 0; 9 | border-style: solid; 10 | border-width: 0 4px 6.9px 4px; 11 | border-color: transparent transparent red transparent; 12 | } 13 | 14 | %gray-circle { 15 | content: ""; 16 | 17 | width: 6px; 18 | height: 6px; 19 | background-color: gray; 20 | border-radius: 50%; 21 | } 22 | 23 | body { 24 | font-family: sans-serif; 25 | margin: 0; 26 | } 27 | 28 | .hide { 29 | display: none; 30 | } 31 | 32 | h1, h2, h3, h4, h5, h6 { 33 | font-family: 'Crete Round', serif; 34 | margin: 0; 35 | } 36 | 37 | h1 { 38 | font-size: 48px; 39 | } 40 | 41 | a { 42 | text-decoration: none; 43 | } 44 | 45 | .github { 46 | float: right; 47 | margin-left: 20px; 48 | 49 | .icon { 50 | width: 48px; 51 | opacity: 0.4; 52 | transition: opacity .15s ease-in-out; 53 | 54 | &:hover { 55 | opacity: 1.0; 56 | } 57 | } 58 | } 59 | 60 | .top { 61 | padding: 20px; 62 | 63 | ul { 64 | list-style: none; 65 | padding: 0; 66 | } 67 | ul li { 68 | display: inline; 69 | padding-right: 40px; 70 | } 71 | p { 72 | margin-top: 10px; 73 | } 74 | } 75 | 76 | .CodeMirror { 77 | height: auto; 78 | } 79 | 80 | .parent { 81 | display: flex; 82 | flex-wrap: wrap; 83 | } 84 | 85 | .child { 86 | width: 50%; 87 | } 88 | 89 | button, select { 90 | padding:.375rem .75rem; 91 | color: #495057; 92 | vertical-align: middle; 93 | border: 1px solid #ced4da; 94 | border-radius: .25rem; 95 | -webkit-appearance: none; 96 | -moz-appearance: none; 97 | appearance: none; 98 | margin: 2px; 99 | } 100 | 101 | button { 102 | color: #007bff; 103 | background-color: transparent; 104 | background-image: none; 105 | border-color: #007bff; 106 | text-align: center; 107 | transition: color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out; 108 | 109 | &:not(:disabled):hover, &:not(:disabled):active { 110 | color: #fff; 111 | background-color: #007bff; 112 | border-color: #007bff; 113 | } 114 | 115 | &:disabled { 116 | color: #007bff; 117 | background-color: transparent; 118 | opacity: .65; 119 | } 120 | 121 | &:not(:disabled) { 122 | cursor: pointer; 123 | } 124 | } 125 | 126 | 127 | 128 | select { 129 | padding-right: 1.75rem; 130 | background: #fff url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3E%3Cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3E%3C/svg%3E") no-repeat right .75rem center; 131 | background-size: 8px 10px; 132 | } 133 | 134 | .nav { 135 | display: flex; 136 | justify-content: space-between; 137 | flex-wrap: wrap; 138 | 139 | border-bottom: 1px solid #dee2e6; 140 | padding: 0 20px; 141 | 142 | .nav-tabs { 143 | display: flex; 144 | list-style: none; 145 | padding: 0; 146 | margin-bottom: 0; 147 | 148 | .nav-item { 149 | margin-bottom: -1px; 150 | } 151 | 152 | .nav-link { 153 | display: block; 154 | padding: .45rem .9rem; 155 | 156 | font-family: 'Crete Round', serif; 157 | border: 1px solid transparent; 158 | border-top-left-radius: .25rem; 159 | border-top-right-radius: .25rem; 160 | 161 | color: #666; 162 | 163 | &.active { 164 | color: black; 165 | background-color: white; 166 | border-color: #dee2e6 #dee2e6 white; 167 | } 168 | 169 | &.error:after { 170 | @extend %red-triangle; 171 | 172 | left: 0.25em; 173 | position: relative; 174 | 175 | display: inline-block; 176 | } 177 | } 178 | } 179 | } 180 | 181 | 182 | .logger { 183 | font-family: monospace; 184 | padding-right: 1em; 185 | 186 | ul { 187 | list-style: none; 188 | padding: 0; 189 | 190 | li { 191 | 192 | &.section { 193 | font-weight: bold; 194 | padding-left: 1em; 195 | padding-top: 1em; 196 | 197 | &:first-child { 198 | padding-top: 0 199 | } 200 | } 201 | 202 | &:not(.section) { 203 | padding-left: 2em; 204 | } 205 | 206 | &.error:before { 207 | @extend %red-triangle; 208 | 209 | left: -0.9em; /* looks better */ 210 | top: 0.8em; 211 | position: relative; 212 | 213 | display: block; 214 | } 215 | 216 | &.log:before { 217 | @extend %gray-circle; 218 | 219 | left: -0.8em; 220 | top: 0.8em; 221 | position: relative; 222 | 223 | display: block; 224 | } 225 | } 226 | 227 | } 228 | } 229 | 230 | 231 | @media (max-width: 800px) { 232 | .child { 233 | width: 100%; 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/app.jsx: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // Removed Node's util.inspect; using JSON.stringify for browser-friendly logging 4 | 5 | import React from 'react' 6 | import ReactDOM from 'react-dom' 7 | 8 | import CodeMirror from '@uiw/react-codemirror' 9 | import { javascript as langJavaScript } from '@codemirror/lang-javascript' 10 | import { json as langJson } from '@codemirror/lang-json' 11 | import { python as langPython } from '@codemirror/lang-python' 12 | import { EditorView, lineNumbers } from '@codemirror/view' 13 | 14 | import frontendCurl from '../lib/frontends/curl' 15 | import frontendHttp from '../lib/frontends/http' 16 | import frontendJson from '../lib/frontends/json' 17 | 18 | import transformDropContentLengthHeader from '../lib/transforms/drop-content-length-header' 19 | import transformDropHostHeader from '../lib/transforms/drop-host-header' 20 | import transformParseJsonData from '../lib/transforms/parse-json-data' 21 | import transformSplitFormData from '../lib/transforms/split-form-data' 22 | 23 | import backendJavascriptXhr from '../lib/backends/javascript-xhr' 24 | import backendJson from '../lib/backends/json' 25 | import backendPythonRequests from '../lib/backends/python-requests' 26 | import backendPythonUrllib from '../lib/backends/python-urllib' 27 | 28 | 29 | class Logger extends React.Component { 30 | constructor() { 31 | super() 32 | this.state = { 33 | messages: [] 34 | } 35 | 36 | this.log = this.log.bind(this) 37 | this.error = this.error.bind(this) 38 | } 39 | 40 | clear() { 41 | this.setState({ messages: [] }) 42 | } 43 | 44 | addSection(name) { 45 | this._appendLog('section', [ name ]) 46 | } 47 | 48 | log() { 49 | this._appendLog('log', Array.from(arguments)) 50 | } 51 | 52 | error() { 53 | console.error.apply(console, Array.from(arguments)) 54 | this._appendLog('error', Array.from(arguments)) 55 | } 56 | 57 | hasError() { 58 | for (let message of this.state.messages) { 59 | if (message.type === 'error') 60 | return true; 61 | } 62 | return false; 63 | } 64 | 65 | _appendLog(type, message) { 66 | let strmsg = ''; 67 | for (let part of message) { 68 | if (typeof part === 'string') { 69 | strmsg += `${part} ` 70 | } else { 71 | strmsg += JSON.stringify(part, null, 2) 72 | } 73 | } 74 | 75 | this.setState({ messages: [ 76 | ...this.state.messages, 77 | { type: type, content: strmsg } 78 | ]}) 79 | } 80 | 81 | render() { 82 | return ( 83 |
84 | 89 |
90 | ) 91 | } 92 | } 93 | 94 | 95 | class App extends React.Component { 96 | constructor() { 97 | super() 98 | this.state = {} 99 | 100 | // List the frontends, transforms, and backends we want to load 101 | this.state.frontends = [ 102 | frontendCurl, 103 | frontendHttp, 104 | frontendJson, 105 | ] 106 | this.state.transforms = [ 107 | transformDropContentLengthHeader, 108 | transformDropHostHeader, 109 | transformParseJsonData, 110 | transformSplitFormData, 111 | ] 112 | this.state.backends = [ 113 | backendJavascriptXhr, 114 | backendJson, 115 | backendPythonRequests, 116 | backendPythonUrllib, 117 | ] 118 | 119 | // Choose the default frontends 120 | this.state.frontend = frontendCurl 121 | this.state.backend = backendPythonUrllib 122 | 123 | // Set the default input and clear output 124 | this.state.input = this.state.frontend.example 125 | this.state.output = '' 126 | 127 | // Set the tab we want to show 128 | this.state.showLogTab = false 129 | 130 | // Bind `this` so that handlers for JavaScript events work 131 | this.handleFrontendChange = this.handleFrontendChange.bind(this) 132 | this.handleBackendChange = this.handleBackendChange.bind(this) 133 | 134 | // These will store our code editor instances, after they've mounted 135 | this.inputEditor = null 136 | this.outputEditor = null 137 | } 138 | 139 | componentDidMount() { 140 | // Kick off the initial generation 141 | this.handleCodeChange() 142 | } 143 | 144 | componentDidUpdate(prevProps, prevState, snapshot) { 145 | // Re-generate if any of the input parameters changed 146 | if ((this.state.input !== prevState.input || 147 | this.state.frontend !== prevState.frontend || 148 | this.state.backend !== prevState.backend) && 149 | this.state.output === prevState.output) { 150 | 151 | this.handleCodeChange() 152 | } 153 | } 154 | 155 | handleCodeChange() { 156 | this.logger.clear() 157 | var p = Promise.resolve(this.state.input) 158 | p.then((input) => { 159 | this.logger.addSection(`Frontend: ${this.state.frontend.name}`) 160 | return this.state.frontend.parse(input, this.logger) 161 | }).then((request) => { 162 | for (let transform of this.state.transforms) { 163 | this.logger.addSection(`Transform: ${transform.name}`) 164 | transform.transform(request, this.logger) 165 | } 166 | return request 167 | }).then((request) => { 168 | this.logger.addSection(`Backend: ${this.state.backend.name}`) 169 | return this.state.backend.generate(request, this.logger) 170 | }).then((output) => { 171 | this.setState({ output: output }) 172 | }).catch((error) => { 173 | this.logger.error(error) 174 | }) 175 | } 176 | 177 | handleFrontendChange(event) { 178 | for (var frontend of this.state.frontends) { 179 | if (frontend.name === event.target.value) { 180 | this.setState({ frontend: frontend }) 181 | return 182 | } 183 | } 184 | } 185 | 186 | handleBackendChange(event) { 187 | for (var backend of this.state.backends) { 188 | if (backend.name === event.target.value) { 189 | this.setState({ backend: backend }) 190 | return 191 | } 192 | } 193 | } 194 | 195 | getLangHighlighter(highlighter) { 196 | switch (highlighter) { 197 | case 'javascript': return langJavaScript(); 198 | case 'json': return langJson(); 199 | case 'python': return langPython(); 200 | // no http or shell highlighter 201 | default: return langJavaScript(); 202 | } 203 | } 204 | 205 | render() { 206 | return ( 207 |
208 |
209 |
210 |
    211 |
  • Input
    • 212 |
213 |
    214 |
  • 215 | 221 |
    • 222 |
  • 223 | 228 |
    • 229 |
230 |
231 |
232 | { this.setState({ input: value }) }} 240 | onCreateEditor={(view) => { this.inputEditor = view }} 241 | /> 242 |
243 |
244 | 245 |
246 |
247 | 265 |
    266 |
  • 267 | 272 |
    • 273 |
274 |
275 |
276 |
277 | this.logger = c} /> 278 |
279 |
280 | { this.outputEditor = view }} 288 | editable={false} 289 | /> 290 |
291 |
292 |
293 |
294 | ) 295 | } 296 | } 297 | 298 | ReactDOM.render(, document.getElementById('root')) 299 | --------------------------------------------------------------------------------