├── .gitignore ├── LICENSE.md ├── README.md ├── captures ├── Local │ ├── log-client.txt │ ├── log-server.txt │ ├── pcap │ │ ├── ngtcp2-pcap-decrypted.pcap │ │ ├── ngtcp2-pcap-encrypted.pcap │ │ └── ngtcp2-pcap.json │ ├── pcapng │ │ ├── ngtcp2-pcapng-decrypted.pcapng │ │ ├── ngtcp2-pcapng-encrypted.pcapng │ │ ├── ngtcp2-pcapng-injected.pcapng │ │ └── ngtcp2-pcapng.json │ └── ssl.key └── QUIC Tracker │ ├── 20181219_handshake_v6_quicker.edm.uhasselt.be.keys │ └── pcap │ ├── 20181219_handshake_v6_quicker.edm.uhasselt.be.json │ └── 20181219_handshake_v6_quicker.edm.uhasselt.be.pcap ├── examples ├── 20181219_handshake_v6_quicker.edm.uhasselt.be.qlog ├── draft-01 │ ├── ack_range.json │ ├── ack_range.keys │ ├── ack_range.pcapng │ ├── ack_range.qlog │ ├── multi_stream.json │ ├── multi_stream.keys │ ├── multi_stream.pcap │ ├── multi_stream.qlog │ ├── new_cid.json │ ├── new_cid.keys │ ├── new_cid.pcap │ ├── new_cid.qlog │ ├── ngtcp2_test.json │ ├── ngtcp2_test.keys │ ├── ngtcp2_test.pcapng │ ├── ngtcp2_test.qlog │ ├── spin_bit.json │ ├── spin_bit.keys │ ├── spin_bit.pcap │ └── spin_bit.qlog ├── draft-02 │ ├── ngtcp2-29-dsb.json │ ├── ngtcp2-29-dsb.pcapng │ └── ngtcp2-29-dsb.qlog ├── ngtcp2-pcap.qlog └── testlist_remote.json ├── package.json ├── pcap_to_qlog_spec.txt ├── src ├── flow │ ├── downloader.ts │ ├── jsontoqlog.ts │ └── pcaptojson.ts ├── main.ts ├── parsers │ └── ParserPCAP.ts └── util │ ├── FileUtil.ts │ └── PCAPUtil.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | out/ 4 | package-lock.json 5 | 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 QUIClog 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pcap2qlog 2 | A tool to convert .pcap and .pcapng files into qlog files. 3 | 4 | It uses Wireshark's TShark utility under the hood to transform the packet captures to Wireshark's JSON format, which is then transposed to qlog. 5 | 6 | # Prerequisites 7 | 8 | - git 9 | - wget (when passing in URLs instead of local files) 10 | - NodeJS (v10+) 11 | - Wireshark / TShark version 3.3+ (support for QUIC draft-29+) 12 | 13 | Disclaimer: this tool has only been tested on Linux, other operating systems might work but installation and usage can differ from this guide. 14 | For an overview of how we build this tool and Wireshark ourselves, refer to the dockerfiles here: https://github.com/quiclog/qvis-server/blob/master/system/docker_setup 15 | 16 | ## Installation 17 | 18 | Start by cloning the repository to your local machine and entering the cloned directory. 19 | 20 | ```sh 21 | git clone https://github.com/quiclog/pcap2qlog 22 | cd pcap2qlog 23 | ``` 24 | 25 | Run the following commands to set up the NodeJS project and compile the TypeScript code to JavaScript which can then be run by NodeJS: 26 | 27 | ```sh 28 | npm install 29 | npx tsc 30 | ``` 31 | 32 | After compiling the TypeScript code, the tool can be executed as follows (assuming you are in the root directory of the project): 33 | 34 | ```sh 35 | node out/main.js 36 | ``` 37 | 38 | ## Options: 39 | 40 | ``` 41 | --tshark=/path/to/tshark, -t /path/to/tshark 42 | Points the tool to your local installation path of tshark 43 | Defaults to /wireshark/run/tshark when omitted 44 | 45 | --input=/path/to/input.(pcap | .pcapng | json), -i /path/to/input.(pcap | .pcapng | json) 46 | Path or URL to a single trace in either PCAP, PCAPNG format or JSON format. 47 | JSON and PCAPNG files are assumed to be decrypted (or containining decryption keys) while PCAP files are assumed to encrypted. 48 | The --secrets flag should therefore be set separately when passing PCAP files. 49 | 50 | --secrets=/path/to/secrets.keys, -s /path/to/secrets 51 | Path or URL to the secrets file which has to be used to decrypt the PCAP given using the --input flag. 52 | 53 | --list=/path/to/list.json, -l /path/to/list.json 54 | Path to a file containing a list of traces which will converted to a single qlog file. 55 | The input file should have a structure along the lines of the following: 56 | { 57 | "description": "top-level description", 58 | "paths": [ 59 | { "capture": "https://...", "secrets": "https://...", "description" : "per-file desc" }, 60 | ... 61 | ] 62 | } 63 | 64 | --output=/path/to/output/directory 65 | Path to a local output directory where the resulting .qlog will be stored. 66 | The tool automatically generates a hashed filename for each .qlog file, based on the input path/URL. 67 | There is currently no option to specify the output filename directly. 68 | pcap2qlog will write the chosen filename to stdout as its only output so it can be found by other tools calling pcap2qlog. 69 | ``` 70 | 71 | NOTE: This tool can also be used to merge together multiple (partial) .qlog files. To do this, just pass the files with the --list option. 72 | 73 | NOTE: set the PCAPDEBUG environment variable to true (e.g., `PCAPDEBUG=true node out/main.js`) to get debugging output. 74 | 75 | ## Examples 76 | 77 | ```sh 78 | # Using a list of traces 79 | node out/main.js --list=/allinone.json --output=/output_dir --tshark=/path/to/tshark 80 | 81 | # TShark is not used if the input is already in the decrypted JSON format 82 | node out/main.js --input=/decrypted.json --output=/output_dir 83 | 84 | # When an encrypted pcap is given, both the secrets file and the tshark executable should be given 85 | node out/main.js --input=/encrypted.pcap --secrets=/secrets.keys --output=/output_dir --tshark=/path/to/tshark 86 | ``` -------------------------------------------------------------------------------- /captures/Local/log-client.txt: -------------------------------------------------------------------------------- 1 | quic@quic-dev:~$ ./ngtcp2-draft-15/examples/client 127.0.0.1 4433 -s 2 | initial_secret=c1fca1ef4cb463592ddc8f0c8d85969e4d93452a98b9e2db02412bf58082eea2 3 | client_in_secret=5db667264614c3b9bd4628c5ed3852af7eb49f8f6815e12c141fa3657e62758f 4 | + client_pp_key=7b53a82024dcb1cbb29b358010728500 5 | + client_pp_iv=ec730e97ba432d5281ee6165 6 | + client_pp_pn=e9f7248015046d3245f081cf33fb30cb 7 | server_in_secret=e6642e04a1e8f39e6aead43b8c093a5eb586adb8e5d2566996ebb042b40af321 8 | + server_pp_key=fbbfb2b2a92271767a49bc2c175378ab 9 | + server_pp_iv=c8718fc87edd9484eb4b9890 10 | + server_pp_pn=7b8b7353475cb4755df085f5ea6904e3 11 | msg_cb: write_p=1 version=772 content_type=22 len=299 12 | I00000000 0x0e8ac117cd49382ca96244849f52c15e42 pkt tx pkt 0 dcid=0x409fb12c4c35d5bd03bf5876588f326413c9 scid=0x0e8ac117cd49382ca96244849f52c15e42 type=Initial(0x7f) len=0 13 | I00000000 0x0e8ac117cd49382ca96244849f52c15e42 frm tx 0 Initial(0x7f) CRYPTO(0x18) offset=0 len=299 14 | I00000000 0x0e8ac117cd49382ca96244849f52c15e42 frm tx 0 Initial(0x7f) PADDING(0x00) len=888 15 | I00000000 0x0e8ac117cd49382ca96244849f52c15e42 rcv loss_detection_timer=1545039830614941952 last_hs_tx_pkt_ts=1545039830414941952 timeout=200 16 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 con recv packet len=1252 17 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt rx pkt 0 dcid=0x0e8ac117cd49382ca96244849f52c15e42 scid=0xac2f47839bc824a9b181381aef78c69d691c type=Initial(0x7f) len=149 18 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm rx 0 Initial(0x7f) CRYPTO(0x18) offset=0 len=123 19 | Ordered CRYPTO data 20 | 00000000 02 00 00 77 03 03 03 cb 67 82 58 a2 2a 42 b2 48 |...w....g.X.*B.H| 21 | 00000010 41 34 98 c7 5d f3 07 3d e1 f2 a6 22 2d 82 2b 49 |A4..]..=..."-.+I| 22 | 00000020 6a 34 37 7a fd 28 00 13 02 00 00 4f 00 2b 00 02 |j47z.(.....O.+..| 23 | 00000030 03 04 00 33 00 45 00 17 00 41 04 61 bc 7a 7e 81 |...3.E...A.a.z~.| 24 | 00000040 af 7d 27 d0 56 1f 4c ef b2 ad 4b ef 2b 4e f7 5e |.}'.V.L...K.+N.^| 25 | 00000050 dd 42 b5 63 04 f0 23 be ad b4 30 97 b4 5b be 8c |.B.c..#...0..[..| 26 | 00000060 65 0a 6e 9d 30 16 50 6a 18 03 be 47 40 01 8e 6f |e.n.0.Pj...G@..o| 27 | 00000070 a1 9c 38 0a cf 00 f8 86 99 19 e8 |..8........| 28 | 0000007b 29 | msg_cb: write_p=0 version=772 content_type=22 len=123 30 | server_handshake_traffic 31 | + secret=d6b6f994f99bd2bdead7f3cd56d171c410c6bef76b973badc2861c68efacaca990822555d4a3469c8b7a2950245c4ecc 32 | + key=745c58d2903d6760588a3df9b9de2991771d3dd20b81d4cecf3a1ab894eb8fa8 33 | + iv=22cd7efc9fc2f7adf556793c 34 | + pn=47e485692c29fc8c001fc04a6470801e9f09c3c1e1b9730c57aa034cc280250d 35 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm rx 0 Initial(0x7f) ACK(0x1a) largest_ack=0 ack_delay=0(0) ack_block_count=0 36 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm rx 0 Initial(0x7f) ACK(0x1a) block=[0..0] block_count=0 37 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 rcv latest_rtt=6 min_rtt=6 smoothed_rtt=6.423 rttvar=3.212 max_ack_delay=0 38 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 rcv packet 0 acked, slow start cwnd=13252 39 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 rcv loss_detection_timer=1545039830427788544 last_hs_tx_pkt_ts=1545039830414941952 timeout=12 40 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt read packet 193 left 1059 41 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt rx pkt 0 dcid=0x0e8ac117cd49382ca96244849f52c15e42 scid=0xac2f47839bc824a9b181381aef78c69d691c type=Handshake(0x7d) len=1016 42 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm rx 0 Handshake(0x7d) CRYPTO(0x18) offset=0 len=995 43 | Ordered CRYPTO data 44 | 00000000 08 00 00 5d 00 5b ff a5 00 4b ff 00 00 0f 04 ff |...].[...K......| 45 | 00000010 00 00 0f 00 40 00 06 00 10 4c 53 a8 e7 73 eb 1a |....@....LS..s..| 46 | 00000020 e8 50 1b 13 d1 f3 8a c3 2d 00 00 00 04 00 04 00 |.P......-.......| 47 | 00000030 00 00 0a 00 04 00 04 00 00 00 0b 00 04 00 04 00 |................| 48 | 00000040 00 00 01 00 04 00 10 00 00 00 02 00 02 00 64 00 |..............d.| 49 | 00000050 03 00 02 00 1e 00 10 00 08 00 06 05 68 71 2d 31 |............hq-1| 50 | 00000060 35 0b 00 03 6d 00 00 03 69 00 03 64 30 82 03 60 |5...m...i..d0..`| 51 | 00000070 30 82 02 48 a0 03 02 01 02 02 09 00 e7 52 39 c2 |0..H.........R9.| 52 | 00000080 ce de 37 47 30 0d 06 09 2a 86 48 86 f7 0d 01 01 |..7G0...*.H.....| 53 | 00000090 0b 05 00 30 45 31 0b 30 09 06 03 55 04 06 13 02 |...0E1.0...U....| 54 | 000000a0 41 55 31 13 30 11 06 03 55 04 08 0c 0a 53 6f 6d |AU1.0...U....Som| 55 | 000000b0 65 2d 53 74 61 74 65 31 21 30 1f 06 03 55 04 0a |e-State1!0...U..| 56 | 000000c0 0c 18 49 6e 74 65 72 6e 65 74 20 57 69 64 67 69 |..Internet Widgi| 57 | 000000d0 74 73 20 50 74 79 20 4c 74 64 30 1e 17 0d 31 38 |ts Pty Ltd0...18| 58 | 000000e0 31 32 31 32 31 35 35 39 30 33 5a 17 0d 31 39 31 |1212155903Z..191| 59 | 000000f0 32 31 32 31 35 35 39 30 33 5a 30 45 31 0b 30 09 |212155903Z0E1.0.| 60 | 00000100 06 03 55 04 06 13 02 41 55 31 13 30 11 06 03 55 |..U....AU1.0...U| 61 | 00000110 04 08 0c 0a 53 6f 6d 65 2d 53 74 61 74 65 31 21 |....Some-State1!| 62 | 00000120 30 1f 06 03 55 04 0a 0c 18 49 6e 74 65 72 6e 65 |0...U....Interne| 63 | 00000130 74 20 57 69 64 67 69 74 73 20 50 74 79 20 4c 74 |t Widgits Pty Lt| 64 | 00000140 64 30 82 01 22 30 0d 06 09 2a 86 48 86 f7 0d 01 |d0.."0...*.H....| 65 | 00000150 01 01 05 00 03 82 01 0f 00 30 82 01 0a 02 82 01 |.........0......| 66 | 00000160 01 00 a9 20 e4 e1 90 c5 5b 31 6d e1 ba 8a 56 27 |... ....[1m...V'| 67 | 00000170 d2 5c 8d a3 69 86 15 fc a3 1e 60 a8 4a ea 47 5c |.\..i.....`.J.G\| 68 | 00000180 90 25 43 06 a5 0a 1f 36 7c 40 1a 5e f8 e8 72 6a |.%C....6|@.^..rj| 69 | 00000190 ed a6 38 28 eb f6 00 4f 04 c0 54 16 ea e5 fb f8 |..8(...O..T.....| 70 | 000001a0 06 a6 b3 e7 3e 35 7b e7 a9 d2 fa 4d f2 d8 03 c2 |....>5{....M....| 71 | 000001b0 8c c1 cb d7 e7 6d a1 3d 4c 59 d1 4d c1 e2 b0 0f |.....m.=LY.M....| 72 | 000001c0 53 d4 8e eb bd 81 26 01 8c 54 a7 f0 4e c4 90 e9 |S.....&..T..N...| 73 | 000001d0 9f 31 a5 63 59 9d b0 19 71 0e 94 dd 0b 2a 45 dc |.1.cY...q....*E.| 74 | 000001e0 19 43 90 26 b3 ca 29 53 d8 a9 15 e8 0b e4 a2 13 |.C.&..)S........| 75 | 000001f0 04 a6 f4 47 69 17 db 7e 47 0e 53 2f 12 94 24 ea |...Gi..~G.S/..$.| 76 | 00000200 d9 81 78 6e cf bd 58 58 6c eb 93 da ba 10 45 fd |..xn..XXl.....E.| 77 | 00000210 dc b8 67 fc 84 50 a7 db 32 05 56 4f 71 af 44 59 |..g..P..2.VOq.DY| 78 | 00000220 3a 71 1d de 69 75 a4 c4 65 3e 77 20 50 cb df 6d |:q..iu..e>w P..m| 79 | 00000230 80 36 d3 75 a9 32 01 d7 76 0b 1e 8f c0 5e e6 f7 |.6.u.2..v....^..| 80 | 00000240 af b9 78 34 1e f3 00 a0 aa 6f e6 ce 6a 8f 34 58 |..x4.....o..j.4X| 81 | 00000250 29 ac 60 28 4a fd 05 cd 4e 3d bc 5c ce a3 d7 43 |).`(J...N=.\...C| 82 | 00000260 04 99 02 03 01 00 01 a3 53 30 51 30 1d 06 03 55 |........S0Q0...U| 83 | 00000270 1d 0e 04 16 04 14 54 fd 92 9e 9a 22 10 ce af f6 |......T...."....| 84 | 00000280 80 9d 3e e8 e7 91 5c e8 df 3e 30 1f 06 03 55 1d |..>...\..>0...U.| 85 | 00000290 23 04 18 30 16 80 14 54 fd 92 9e 9a 22 10 ce af |#..0...T...."...| 86 | 000002a0 f6 80 9d 3e e8 e7 91 5c e8 df 3e 30 0f 06 03 55 |...>...\..>0...U| 87 | 000002b0 1d 13 01 01 ff 04 05 30 03 01 01 ff 30 0d 06 09 |.......0....0...| 88 | 000002c0 2a 86 48 86 f7 0d 01 01 0b 05 00 03 82 01 01 00 |*.H.............| 89 | 000002d0 8d e1 74 49 61 66 b5 30 c3 00 4a 7b e3 aa 73 6a |..tIaf.0..J{..sj| 90 | 000002e0 5d 3c c2 48 c5 5c 69 1c dc 55 6c 16 55 f4 96 5e |]<.H.\i..Ul.U..^| 91 | 000002f0 ee 1c dc f1 b1 21 07 69 71 2d dd 44 96 67 13 aa |.....!.iq-.D.g..| 92 | 00000300 cc 7a 32 6f f9 44 77 a7 bd 32 c5 c4 30 06 1d 15 |.z2o.Dw..2..0...| 93 | 00000310 60 df 66 22 25 ff cf b4 66 02 ad 70 c2 6e c9 63 |`.f"%...f..p.n.c| 94 | 00000320 e8 b7 c5 ed 3e 2e d8 9b a4 02 73 8f 46 34 33 74 |....>.....s.F43t| 95 | 00000330 30 94 0e 45 a6 75 63 8c b1 88 f0 bd c9 5c 91 79 |0..E.uc......\.y| 96 | 00000340 ac 29 ec 89 c3 3a 48 f4 53 58 6f 10 f2 fd f9 c4 |.)...:H.SXo.....| 97 | 00000350 e0 5e 87 a4 c8 ac fb 48 54 eb 6f 9d 6f f7 e9 cf |.^.....HT.o.o...| 98 | 00000360 4c e4 97 75 65 14 0e 64 f7 01 a5 6b 69 b6 24 ea |L..ue..d...ki.$.| 99 | 00000370 c8 88 5f 1b c5 de fb 1f a2 5b 2a 05 76 ea a9 2a |.._......[*.v..*| 100 | 00000380 65 9b 4f 05 0d bd b2 c3 d4 aa 34 33 a9 bf 86 5b |e.O.......43...[| 101 | 00000390 2e 29 4a e7 08 43 1a 53 1d c0 8a 0d 5e 45 16 0c |.)J..C.S....^E..| 102 | 000003a0 f7 76 8d 81 b0 0d 15 ab 87 c8 7f 37 ec d0 47 02 |.v.........7..G.| 103 | 000003b0 7c 2d c1 5c ce fe e0 34 0d 4e 3d f8 c9 bd 75 be ||-.\...4.N=...u.| 104 | 000003c0 33 82 c0 fc f6 90 46 b4 bb f9 d9 28 13 cd 6d d7 |3.....F....(..m.| 105 | 000003d0 00 00 0f 00 01 04 08 04 01 00 3c 81 c3 ba 39 73 |..........<...9s| 106 | 000003e0 71 a6 b6 |q..| 107 | 000003e3 108 | msg_cb: write_p=0 version=772 content_type=22 len=97 109 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters negotiated_version=0xff00000f 110 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters supported_version[0]=0xff00000f 111 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters stateless_reset_token=0x4c53a8e773eb1ae8501b13d1f38ac32d 112 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_stream_data_bidi_local=262144 113 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_stream_data_bidi_remote=262144 114 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_stream_data_uni=262144 115 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_data=1048576 116 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_bidi_streams=100 117 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters initial_max_uni_streams=0 118 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters idle_timeout=30 119 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters max_packet_size=65527 120 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters ack_delay_exponent=3 121 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 cry remote transport_parameters max_ack_delay=25 122 | msg_cb: write_p=0 version=772 content_type=22 len=881 123 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt read packet 1059 left 0 124 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt tx pkt 1 dcid=0xac2f47839bc824a9b181381aef78c69d691c scid=0x0e8ac117cd49382ca96244849f52c15e42 type=Initial(0x7f) len=0 125 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm tx 1 Initial(0x7f) ACK(0x1a) largest_ack=0 ack_delay=0(0) ack_block_count=0 126 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm tx 1 Initial(0x7f) ACK(0x1a) block=[0..0] block_count=0 127 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm tx 1 Initial(0x7f) PADDING(0x00) len=1186 128 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 rcv loss_detection_timer=1545039830434211840 last_hs_tx_pkt_ts=1545039830421365248 timeout=12 129 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 con recv packet len=364 130 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 pkt rx pkt 1 dcid=0x0e8ac117cd49382ca96244849f52c15e42 scid=0xac2f47839bc824a9b181381aef78c69d691c type=Handshake(0x7d) len=321 131 | I00000006 0x0e8ac117cd49382ca96244849f52c15e42 frm rx 1 Handshake(0x7d) CRYPTO(0x18) offset=995 len=299 132 | Ordered CRYPTO data 133 | 00000000 ee 25 d8 f2 23 83 f1 02 c4 36 39 d1 0a 4e 34 72 |.%..#....69..N4r| 134 | 00000010 4e 37 86 88 22 9a bb 6a 85 b8 75 45 ee 4b 10 53 |N7.."..j..uE.K.S| 135 | 00000020 6b c5 bc ea 6c 62 e5 c2 16 16 74 ab 3c 29 81 a4 |k...lb....t.<)..| 136 | 00000030 d3 c2 d2 eb 83 fc eb d3 f2 28 dc a0 6b 7f e3 12 |.........(..k...| 137 | 00000040 7d bd 7f a3 00 4c 1e 8c 10 7f d1 9d fc d4 16 f3 |}....L..........| 138 | 00000050 56 e5 b1 12 83 6b c6 87 a2 53 ae 67 9d 5e c3 dd |V....k...S.g.^..| 139 | 00000060 e5 79 8e a3 e0 a7 36 61 2e 0e db e9 ec 4d 4d 7f |.y....6a.....MM.| 140 | 00000070 c6 65 97 58 96 b1 4b 2e 17 50 e0 3b 92 82 68 59 |.e.X..K..P.;..hY| 141 | 00000080 b6 5d 25 1e bc 87 af 97 f0 b8 90 04 4c 13 60 6f |.]%.........L.`o| 142 | 00000090 8e 89 70 bc 73 f7 3b 1d f7 c7 e2 b1 1f 66 1b f6 |..p.s.;......f..| 143 | 000000a0 8a 5b 68 b7 13 86 95 f5 9d 4a 09 99 5c 7d 74 d2 |.[h......J..\}t.| 144 | 000000b0 54 04 7a bd 1d 8a 02 bb 99 c2 e5 4a 42 0d 04 aa |T.z........JB...| 145 | 000000c0 f0 40 4c 1b 70 df 19 2a 3d 7f d5 33 b9 62 d3 b2 |.@L.p..*=..3.b..| 146 | 000000d0 a8 fa 92 a7 92 e2 1d fa f6 15 f8 c7 ec d6 f5 33 |...............3| 147 | 000000e0 21 23 31 fd 49 80 db 3c ed 6f 0b be 3d 59 17 6f |!#1.I..<.o..=Y.o| 148 | 000000f0 e9 3c 52 ef c1 19 f7 14 00 00 30 62 9c 8b 37 b6 |.=0.0.16", 8 | "minimist": "^1.2.0" 9 | }, 10 | "devDependencies": { 11 | "@types/minimist": "^1.2.0", 12 | "@types/node": "^10.14.22", 13 | "typescript": "^3.6.4" 14 | }, 15 | "scripts": { 16 | "test": "echo \"Error: no test specified\" && exit 1" 17 | }, 18 | "author": "", 19 | "license": "ISC" 20 | } 21 | -------------------------------------------------------------------------------- /pcap_to_qlog_spec.txt: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "quic_version": "0xff00000b", 4 | "qlog_version": "0.1", 5 | "vantagepoint": "NETWORK", 6 | "connectionid": "9e:f8:43:3b:05:5c:c3:6e:2d:81:31:31:9b:55:32:8b:16", 7 | "starttime": 1524571067.892647585, 8 | "fields": 9 | ["time","category", "type", "trigger", "data"], 10 | "events": [ 11 | [0, "CONNECTIVITY", "NEW_CONNECTION", "LINE", {"ip_version": 4, "srcip": "127.0.0.1", "dstip": "127.0.0.1", "srcport": 12456, "dstport": 4433}], 12 | 13 | [0, "SECURITY", "KEY_UPDATE", "KEYLOG", {"CLIENT_EARLY_ENCRYPT": abcdefabcdef, "SERVER_HANDSHAKE_ENCRYPT": abcdefabcdef, ...], 14 | 15 | [15, "TRANSPORT", "PACKET_RX", "LINE" { 16 | "raw_encrypted": "9c3f19b02d98d8ef1b564ae6a5d4b9a59550627c31c226f632a2aa3c117c7243f00f2f7b534d6ff35742e429b2f9c4bc66319a89eb6dbdc9cc84bbb40560ccca6d3b90...", 17 | 18 | "header": { 19 | "form": "long", 20 | "type": "initial", 21 | "version": "0xff00000b", 22 | "scil": 14, 23 | "dcil": 15, 24 | "dcid": "e3b392f86ffcab1bb28c97157d07692f169f", 25 | "scid": "9ef8433b055cc36e2d8131319b55328b16", 26 | "payload_length": 1280, 27 | "packet_number": 0 28 | }, 29 | "frames" : [ 30 | { 31 | "type": "CRYPTO", 32 | "length": 360, 33 | #... other fields directly copied over without extraneous prefixes 34 | "raw": "160301011b010001170303c8b38100267e126813531310afab6da3cf092da15c67dce07323591830b7c7e200000813011302130300ff010000e60" 35 | } 36 | ] 37 | }], 38 | 39 | [16, "TRANSPORT", "NEW_TRANSPORT_PARAMETERS", "PACKET_RX", { 40 | "parameters" 41 | 42 | }], 43 | 44 | [30, "TRANSPORT", "PACKET_RX", "LINE" { 45 | "raw_encrypted": "9c3f19b02d98d8ef1b564ae6a5d4b9a59550627c31c226f632a2aa3c117c7243f00f2f7b534d6ff35742e429b2f9c4bc66319a89eb6dbdc9cc84bbb40560ccca6d3b90...", 46 | 47 | "header": { 48 | "form": "short", 49 | "dcid": "e3b392f86ffcab1bb28c97157d07692f169f", 50 | "payload_length": 1280, 51 | "packet_number": 12 52 | }, 53 | "frames" : [ 54 | { 55 | "type": "CRYPTO", 56 | "length": 400, 57 | #... other fields directly copied over without extraneous prefixes (e.g., quic.frame_type.crypto.crypto_data becomes data) 58 | "raw": "193870301011b010001170303c8b38100267e126813531310afab6da3cf092da15c67dce07323591830b7c7e200000813011302130300ff010000e60" 59 | } 60 | ] 61 | }] 62 | ]} 63 | 64 | 65 | 66 | # header 67 | "quic_version": `SELECT packet WHERE payload CONTAINS client initial SORT BY time DESC LIMIT 1`.header.version # DESC because version negotiation 68 | "vantagepoint" "CLIENT" | "SERVER" | "NETWORK", # set manually, unable to determine for pcaps 69 | "connectionid": `SELECT packet WHERE payload CONTAINS client initial SORT BY time ASC LIMIT 1`.scid, #TODO: decide if we should log client scid, because it can change when sending multiple initials and depends on which one server chooses. Maybe just go for server scid? but can't that also change? look up! 70 | "starttime": `frame.time_epoch`, 71 | 72 | 73 | # is always the first event 74 | cat: CONNECTIVITY 75 | evt: NEW_CONNECTION 76 | trigger: LINE 77 | data {ipversion, src, dst, port, port} 78 | # fill stuff from packet[0].ip data. For now: assume this doesn't change 79 | 80 | 81 | # TLS keys needed by wireshark 82 | # https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob;f=epan/dissectors/packet-tls-utils.c;h=6e274ddda41358f2af49b3cca301c019a0a70e88;hb=HEAD 83 | # what is logged in ngtcp2 84 | # https://github.com/ngtcp2/ngtcp2/blob/draft-15/examples/keylog.cc 85 | 86 | # voeg decryption keys toe vanuit de log file: qlog heeft all-in-one 87 | cat: TLS 88 | evt: KEY_UPDATE 89 | trigger: KEYLOG 90 | data { 91 | type: CLIENT_EARLY_ENCRYPT | 92 | SERVER_HANDSHAKE_ENCRYPT | SERVER_HANDSHAKE_DECRYPT | CLIENT_HANDSHAKE_ENCRYPT | CLIENT_HANDSHAKE_DECRYPT | 93 | SERVER_PROTECTED_ENCRYPT | SERVER_PROTECTED_DECRYPT | CLIENT_PROTECTED_ENCRYPT | CLIENT_PROTECTED_DECRYPT 94 | key: "value" 95 | } 96 | 97 | time: value (frame.time_relative) #cast to int in ms accuracy // see also https://www.wireshark.org/docs/wsug_html_chunked/ChWorkTimeFormatsSection.html 98 | cat: "TRANSPORT" 99 | evt: "PACKET_RX" 100 | trigger: "LINE" 101 | data: 102 | { 103 | raw_encrypted : "hex of full encrypted packet" (quic.initial_payload) (only if logging level >= "raw") 104 | header: { 105 | form: "long" | "short", (quic.header_form) 106 | type: "initial" | "retry" | "handshake" | "0RTT" (quic.long.packet_type) (not present if form == "short") 107 | version: "version" (not present if form == "short") 108 | scil: value, (not present if form == "short") 109 | dcil: value, (not present if form == "short") 110 | scid: value, (not present if form == "short") 111 | dcid: value, 112 | payload_length: value, (quic.payload_length if form == "long", length(quic.protected_payload) if form == "short") 113 | packet_number: "value" (quic.packet_number_full) 114 | }, 115 | frames : [ 116 | { 117 | type: "CRYPTO" | "STREAM" | ... (see spec for all possible values and how they map to integers in pcap) 118 | length: (probably need to calculate this yourself from a payload field, is not included in the wire image itself) 119 | other fields can just be copied over in full here for now 120 | raw: "hex of full frame" (only if logging level >= "raw") 121 | } 122 | ] 123 | } 124 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /src/flow/downloader.ts: -------------------------------------------------------------------------------- 1 | import {exec, execSync} from "child_process"; 2 | import * as path from "path"; 3 | import { getFileExtension } from "../util/FileUtil"; 4 | const URL = require("url").URL; 5 | 6 | export class Downloader{ 7 | 8 | public static async DownloadIfRemote(inputPath:string, outputDirectory:string):Promise { 9 | 10 | if( inputPath.indexOf("http") >= 0 || inputPath.indexOf("https") >= 0 ){ 11 | //console.log("File is a URL, downloading...", inputPath, outputDirectory ); 12 | 13 | let randomFilename = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); 14 | randomFilename += getFileExtension( inputPath ); 15 | let outputPath:string = outputDirectory + path.sep + randomFilename; 16 | 17 | return Downloader.DownloadAsync( inputPath, outputPath ); 18 | } 19 | else 20 | return inputPath; 21 | 22 | } 23 | 24 | public static ValidateRemoteURL(url:string):string { 25 | 26 | if( url === undefined || url === "" ){ 27 | throw new Error("url was empty"); 28 | } 29 | 30 | let validURL = new URL(url); // throws error if not valid remote URL 31 | 32 | // URL ctor validator apparently doesn't work 100%, so perform some additional regex magics 33 | // https://github.com/xxorax/node-shell-escape/blob/master/shell-escape.js 34 | // https://github.com/ogt/valid-url/blob/master/index.js 35 | 36 | // check invalid characters 37 | if (/[^a-z0-9\:\/\?\#\|\[\]\@\!\$\&\'\(\)\*\+\,\;\=\.\-\_\~\%]/i.test(url)){ 38 | throw new Error("invalid character present " + url); 39 | } 40 | 41 | // check for hex escapes that aren't complete 42 | if (/%[^0-9a-f]/i.test(url)) { 43 | throw new Error("invalid character present " + url); 44 | } 45 | if (/%[0-9a-f](:?[^0-9a-f]|$)/i.test(url)) { 46 | throw new Error("invalid character present " + url); 47 | } 48 | 49 | // https://stackoverflow.com/questions/49512370/sanitize-user-input-for-child-process-exec-command 50 | url = url.replace(/(["\s'$`\\])/g,'\\$1'); 51 | 52 | return url; 53 | } 54 | 55 | public static DownloadAsync(inputPath:string, outputPath:string):Promise{ 56 | 57 | let output:Promise = new Promise( (resolver, rejecter) => { 58 | 59 | try { 60 | inputPath = Downloader.ValidateRemoteURL( inputPath ); 61 | } 62 | catch(e){ 63 | rejecter(e.toString()); 64 | } 65 | 66 | let timeoutMin = 1; 67 | let wgetLocation:string = "wget"; 68 | 69 | let timeoutHappened:boolean = false; 70 | let timer = setTimeout( function(){ 71 | timeoutHappened = true; 72 | //console.log("|||||||||||||||||||||||||||||||||||||||||||||"); 73 | //console.log("DownloadAsync : timeout happened, downloading next URL", inputPath ); 74 | //console.log("|||||||||||||||||||||||||||||||||||||||||||||"); 75 | 76 | rejecter("Timeout, URL could not be reached within " + timeoutMin + " minutes : " + inputPath); 77 | 78 | }, timeoutMin * 60 * 1000 ); // if not done after the expected timeout, we will assume the wget call to hang and proceed 79 | 80 | exec( wgetLocation + ` --timeout=${timeoutMin} --tries=2 --retry-connrefused -O ${outputPath} ${inputPath}`, function(error, {}, stderr){ 81 | if( timeoutHappened ) 82 | return; 83 | 84 | clearTimeout(timer); 85 | 86 | if( error ){ 87 | //console.log("-----------------------------------------"); 88 | //console.log("DownloadAsync : ERROR : ", error, stderr, inputPath, outputPath); 89 | 90 | // on error, wget still writes an empty file (or half-downloaded file), which we want to get rid of 91 | let result = execSync( `rm -f ${outputPath}` ); // we assume this will work. If not, nothing we can do about it either way... 92 | //console.log(`Removing file ${outputPath} : ${result}`); 93 | //console.log("-----------------------------------------"); 94 | 95 | rejecter( "DownloadAsync : error : " + inputPath + " : " + error ); 96 | } 97 | else{ 98 | resolver( outputPath ); 99 | } 100 | }); 101 | }); 102 | 103 | return output; 104 | } 105 | } -------------------------------------------------------------------------------- /src/flow/jsontoqlog.ts: -------------------------------------------------------------------------------- 1 | 2 | import * as fs from "fs"; 3 | import {promisify} from "util"; 4 | import {ParserPCAP} from "../parsers/ParserPCAP"; 5 | import * as qlog from "@quictools/qlog-schema"; 6 | 7 | 8 | const readFileAsync = promisify(fs.readFile); 9 | const writeFileAsync = promisify(fs.writeFile); 10 | 11 | export class JSONToQLog{ 12 | 13 | public static async TransformToQLog(jsonPath:string, outputDirectory:string, originalFile: string, logRawPayloads: boolean, secretsPath?:string, logUnknownFramesFields: boolean = false):Promise { 14 | 15 | // assumptions: 16 | // - jsonPath and secretsPath are LOCAL (if it was a URL, it has to be pre-downloaded) 17 | // - the .json file is the DECRYPTED output of tshark when run on a .pcap or .pcapng 18 | // - outputDirectory exists 19 | 20 | let fileContents:Buffer = await readFileAsync(jsonPath); 21 | let jsonContents:any = JSON.parse( fileContents.toString() ); 22 | 23 | let secretsContents:any = undefined; 24 | if( secretsPath ){ 25 | let secretsFileContents:Buffer = await readFileAsync(secretsPath); 26 | // TODO :parse the secrets 27 | secretsContents = secretsFileContents.toString(); 28 | } 29 | 30 | 31 | // TODO: properly deal with different versions of QUIC and address the correct parser 32 | // see how we did this in @quictools/qlog-schema and replicate something similar here 33 | let qlog:qlog.IQLog = ParserPCAP.Parse( jsonContents, originalFile, logRawPayloads, secretsContents, logUnknownFramesFields ); 34 | 35 | // we could write this to file directly now 36 | // BUT we want to aggregate possible different IQLogs together in 1 combined/grouped IQLog before writing the final output 37 | // so we return the QLog instead of writing, so the caller can decide what to do 38 | // (otherwhise we would write here and have to re-read again later, which isn't very efficient) 39 | 40 | return qlog; 41 | } 42 | } -------------------------------------------------------------------------------- /src/flow/pcaptojson.ts: -------------------------------------------------------------------------------- 1 | import {exec, execSync} from "child_process"; 2 | import * as path from "path"; 3 | 4 | export class PCAPToJSON{ 5 | 6 | public static async TransformToJSON(pcapPath:string, outputDirectory:string, tsharkLocation: string, secretsPath?:string ):Promise { 7 | 8 | // assumptions: 9 | // - pcapPath and secretsPath are LOCAL (if it was a URL, it has to be pre-downloaded) 10 | // - outputDirectory exists 11 | 12 | let output:Promise = new Promise( (resolver, rejecter) => { 13 | 14 | let timeoutMin = 1; 15 | 16 | let randomFilename = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); 17 | let outputPath = outputDirectory + path.sep + randomFilename + ".json"; 18 | 19 | 20 | // if not done after the expected timeout, we will assume the tshark call to hang and proceed 21 | let timeoutHappened:boolean = false; 22 | let timer = setTimeout( function(){ 23 | timeoutHappened = true; 24 | 25 | rejecter("Timeout, tshark didn't complete within " + timeoutMin + " minutes"); 26 | 27 | }, timeoutMin * 60 * 1000 ); 28 | 29 | // /wireshark/run/tshark --no-duplicate-keys -r pcap.pcap -T json -o tls.keylog_file:/srv/secrets.keys > output.json 30 | 31 | //console.log("About to exec tshark"); 32 | let option = ""; 33 | if( secretsPath ) 34 | option = `-o tls.keylog_file:${secretsPath}`; 35 | 36 | //exec( `echo ${pcapPath} > ${outputPath}`, function(error, {}, stderr){ 37 | exec( tsharkLocation + ` --no-duplicate-keys -r ${pcapPath} -T json ${option} > ${outputPath}`, function(error, {}, stderr){ 38 | if( timeoutHappened ) 39 | return; 40 | 41 | clearTimeout(timer); 42 | 43 | //console.log("Execed tshark"); 44 | 45 | if( error ){ 46 | //console.log("-----------------------------------------"); 47 | //console.log("TransformToJSON : ERROR : ", error, stderr, pcapPath, outputPath); 48 | //console.log("-----------------------------------------"); 49 | 50 | rejecter( "tshark:TransformToJSON : error : " + error ); 51 | } 52 | else{ 53 | resolver( outputPath ); 54 | } 55 | }); 56 | }); 57 | 58 | return output; 59 | } 60 | 61 | } -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import * as path from "path"; 3 | import {promisify} from "util"; 4 | 5 | const readFileAsync = promisify(fs.readFile); 6 | const writeFileAsync = promisify(fs.writeFile); 7 | 8 | import {Downloader} from "./flow/downloader"; 9 | import { mkDirByPathSync, createHash, fileIsJSON, fileIsPCAP, fileIsSECRETS, fileIsQLOG } from "./util/FileUtil"; 10 | import {PCAPToJSON} from "./flow/pcaptojson"; 11 | import {JSONToQLog} from "./flow/jsontoqlog"; 12 | import * as qlog from "@quictools/qlog-schema"; 13 | import { VantagePointType, ITrace, ITraceError } from "@quictools/qlog-schema"; 14 | 15 | // Parse CLI arguments 16 | let args = require('minimist')(process.argv.slice(2)); 17 | 18 | // CLI arguments 19 | // we expect the input file to be EITHER a JSON file of the format: 20 | /* 21 | { 22 | "description": "top-level description", 23 | "paths": [ 24 | { "capture": "https://...", "secrets": "https://...", "description" : "per-file desc" } 25 | ] 26 | } 27 | */ 28 | // then use the --list option 29 | // OR a single file-path (or URL), with or without the secrets_file set 30 | // then use the --input option 31 | // e.g., 32 | // node main.js --list=/allinone.json --output=/srv/qvis-cache 33 | // OR 34 | // node main.js --input=/decrypted.json --output=/srv/qvis-cache 35 | // OR 36 | // node main.js --input=/encrypted.pcap --secrets=/secrets.keys --output=/srv/qvis-cache 37 | // NOTE: this tool can also be used to merge together multiple (partial) qlog files 38 | // for this, just pass the files with the --list option 39 | // This tool uses tshark for the conversion of pcap to json and should therefore be given a path to your local install of the program 40 | // This can be done using the --tshark flag (or -t) 41 | // The default tshark location used is /wireshark/run/tshark which will generally not be correct if you are running this outside of a docker. It is thus recommended to pass the path to your local install. 42 | let input_list: string = args.l || args.list; 43 | let input_file: string = args.i || args.input; 44 | let secrets_file: string = args.s || args.secrets; 45 | let output_directory: string = args.o || args.output || "/srv/qvis-cache"; // output will be placed in args.o/cache/generatedfilename.json and temp storage is args.o/inputs/ 46 | let output_path: string = args.p || args.outputpath; // output will be placed in args.p and temp storage is args.o/inputs/ 47 | let tsharkLocation: string = args.t || args.tshark || "/wireshark/run/tshark"; // Path to the TShark executable 48 | let logRawPayloads: boolean = args.r || args.raw || false; // If set to false, raw decrypted payloads will not be logged. This is default behaviour as payloads have a huge impact on log size. 49 | let logUnkFramesFields: boolean = args.u || args.logunknownframesfields || false; // if set to true, adds to the qlog file the fields of the unknown frames present in the pcap parsed by TShark 50 | 51 | if( !input_file && !input_list ){ 52 | console.error("No input file or list of files specified, use --input or --list"); 53 | process.exit(1); 54 | } 55 | 56 | if( !output_directory ){ 57 | console.error("Please specify an output_directory, even if you have specified an output_path (needed for temp file storage)"); 58 | process.exit(1); 59 | } 60 | 61 | async function Flow() { 62 | 63 | let inputIsList:boolean = input_list !== undefined; 64 | 65 | // each session gets their own temporary input directory so we can easily remove it from disk after everything is done 66 | let tempDirectory = path.resolve( output_directory + path.sep + "inputs" ); 67 | tempDirectory += path.sep + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); 68 | mkDirByPathSync( tempDirectory ); 69 | 70 | // 1. make single input format 71 | // we have 3 options: 72 | // a. it's a custom json file with many different input files (either .qlog, .json or .pcap/pcapng) and possibly keys 73 | // b. it's just a single decrypted json file (tshark output) 74 | // c. it's just a single pcap or pcapng file and possibly a single key file 75 | // -> we rework b. and c. to the format of a. for easier manipulation later 76 | 77 | interface ICapture { 78 | qlog?:qlog.IQLog; 79 | capture_original:string; // for proper debugging 80 | capture:string; 81 | secrets_original?:string; // for proper debugging 82 | secrets?:string; 83 | error?:string; 84 | description?:string; 85 | } 86 | 87 | let inputList:Array = []; 88 | let inputListDescription:string = ""; 89 | let inputListRawString:string = ""; 90 | 91 | if( !inputIsList ){ 92 | if( fileIsJSON(input_file) ){ // tshark json file 93 | inputList.push( {capture : input_file, capture_original: input_file } ); 94 | } 95 | else{ 96 | if( secrets_file !== undefined && secrets_file !== "" ) 97 | inputList.push( {capture : input_file, capture_original: input_file, secrets: secrets_file, secrets_original: secrets_file } ); 98 | else 99 | inputList.push( {capture : input_file, capture_original: input_file } ); 100 | } 101 | } 102 | else{ 103 | try{ 104 | let listLocalFilePath:string = await Downloader.DownloadIfRemote( input_list, tempDirectory ); 105 | inputListRawString = fs.readFileSync( listLocalFilePath ).toString(); 106 | let inputListJSON:any = JSON.parse( inputListRawString ); 107 | 108 | inputListDescription = inputListJSON.description ? inputListJSON.description : input_file; // default description is the file path itself 109 | inputList = inputListJSON.paths; 110 | for( let capt of inputList ){ 111 | capt.capture_original = capt.capture; 112 | capt.secrets_original = capt.secrets_original; 113 | } 114 | } 115 | catch(e){ 116 | console.error("Error downloading list file " + e.toString()); 117 | process.exit(2); 118 | } 119 | } 120 | 121 | // TODO: first check cache to see if we haven't already done this process for this file or file-list before 122 | let calculateStableHash = function():string { 123 | let output:string = ""; 124 | if( inputIsList ){ 125 | // we're not just using the filename, because this is more flexible, especially if the contents are made on-the-fly 126 | output = createHash( inputListRawString ); 127 | } 128 | else{ 129 | output = createHash( input_file + secrets_file ); 130 | } 131 | 132 | return output; 133 | } 134 | 135 | 136 | // 2. now we have a nice list of files (possibly with secrets) that we would like to download 137 | // These files can either be local or remote, so we should always download them first if needed 138 | 139 | let download = async function(capt:ICapture, outputDirectory:string):Promise { 140 | //let output:ICapture = { capture: "", capture_original: capt.capture_original, secrets_original: capt.secrets_original }; 141 | 142 | if( capt.error ) 143 | return capt; 144 | 145 | // we can't just throw errors, because that would fail the full Promise.all, 146 | // while we just want to skip the files that don't work and show an error message 147 | try{ 148 | capt.capture = await Downloader.DownloadIfRemote( capt.capture, tempDirectory ); 149 | } 150 | catch(e){ 151 | capt.error = e; 152 | } 153 | if( capt.secrets ){ 154 | try{ 155 | capt.secrets = await Downloader.DownloadIfRemote( capt.secrets, tempDirectory ); 156 | } 157 | catch(e){ 158 | capt.error = e; 159 | } 160 | } 161 | 162 | return capt; 163 | }; 164 | 165 | // note: we could do download + tshark calls in 1 long async function, but for clarity we keep them separate 166 | // performance should be relatively ok, unless there is a single file that is MUCH bigger than the rest of course 167 | let downloadPromises = []; 168 | for( let capture of inputList ){ 169 | downloadPromises.push( download(capture, tempDirectory) ); 170 | } 171 | 172 | let downloadedFiles = await Promise.all( downloadPromises ); 173 | 174 | 175 | // 3. now we have the files all local and the like 176 | // now we need to transform any pcap or pcapng files to the tshark json format 177 | let tshark = async function(capt:ICapture, outputDirectory:string):Promise { 178 | if( capt.error ) 179 | return capt; 180 | 181 | if( fileIsJSON(capt.capture) || fileIsQLOG(capt.capture) ) 182 | return capt; 183 | 184 | // TODO: we explicitly do NOT check for .pcap or .pcapng here to allow for most flexibility 185 | // this may come back to bite us in the *ss later 186 | 187 | try{ 188 | capt.capture = await PCAPToJSON.TransformToJSON( capt.capture, tempDirectory, tsharkLocation, capt.secrets ); 189 | } 190 | catch(e){ 191 | capt.error = e; 192 | } 193 | 194 | return capt; 195 | }; 196 | 197 | let tsharkPromises = []; 198 | for( let capture of downloadedFiles ){ 199 | tsharkPromises.push( tshark(capture, tempDirectory) ); 200 | } 201 | 202 | let tsharkFiles = await Promise.all( tsharkPromises ); 203 | 204 | 205 | // 3. so now everything should be in either .json or .qlog format 206 | // we now want to transform tshark's .json schema into our .qlog schema 207 | let transform = async function(capt:ICapture, outputDirectory:string):Promise{ 208 | if( capt.error ) 209 | return capt; 210 | 211 | if( fileIsQLOG(capt.capture) ){ 212 | // we already had a qlog file from the beginning 213 | // just read it and use it directly 214 | let fileContents:Buffer = await readFileAsync( capt.capture ); 215 | capt.qlog = JSON.parse(fileContents.toString()); 216 | return capt; 217 | } 218 | 219 | try{ 220 | // we don't write to file here, but pass the qlog object around directly to write a combined file later 221 | capt.qlog = await JSONToQLog.TransformToQLog( capt.capture, tempDirectory, capt.capture_original, logRawPayloads, capt.secrets, logUnkFramesFields ); 222 | } 223 | catch(e){ 224 | // console.error("ERROR transforming", e); 225 | capt.error = e; 226 | } 227 | 228 | return capt; 229 | } 230 | 231 | let transformPromises = []; 232 | for( let capture of tsharkFiles ){ 233 | transformPromises.push( transform(capture, tempDirectory) ); 234 | } 235 | 236 | let transformFiles = await Promise.all( transformPromises ); 237 | 238 | 239 | // 4. now, we finally have all the qlog files 240 | // If there are indeed multiple, we want to combine those into a single big qlog file 241 | 242 | // the capt.qlog data structures are FULL qlogs, so we need to extract the separate connections 243 | // to make a new FULL qlog that combines all of them 244 | 245 | let combined:qlog.IQLog = { 246 | qlog_version: "draft-01", 247 | // TODO Title? 248 | description: inputListDescription, 249 | // TODO Summary? 250 | traces: [] 251 | }; 252 | 253 | for( let capt of transformFiles ){ 254 | if( capt.qlog ){ 255 | combined.qlog_version = capt.qlog.qlog_version; 256 | 257 | // valid qlog found 258 | //if( !capt.description ) 259 | // capt.description = capt.capture_original; // use the filename as the description 260 | 261 | // we basically just throw away all the top-level qlog stuff 262 | // and transfer over all connection-specific information to the combined file 263 | for( let trace of capt.qlog.traces ){ 264 | if ( (trace as ITrace).description !== undefined ) { 265 | trace = trace as ITrace; 266 | trace.description = capt.description; 267 | } 268 | 269 | combined.traces.push( trace ); 270 | } 271 | } 272 | else if( capt.error ){ 273 | // we want to reflect the errors in the resulting qlog file instead of just returning nothing 274 | const err:qlog.ITraceError = { 275 | error_description: ("" + capt.error), 276 | uri: capt.capture_original, 277 | }; 278 | 279 | combined.traces.push( err ); 280 | } 281 | else{ 282 | console.error("main:combining : something went wrong. we have a capture we cannot process.", capt); 283 | } 284 | } 285 | 286 | // 5. we now have a single, combined IQLog file 287 | // we want to save this to disk so we can later use that as a cache 288 | // we need the filename to be stable, so we hash either the file+secrets paths OR the contents of the json list file 289 | let outputPath:string = output_path; 290 | if( !outputPath ){ 291 | let outputFilename:string = calculateStableHash() + ".qlog"; 292 | 293 | let outputDirectory = path.resolve( output_directory + path.sep + "cache" ); 294 | mkDirByPathSync( outputDirectory ); 295 | 296 | outputPath = outputDirectory + path.sep + outputFilename; 297 | } 298 | 299 | await writeFileAsync( outputPath, JSON.stringify(combined, null, 4) ); 300 | 301 | console.log( outputPath ); 302 | 303 | /* 304 | console.log("Input list : ", inputList); 305 | console.log("Downloaded files : ", downloadedFiles); 306 | console.log("Tsharked files : ", tsharkFiles); 307 | console.log("Transformed files : ", transformFiles); 308 | 309 | console.log("Combined output path : ", outputDirectory + path.sep + outputFilename ); 310 | */ 311 | 312 | }; 313 | 314 | Flow().then( () => { 315 | //console.log("Executing fully done"); 316 | process.exit(0); 317 | }).catch( (reason) => { 318 | console.error("Top level error " + reason); 319 | process.exit(3); 320 | }); 321 | 322 | /* 323 | process.exit(2); 324 | 325 | 326 | 327 | // 1. figure out which kind of input file we have 328 | let inputExt = path.extname(input_file); 329 | if( inputExt == "json" ){ 330 | fs.readFileSync("../" + input_file).toString() 331 | } 332 | 333 | // Determine file extension 334 | let input_file_extension: string = input_file.substr(input_file.lastIndexOf('.') + 1); 335 | 336 | 337 | if (input_file_extension === "json") { 338 | // JSON flow 339 | 340 | 341 | let jsonTrace = JSON.parse(fs.readFileSync("../" + input_file).toString()) 342 | // I don't like this. Should make better 343 | let pcapParser: ParserPCAP = new ParserPCAP(jsonTrace); 344 | 345 | let myQLogConnection: qlog.IConnection; 346 | myQLogConnection = { 347 | quic_version: pcapParser.getQUICVersion(), 348 | vantagepoint: qlog.VantagePoint.NETWORK, 349 | connectionid: pcapParser.getConnectionID(), 350 | starttime: pcapParser.getStartTime(), 351 | fields: ["time", "category", "type", "trigger", "data"], 352 | events: [] 353 | }; 354 | 355 | // First event = new connection 356 | myQLogConnection.events.push([ 357 | 0, 358 | qlog.EventCategory.CONNECTIVITY, 359 | qlog.ConnectivityEventType.NEW_CONNECTION, 360 | qlog.ConnectivityEventTrigger.LINE, 361 | pcapParser.getConnectionInfo() as qlog.IEventNewConnection 362 | ]); 363 | 364 | //TODO keys 365 | 366 | for (let x of jsonTrace) { 367 | let frame = x['_source']['layers']['frame']; 368 | let quic = x['_source']['layers']['quic']; 369 | 370 | let time = parseFloat(frame['frame.time_epoch']); 371 | let time_relative: number = Math.round((time - myQLogConnection.starttime) * 1000); 372 | 373 | let header = {} as qlog.IPacketHeader; 374 | 375 | if (quic['quic.header_form'] == '1') // LONG header 376 | { 377 | header.form = 'long'; 378 | header.type = PCAPUtil.getPacketType(quic['quic.long.packet_type']); 379 | header.version = quic['quic.version']; 380 | header.scid = quic['quic.scid'].replace(/:/g, ''); 381 | header.dcid = quic['quic.dcid'].replace(/:/g, ''); 382 | header.scil = quic['quic.scil'].replace(/:/g, ''); 383 | header.dcil = quic['quic.dcil'].replace(/:/g, ''); 384 | header.payload_length = quic['quic.length']; 385 | header.packet_number = quic['quic.packet_number_full']; 386 | } 387 | else { 388 | header.form = 'short'; 389 | header.dcid = quic['quic.dcid'].replace(/:/g, ''); 390 | header.payload_length = 0; // TODO! 391 | header.packet_number = quic['quic.packet_number_full']; 392 | } 393 | 394 | let entry: any = { 395 | raw_encrypted: "TODO", 396 | header: header, 397 | }; 398 | 399 | let keys = Object.keys(quic['quic.frame']); 400 | let tmp: any = {}; 401 | for (var j = 0; j < keys.length; j++) { 402 | var key = keys[j].replace(/^quic\./, ""); 403 | tmp[key] = quic['quic.frame'][keys[j]]; 404 | } 405 | 406 | 407 | tmp['frame_type'] = PCAPUtil.getFrameTypeName(tmp['frame_type']); 408 | 409 | entry.frames = []; 410 | entry.frames.push(tmp); 411 | 412 | x = [] as qlog.IEventPacketRX; 413 | 414 | 415 | myQLogConnection.events.push([ 416 | time_relative, 417 | qlog.EventCategory.TRANSPORT, 418 | qlog.TransportEventType.TRANSPORT_PACKET_RX, 419 | qlog.TransporEventTrigger.LINE, 420 | entry 421 | ]); 422 | } 423 | 424 | let myQLog: qlog.IQLog; 425 | myQLog = { 426 | qlog_version: "0.1", 427 | description: input_file, 428 | connections: [myQLogConnection] 429 | }; 430 | 431 | //TODO write to file 432 | console.log(JSON.stringify(myQLog, null, 4)); 433 | 434 | fs.writeFileSync("20181219_handshake_v6_quicker.edm.uhasselt.be.qlog", JSON.stringify(myQLog, null, 4)); 435 | } 436 | else if (input_file_extension === "pcap" || input_file_extension === "pcapng") { 437 | // PCAP(NG) flow 438 | // - encrypted PCAP? 439 | // -- supply secret 440 | // -> decrypted PCAP 441 | // -- run tshark 442 | // -- goto JSON flow 443 | 444 | } 445 | 446 | 447 | */ -------------------------------------------------------------------------------- /src/util/FileUtil.ts: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | 4 | const crypto = require("crypto"); 5 | 6 | // https://stackoverflow.com/questions/31645738/how-to-create-full-path-with-nodes-fs-mkdirsync 7 | export function mkDirByPathSync(targetDir:string, {isRelativeToScript = false} = {}) { 8 | const sep = path.sep; 9 | const initDir = path.isAbsolute(targetDir) ? sep : ''; 10 | const baseDir = isRelativeToScript ? __dirname : '.'; 11 | 12 | targetDir.split(sep).reduce((parentDir, childDir) => { 13 | const curDir = path.resolve(baseDir, parentDir, childDir); 14 | try { 15 | if (!fs.existsSync(curDir)) { 16 | fs.mkdirSync(curDir); 17 | } 18 | //console.log(`Directory ${curDir} created!`); 19 | } catch (err) { 20 | if (err.code !== 'EEXIST' && err.code !== "EISDIR" && 21 | !(err.code == 'EPERM' && curDir == "C:\\") ) { 22 | throw err; 23 | } 24 | 25 | //console.log(`Directory ${curDir} already exists!`); 26 | } 27 | 28 | return curDir; 29 | }, initDir); 30 | 31 | } 32 | 33 | export function createHash(contents:string){ 34 | return crypto.createHash('sha1').update( contents ).digest('hex'); 35 | } 36 | 37 | 38 | export function getFileExtension(target:string){ 39 | 40 | let ext:string = path.extname(target); // gives extensions including the dot, e.g., .json 41 | 42 | if( !ext || ext.length == 0 ){ 43 | if( fileIsJSON(target) ) 44 | ext = ".json"; 45 | else if( fileIsPCAP(target) ) 46 | ext = ".pcap"; 47 | else if( fileIsSECRETS(target) ) 48 | ext = ".keys"; 49 | else if( fileIsQLOG(target) ){ 50 | ext = ".qlog"; 51 | } 52 | } 53 | 54 | return ext; 55 | } 56 | 57 | export function fileIsJSON(path:string){ 58 | return path.indexOf(".json") >= 0; 59 | } 60 | export function fileIsPCAP(path:string){ 61 | // either .pcap or simply pcap at the end (e.g., quic-tracker has a url in the form quick-tracker.com/trace/xyz/pcap) 62 | return path.indexOf(".pcap") >= 0 || path.slice(-5) === "/pcap"; 63 | } 64 | export function fileIsSECRETS(path:string){ 65 | // either .keys/.secrets or simply secrets at the end (e.g., quic-tracker has a url in the form quick-tracker.com/trace/xyz/secrets) 66 | return path.indexOf(".secrets") >= 0 || path.indexOf(".keys") >= 0 || path.slice(-8) === "/secrets"; 67 | } 68 | export function fileIsQLOG(path:string){ 69 | return path.indexOf(".qlog") >= 0; 70 | } 71 | -------------------------------------------------------------------------------- /src/util/PCAPUtil.ts: -------------------------------------------------------------------------------- 1 | 2 | import * as qlog from "@quictools/qlog-schema/dist/draft-01/QLog"; 3 | 4 | export class PCAPUtil { 5 | 6 | 7 | public static getFrameTypeName(frameType: number): qlog.QUICFrameTypeName { 8 | if (frameType === 0x00) 9 | return qlog.QUICFrameTypeName.padding; 10 | else if (frameType === 0x01) 11 | return qlog.QUICFrameTypeName.ping; 12 | else if (frameType === 0x02 || frameType === 0x03) // 0x02 is without ECN while 0x03 is with ECN 13 | return qlog.QUICFrameTypeName.ack; 14 | else if (frameType === 0x04) 15 | return qlog.QUICFrameTypeName.reset_stream; 16 | else if (frameType === 0x05) 17 | return qlog.QUICFrameTypeName.stop_sending; 18 | else if (frameType === 0x06) 19 | return qlog.QUICFrameTypeName.crypto; 20 | else if (frameType === 0x07) 21 | return qlog.QUICFrameTypeName.new_token; 22 | else if (frameType >= 0x08 && frameType <= 0x0f) 23 | return qlog.QUICFrameTypeName.stream; 24 | else if (frameType === 0x10) 25 | return qlog.QUICFrameTypeName.max_data; 26 | else if (frameType === 0x11) 27 | return qlog.QUICFrameTypeName.max_stream_data; 28 | else if (frameType === 0x12 || frameType === 0x013) // 0x12 is bidi while 0x13 is uni 29 | return qlog.QUICFrameTypeName.max_streams; 30 | else if (frameType === 0x14) 31 | return qlog.QUICFrameTypeName.data_blocked; 32 | else if (frameType === 0x15) 33 | return qlog.QUICFrameTypeName.stream_data_blocked; 34 | else if (frameType === 0x16 || frameType === 0x017) // 0x16 is bidi while 0x13 is uni 35 | return qlog.QUICFrameTypeName.streams_blocked; 36 | else if (frameType === 0x18) 37 | return qlog.QUICFrameTypeName.new_connection_id; 38 | else if (frameType === 0x19) 39 | return qlog.QUICFrameTypeName.retire_connection_id; 40 | else if (frameType === 0x1a) 41 | return qlog.QUICFrameTypeName.path_challenge; 42 | else if (frameType === 0x1b) 43 | return qlog.QUICFrameTypeName.path_response; 44 | else if (frameType === 0x1c || frameType === 0x1d) // 0x1c is QUIC-layer error (or no error) while 0x1d is application layer error 45 | return qlog.QUICFrameTypeName.connection_close; 46 | return qlog.QUICFrameTypeName.unknown_frame_type; 47 | } 48 | 49 | public static extractQUICPackets( entry:any ) { 50 | // in wireshark, an entry can contain 0, 1 or multiple QUIC packets 51 | // multiple is always an array, 1 is always a normal object, 0 is a normal object but without useful fields 52 | if ( Array.isArray(entry) ) { 53 | return entry; 54 | } 55 | else { 56 | // TODO: now we assume all quic packets are useful, revise once we've seen the contrary (have seen this many times with TCP) 57 | return [entry]; 58 | } 59 | } 60 | 61 | // jsonPath should be marked by / instead of ., because wireshark .jsons often use . inside of keys as well (because... logic) 62 | // so it's more like xPath 63 | public static ensurePathExists( jsonPath:string, obj:any, throwError:boolean = true ):boolean { 64 | let pathElements = jsonPath.split("/"); 65 | let currentObj = obj; 66 | for ( const el of pathElements ) { 67 | if ( currentObj[el] === undefined ) { 68 | let summary = JSON.stringify(currentObj).substr(0,5000); 69 | if ( throwError ) { 70 | throw new Error("ParserPCAP:ensurePathExists : path not found : " + jsonPath + " at " + el + " : " + summary); 71 | } 72 | else { 73 | return false; 74 | } 75 | } 76 | 77 | currentObj = currentObj[el]; 78 | } 79 | 80 | return true; 81 | } 82 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "esnext", 5 | /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ 6 | "module": "commonjs", // node cannot deal with 'import' statements 7 | "moduleResolution": "node", 8 | /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 9 | "watch": false, 10 | "esModuleInterop": true, 11 | "allowSyntheticDefaultImports": true, 12 | // "lib": [], /* Specify library files to be included in the compilation: */ 13 | // "allowJs": true, /* Allow javascript files to be compiled. */ 14 | // "checkJs": true, /* Report errors in .js files. */ 15 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 16 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 17 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 18 | // "outFile": "./", /* Concatenate and emit output to single file. */ 19 | "outDir": "./out", 20 | /* Redirect output structure to the directory. */ 21 | //"rootDir": "./src", 22 | "rootDirs": [ 23 | "./src" 24 | ], 25 | "noEmitOnError": true, 26 | //"include": ["./node_modules/@quictools/qlog-schema/**/*.ts"], 27 | /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 28 | // "removeComments": true, /* Do not emit comments to output. */ 29 | // "noEmit": true, /* Do not emit outputs. */ 30 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 31 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 32 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 33 | /* Strict Type-Checking Options */ 34 | "strict": true, 35 | /* Enable all strict type-checking options. */ 36 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 37 | // "strictNullChecks": true, /* Enable strict null checks. */ 38 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 39 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 40 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 41 | /* Additional Checks */ 42 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 43 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 44 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 45 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 46 | /* Module Resolution Options */ 47 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 48 | "baseUrl": ".", 49 | /* Base directory to resolve non-absolute module names. */ 50 | // "paths": { 51 | // /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 52 | // "*": [] 53 | // }, 54 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 55 | // "typeRoots": [], /* List of folders to include type definitions from. */ 56 | // "types": [], /* Type declaration files to be included in compilation. */ 57 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 58 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 59 | /* Source Map Options */ 60 | // "sourceRoot": "./", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 61 | // "mapRoot": "./", /* Specify the location where debugger should locate map files instead of generated locations. */ 62 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 63 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 64 | /* Experimental Options */ 65 | "experimentalDecorators": true, 66 | /* Enables experimental support for ES7 decorators. */ 67 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 68 | "paths": { 69 | "@/*": [ 70 | "src/*" 71 | ] 72 | }, 73 | "lib": [ 74 | "esnext", 75 | "dom", 76 | "dom.iterable", 77 | "scripthost" 78 | ] 79 | } 80 | } --------------------------------------------------------------------------------