├── Makefile
├── README.md
├── fFile.c
├── fFile.h
├── fProfile.c
├── fProfile.h
├── fTypes.h
├── main.c
├── sha1.c
├── tcpstream.c
├── tcpstream.h
├── udpstream.c
└── udpstream.h


/Makefile:
--------------------------------------------------------------------------------
 1 | OBJS =
 2 | OBJS += tcpstream.o
 3 | OBJS += udpstream.o
 4 | OBJS += main.o
 5 | OBJS += sha1.o
 6 | OBJS += fProfile.o
 7 | OBJS += fFile.o
 8 | 
 9 | DEF = 
10 | DEF += -O3 
11 | DEF += --std=c99 
12 | DEF += -D_LARGEFILE64_SOURCE 
13 | DEF += -D_GNU_SOURCE 
14 | DEF += -g
15 | 
16 | LIBS =
17 | LIBS += -lm
18 | LIBS += -lpthread
19 | 
20 | %.o: %.c
21 | 	gcc $(DEF) -c -o $@ -g $<
22 | 
23 | all: $(OBJS) 
24 | 	gcc -g -o pcap_flows $(OBJS)  $(LIBS)
25 | 
26 | clean:
27 | 	rm -f $(OBJS)
28 | 	rm -f pcap_flows
29 | 
30 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # pcap_flow
  2 | 
  3 | ![Alt text](http://firmware.fmad.io/images/logo_flow_analyzer.png "fmadio flow analyzer logo")
  4 | 
  5 | [https://fmad.io/ FMADIO Network Packet Capture](https://fmad.io)
  6 | 
  7 | 
  8 | displays flow information from a PCAP and can extract individual (or all) TCP streams
  9 | 
 10 | ### Options
 11 | 
 12 | Command line options
 13 | 
 14 | ```
 15 | --output_tcp <filename>                    | write filtered TCP flows to the specified directory
 16 | --output_udp <filename>                    | write filtered UDP flows to the specified directory
 17 | --packet-max  <number>                     | only process the first <number> packets
 18 | --extract <number>                         | extract FlowID <number> into the output PCAP file
 19 | --extract-tcp <number>                     | extract FlowID <number> as a TCP stream to the output file name
 20 | --extract-tcp-port <start port> <end port> | extract all TCP flows with the specified port in src or dest 
 21 | --extract-ip 1.2.3.4/255.255.255.255       | extract all IP`s matching the sepcificed mask into the output PCAP
 22 | --extract-port <start port> <end port>     | extract all UDP/TCP packets matching the range into a seperate PCAP 
 23 | --stdin                                    | read PCAP from stdin. e.g. zcat capture.pcap | pcap_flow --stdin
 24 | --disable-display                          | do not display flow information to stdout
 25 | --tcpheader                                | include header in TCP output stream 
 26 | ```
 27 | 
 28 | ### Examples
 29 | 
 30 | 
 31 | 1) generate flow information from a compressed PCAP file
 32 | 
 33 | ```
 34 | zcat capture.pcap.gz | pcap_flows --stdin
 35 | ```
 36 | 
 37 | 2) output a specific flow to a separate PCAP file 
 38 | 
 39 | ```
 40 | pcap_flows --extract 1234 raw_capture.pcap -o capture_flow_1234.pcap
 41 | ```
 42 | 
 43 | 3) extract a TCP stream from a pcap
 44 | 
 45 | ```
 46 | pcap_flows --extract-tcp 1234 raw_capture.pcap -o capture_flow_as_tcp1234.pcap
 47 | ```
 48 | 
 49 | 3) extract all TCP streams from port 80 to port 80 
 50 | 
 51 | Note: this can generate a very large number of files (one per stream) in the output directory. e.g. /tmp/tcp_stream_directory/extract_192.168.1.1-80->12345.pcap 
 52 | 
 53 | ```
 54 | pcap_flows /mnt/capture/hitcon_small.pcap --extract-tcp-port 80 80 -o ./tmp/port80_
 55 | 
 56 | $ ls tmp/port80* | wc -l
 57 | 20217
 58 | 
 59 | $ hexdump -Cv "tmp/port80__00:10:18:72:00:3c->e0:3f:49:6a:af:a1_117. 27.153. 29-> 10.  5.  9.102_    80-> 62374" | head
 60 | 00000000  48 54 54 50 2f 31 2e 31  20 32 30 30 20 4f 4b 0d  |HTTP/1.1 200 OK.|
 61 | 00000010  0a 53 65 72 76 65 72 3a  20 6e 67 69 6e 78 0d 0a  |.Server: nginx..|
 62 | 00000020  44 61 74 65 3a 20 46 72  69 2c 20 30 38 20 41 75  |Date: Fri, 08 Au|
 63 | 00000030  67 20 32 30 31 34 20 31  37 3a 34 39 3a 35 38 20  |g 2014 17:49:58 |
 64 | 00000040  47 4d 54 0d 0a 43 6f 6e  74 65 6e 74 2d 54 79 70  |GMT..Content-Typ|
 65 | 00000050  65 3a 20 69 6d 61 67 65  2f 6a 70 65 67 0d 0a 43  |e: image/jpeg..C|
 66 | 00000060  6f 6e 74 65 6e 74 2d 4c  65 6e 67 74 68 3a 20 31  |ontent-Length: 1|
 67 | 00000070  32 32 33 32 0d 0a 43 6f  6e 6e 65 63 74 69 6f 6e  |2232..Connection|
 68 | 00000080  3a 20 63 6c 6f 73 65 0d  0a 4c 61 73 74 2d 4d 6f  |: close..Last-Mo|
 69 | 00000090  64 69 66 69 65 64 3a 20  54 75 65 2c 20 32 39 20  |dified: Tue, 29 |
 70 | 
 71 | 
 72 | ```
 73 | 
 74 | 
 75 | ### TCP Output format 
 76 | 
 77 | The default TCP Output format is a flat linear file of the re-assemabled TCP stream. However with the `--tcpheader` flag each succesfully re-assembled TCP segment contains a header. The header format is: 
 78 | 
 79 | 
 80 | ```
 81 | 
 82 | typedef struct
 83 | {
 84 |     u64     TS;                 // nanoseccond timestamp 
 85 |     u16     Length;             // number of bytes in this packet
 86 |     u16     StreamID;           // unique id per flow
 87 | 
 88 | } TCPOutputHeader_t;
 89 | 
 90 | 
 91 | ```
 92 | 
 93 | This allows parsing a TCP stream is like parsing a UDP packet stream. Each outputed TCP packet is a single reassembled TCP segment written in-order, with no re-sends and no sequence gaps. 
 94 | 
 95 | 
 96 | ### Output
 97 | 
 98 | Display flow info from hitcon defcon CTF capture
 99 | 
100 | 
101 | ```
102 | 
103 | pcap_flows  /hitcon.pcap  --flow-packet-min 1000
104 | 
105 | 1048549 FlowID:   592897 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5. 17.  2 ->  10.  5.  9.  2 |  43942 ->   8888  |             4,102 Pkts           288,909 Bytes
106 | 1048550 FlowID:   761379 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5.  3.  2 ->  10.  5.  9.  2 |  48716 ->   8888  |             4,113 Pkts           289,197 Bytes
107 | 1048551 FlowID:   981924 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5.  7.  2 ->  10.  5.  9.  2 |  42653 ->   8888  |             4,183 Pkts           294,250 Bytes
108 | 1048552 FlowID:   642639 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  17.253.  2.226 |  63281 ->     80  |             4,301 Pkts           295,014 Bytes
109 | 1048553 FlowID:   902015 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5. 12.  2 ->  10.  5.  9.  2 |  36486 ->   8888  |             4,352 Pkts           305,988 Bytes
110 | 1048554 FlowID:    53839 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5. 16.  2 ->  10.  5.  9.  2 |  43103 ->   8888  |             4,715 Pkts           331,990 Bytes
111 | 1048555 FlowID:   658515 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5. 15.  2 ->  10.  5.  9.  2 |  45683 ->   8888  |             4,786 Pkts           337,001 Bytes
112 | 1048556 FlowID:    33656 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 | 130.204. 67.136 ->  10.  5.  9.102 |   9025 ->  56574  |             4,930 Pkts           537,324 Bytes
113 | 1048557 FlowID:   643944 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5. 10.  2 ->  10.  5.  9.  2 |  44934 ->   8888  |             4,995 Pkts           351,892 Bytes
114 | 1048558 FlowID:     8462 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5.  2.  2 ->  10.  5.  9.  2 |  41809 ->   8888  |             5,126 Pkts           360,763 Bytes
115 | 1048559 FlowID:   627433 | TCP  00:10:18:72:00:3c -> 00:16:3e:ef:36:38 |  10.  5.  8.  2 ->  10.  5.  9.  2 |  44283 ->   8888  |             5,394 Pkts           379,946 Bytes
116 | 1048560 FlowID:    88064 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 | 140.115. 50. 51 ->  10.  5.  9.102 |     22 ->  42271  |             6,102 Pkts           417,083 Bytes
117 | 1048561 FlowID:    24006 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  74.125.129.189 ->  10.  5.  9.102 |    443 ->  61860  |             6,502 Pkts           658,192 Bytes
118 | 1048562 FlowID:   785299 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  10.  5.  6.108 ->  10.  5.  9.102 |     80 ->  53303  |             6,559 Pkts         9,849,540 Bytes
119 | 1048563 FlowID:    23999 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  74.125.129.189 |  61860 ->    443  |             6,588 Pkts         2,583,463 Bytes
120 | 1048564 FlowID:    33651 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 -> 130.204. 67.136 |  56574 ->   9025  |             6,609 Pkts           622,258 Bytes
121 | 1048565 FlowID:  1005605 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  10.  5.  6.108 |  63779 ->     80  |             7,149 Pkts           453,291 Bytes
122 | 1048566 FlowID:   786260 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  10.  5.  6.108 ->  10.  5.  9.102 |     80 ->  53413  |             8,367 Pkts        12,625,278 Bytes
123 | 1048567 FlowID:   642795 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  54.183.128. 64 |  52940 ->  22222  |            10,502 Pkts         2,409,657 Bytes
124 | 1048568 FlowID:    88059 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 -> 140.115. 50. 51 |  42271 ->     22  |            10,955 Pkts        16,496,355 Bytes
125 | 1048569 FlowID:        1 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  54.183.128. 64 |  51697 ->  22222  |            11,666 Pkts         3,839,832 Bytes
126 | 1048570 FlowID:  1005606 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  10.  5.  6.108 ->  10.  5.  9.102 |     80 ->  63779  |            14,670 Pkts        21,774,873 Bytes
127 | 1048571 FlowID:        2 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  54.183.128. 64 ->  10.  5.  9.102 |  22222 ->  51697  |            16,714 Pkts         1,830,744 Bytes
128 | 1048572 FlowID:   642798 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  54.183.128. 64 ->  10.  5.  9.102 |  22222 ->  52940  |            16,997 Pkts         1,921,123 Bytes
129 | 1048573 FlowID:   642638 | TCP  e0:3f:49:6a:af:a1 -> 00:10:18:72:00:3c |  10.  5.  9.102 ->  17.253.  2.226 |  63280 ->     80  |            98,135 Pkts         6,584,162 Bytes
130 | 1048574 FlowID:   642642 | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  17.253.  2.226 ->  10.  5.  9.102 |     80 ->  63280  |           115,911 Pkts       245,630,927 Bytes
131 | ```
132 | 
133 | Extract only port 80 traffic from hitcon.pcap to a separate file. This is the individual TCP port 80 -> 63280 flow.
134 | 
135 | 1048574 **FlowID:   642642** | TCP  00:10:18:72:00:3c -> e0:3f:49:6a:af:a1 |  17.253.  2.226 ->  10.  5.  9.102 |     80 ->  63280  |           115,911 Pkts       245,630,927 Bytes
136 | 
137 | ```
138 | $ pcap_flows  hitcon.pcap  --extract 642642 -o /mnt/capture/hitcon_http.pcap --disable-display 
139 | 
140 | writing PCAP to [/mnt/capture/hitcon_http.pcap]
141 | [/mnt/capture/hitcon_small.pcap] FileSize: 2GB
142 | [02:00:30.000.332.313 0.000%] Flows:2 0.00M Pkts 0.000Gbps : 0.00GB Out:0.00GB
143 | [05:10:22.000.316.568 0.307%] Flows:899478 2.66M Pkts 7.419Gbps : 0.79GB Out:0.25GB
144 | [02:27:22.000.060.690 0.816%] Flows:1048576 9.71M Pkts 12.695Gbps : 2.16GB Out:0.25GB
145 | ```
146 | 


--------------------------------------------------------------------------------
/fFile.c:
--------------------------------------------------------------------------------
  1 | //---------------------------------------------------------------------------------------------
  2 | //
  3 | // Copyright (c) 2019 fmad engineering llc 
  4 | //
  5 | // The MIT License (MIT) see LICENSE file for details 
  6 | //
  7 | // file wrapper 
  8 | //
  9 | //---------------------------------------------------------------------------------------------
 10 | 
 11 | #include <stdio.h>
 12 | #include <stdint.h>
 13 | #include <stdlib.h>
 14 | #include <unistd.h>
 15 | #include <string.h>
 16 | #include <errno.h>
 17 | #include <sys/stat.h>
 18 | #include <sys/mman.h>
 19 | #include <sys/shm.h>
 20 | #include <fcntl.h>
 21 | #include <locale.h>
 22 | #include <assert.h>
 23 | #include <errno.h>
 24 | 
 25 | #include "fTypes.h"
 26 | #include "fProfile.h"
 27 | #include "fFile.h"
 28 | 
 29 | //---------------------------------------------------------------------------------------------
 30 | 
 31 | 
 32 | //---------------------------------------------------------------------------------------------
 33 | 
 34 | typedef struct fFile_t
 35 | {
 36 | 	u8		Path[1024];							// path to file
 37 | 
 38 | 	FILE*	File;								// usual file handle if its created
 39 | 
 40 | 	bool	IsBuffer;							// file in buffer or IO mode
 41 | 	u8*		Buffer;								// temp packet buffer
 42 | 	u64		BufferPos;							// current position in packet buffer
 43 | 	u64		BufferMax;							// max buffer size
 44 | 
 45 | 	u64		TotalByte;
 46 | 	u64		TotalPayload;						// total payload bytes
 47 | 
 48 | } fFile_t;
 49 | 
 50 | static u64		s_TotalFile			= 0;		// total number of files allocated
 51 | static u64		s_TotalFileActive	= 0;		// total number of files active 
 52 | static u64		s_TotalFileClose	= 0;		// number of files closed 
 53 | 
 54 | static u64 		s_FileSizeMin		= 128;		// minimum file size
 55 | static bool		s_ForceFlush		= false;	// force flushing after every write 
 56 | 
 57 | //---------------------------------------------------------------------------------------------
 58 | 
 59 | struct fFile_t* fFile_Open(u8* Path, u8* Mode)
 60 | {
 61 | 	fFile_t* F = (fFile_t*)malloc( sizeof(fFile_t) );
 62 | 	memset(F, 0, sizeof(fFile_t));
 63 | 
 64 | 	strncpy(F->Path, Path, sizeof(F->Path));
 65 | 
 66 | 	// initial output buffer 
 67 | 	F->IsBuffer		= true;
 68 | 	F->BufferMax	= s_FileSizeMin;
 69 | 	F->BufferPos	= 0;
 70 | 	F->Buffer		= malloc( F->BufferMax );		
 71 | 	memset(F->Buffer, 0, F->BufferMax );
 72 | 
 73 | 	F->TotalByte	= 0;
 74 | 
 75 | 	// if file existed before get total length
 76 | 	struct stat fstat;	
 77 | 	if (stat(F->Path, &fstat) >= 0) 
 78 | 	{
 79 | 		F->TotalByte = fstat.st_size;
 80 | 	}
 81 | 
 82 | 	// new file allocated 
 83 | 	s_TotalFile++;
 84 | 
 85 | 	return F;
 86 | }
 87 | 
 88 | //---------------------------------------------------------------------------------------------
 89 | 
 90 | void fFile_Write(struct fFile_t* F, void* Buffer, u32 Length, bool IsPayload)
 91 | {
 92 | 	// should never write with a null file handle
 93 | 	assert(F != NULL);
 94 | 
 95 | 	// need to create a file 
 96 | 	if (F->IsBuffer)
 97 | 	{
 98 | 		if (F->BufferPos + Length > F->BufferMax)
 99 | 		{
100 | 			F->File = fopen(F->Path, "a");
101 | 			assert(F->File != NULL);
102 | 
103 | 			fwrite(F->Buffer, 1, F->BufferPos, F->File);
104 | 			F->TotalByte	+= F->BufferPos;
105 | 
106 | 			F->IsBuffer 	= false;
107 | 			F->BufferPos	= 0;
108 | 
109 | 			s_TotalFileActive++;
110 | 		}
111 | 	}
112 | 
113 | 	// buffer it
114 | 	if (F->IsBuffer)
115 | 	{
116 | 		memcpy(F->Buffer + F->BufferPos, Buffer, Length);
117 | 		F->BufferPos += Length;
118 | 	}
119 | 	else
120 | 	{
121 | 		assert(F->File != NULL);
122 | 
123 | 		fwrite(Buffer, 1, Length, F->File);
124 | 		if (s_ForceFlush) fflush(F->File);
125 | 		F->TotalByte	+= Length; 
126 | 	}
127 | 
128 | 	// if this is payload
129 | 	if (IsPayload) F->TotalPayload += Length;
130 | }
131 | 
132 | //---------------------------------------------------------------------------------------------
133 | 
134 | void fFile_Close(struct fFile_t* F)
135 | {
136 | 	if (!F) return;
137 | 
138 | 	// flush buffer contents if not written to disk
139 | 	if (F->IsBuffer && (F->BufferPos > 0))
140 | 	{
141 | 		// if theres real substational data the flush it
142 | 		if (F->TotalPayload > s_FileSizeMin)
143 | 		{
144 | 			F->File = fopen(F->Path, "a");
145 | 			assert(F->File != NULL);
146 | 
147 | 			fwrite(F->Buffer, 1, F->BufferPos, F->File);
148 | 			F->BufferPos = 0;
149 | 		}
150 | 	}
151 | 
152 | 	// close
153 | 	if (F->File)
154 | 	{
155 | 		fclose(F->File);
156 | 		s_TotalFileActive--;
157 | 	}
158 | 
159 | 	if (F->Buffer)
160 | 	{
161 | 		free(F->Buffer);
162 | 		F->Buffer = NULL;
163 | 	}
164 | 	memset(F, 0, sizeof(fFile_t));
165 | 	free(F);
166 | 
167 | 	s_TotalFileClose++;
168 | }
169 | 
170 | //---------------------------------------------------------------------------------------------
171 | 
172 | void fFile_Flush(struct fFile_t* F)
173 | {
174 | 	if (F->File)
175 | 	{
176 | 		fflush(F->File);
177 | 	}
178 | }
179 | 
180 | //---------------------------------------------------------------------------------------------
181 | // set the minimum file size before creating a file 
182 | void fFile_SizeMin(u32 SizeMin)
183 | {
184 | 	s_FileSizeMin = SizeMin;
185 | }
186 | 
187 | //---------------------------------------------------------------------------------------------
188 | // force flushing data after each write 
189 | void fFile_ForceFlush(void)
190 | {
191 | 	s_ForceFlush = true;	
192 | }
193 | 
194 | //---------------------------------------------------------------------------------------------
195 | 


--------------------------------------------------------------------------------
/fFile.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FILE_FMAD_H__
 2 | #define __FILE_FMAD_H__
 3 | 
 4 | struct fFile_t;
 5 | 
 6 | struct fFile_t* fFile_Open			(u8* Path, u8* Mode);
 7 | void 			fFile_Write			(struct fFile_t* F, void* Buffer, u32 Length, bool IsPayload);
 8 | void 			fFile_Close			(struct fFile_t* F);
 9 | void 			fFile_Flush			(struct fFile_t* F);
10 | 
11 | void 			fFile_SizeMin		(u32 SizeMin);
12 | void 			fFile_ForceFlush	();
13 | 
14 | #endif
15 | 


--------------------------------------------------------------------------------
/fProfile.c:
--------------------------------------------------------------------------------
 1 | //---------------------------------------------------------------------------------------------
 2 | //
 3 | // Copyright (c) 2015, fmad engineering llc 
 4 | //
 5 | // quick and dirty profiling 
 6 | //
 7 | //---------------------------------------------------------------------------------------------
 8 | 
 9 | #include <stdio.h>
10 | #include <stdint.h>
11 | #include <stdlib.h>
12 | #include <unistd.h>
13 | #include <string.h>
14 | #include <errno.h>
15 | #include <sys/stat.h>
16 | #include <sys/mman.h>
17 | #include <sys/shm.h>
18 | 
19 | #include "fTypes.h"
20 | #include "fProfile.h"
21 | 
22 | 
23 | u64 g_ProfileStart[32];
24 | u64 g_ProfileStop[32];
25 | u64 g_ProfileTotal[32];
26 | char* g_ProfileDesc[32];
27 | 
28 | void fProfile_Reset(void)
29 | {
30 | 	for (int i=0; i < 16; i++)
31 | 	{
32 | 		g_ProfileStart	[i] = 0;
33 | 		g_ProfileStop	[i] = 0;
34 | 		g_ProfileTotal	[i] = 0;
35 | 	}
36 | }
37 | 
38 | void fProfile_Dump(u32 Index)
39 | {
40 | 	double oot = 1.0 / g_ProfileTotal[Index];
41 | 	for (int i=0; i < 16; i++)
42 | 	{
43 | 		fprintf(stderr, "    [%2i] %016llx %20lli : (%.4f) : %s\n", 
44 | 				i,
45 | 				g_ProfileTotal[i],
46 | 				g_ProfileTotal[i],
47 | 				g_ProfileTotal[i] * oot,
48 | 				g_ProfileDesc[i]);
49 | 	}
50 | 	fProfile_Reset();
51 | }
52 | 


--------------------------------------------------------------------------------
/fProfile.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FMAD_COMMON_PROFILE_H__
 2 | #define __FMAD_COMMON_PROFILE_H__
 3 | 
 4 | extern u64 g_ProfileStart[32];
 5 | extern u64 g_ProfileStop[32];
 6 | extern u64 g_ProfileTotal[32];
 7 | extern char* g_ProfileDesc[32];
 8 | 
 9 | //---------------------------------------------------------------------------------------------
10 | 
11 | void fProfile_Reset(void);
12 | void fProfile_Dump(u32 Index);
13 | 
14 | static inline void fProfile_Start(u32 Index, char* Desc)
15 | {
16 | 	g_ProfileStart[Index] = rdtsc();
17 | 	g_ProfileDesc[Index] = Desc; 
18 | }
19 | 
20 | static inline void fProfile_Stop(u32 Index)
21 | {
22 | 	g_ProfileTotal[Index] += rdtsc() - g_ProfileStart[Index];
23 | }
24 | 
25 | static inline u64 fProfile_Cycles(u32 Index)
26 | {
27 | 	return g_ProfileTotal[Index];
28 | }
29 | 
30 | //---------------------------------------------------------------------------------------------
31 | // intel code analyzer
32 | 
33 | #if defined (__GNUC__)
34 | #define IACA_SSC_MARK( MARK_ID )                        \
35 | __asm__ __volatile__ (                                  \
36 |                       "\n\t  movl $"#MARK_ID", %%ebx"   \
37 |                       "\n\t  .byte 0x64, 0x67, 0x90"    \
38 |                       : : : "memory" );
39 | 
40 | #define IACA_UD_BYTES __asm__ __volatile__ ("\n\t .byte 0x0F, 0x0B");
41 | 
42 | #define IACA_START {IACA_UD_BYTES     IACA_SSC_MARK(111)}
43 | #define IACA_END {IACA_SSC_MARK(222)  IACA_UD_BYTES}
44 | 
45 | #endif
46 | 
47 | 
48 | #endif
49 | 


--------------------------------------------------------------------------------
/fTypes.h:
--------------------------------------------------------------------------------
  1 | //---------------------------------------------------------------------------------------------
  2 | //
  3 | // Copyright (c) 2015, fmad engineering llc 
  4 | //
  5 | // The MIT License (MIT) see LICENSE file for details 
  6 | //
  7 | // pcap latency diff  
  8 | //
  9 | //---------------------------------------------------------------------------------------------
 10 | 
 11 | #ifndef __F_TYPES_H__
 12 | #define __F_TYPES_H__
 13 | 
 14 | #include <math.h>
 15 | #include <limits.h>
 16 | #include <assert.h>
 17 | #include <time.h>
 18 | #include <sys/time.h>
 19 | 
 20 | typedef unsigned int		bool;
 21 | #define true				1
 22 | #define false				0
 23 | 
 24 | typedef unsigned char		u8;
 25 | typedef char				s8;
 26 | 
 27 | typedef unsigned short		u16;
 28 | typedef short				s16;
 29 | 
 30 | typedef unsigned int 		u32;
 31 | typedef int					s32;
 32 | 
 33 | typedef unsigned long long	u64;
 34 | typedef long long			s64;
 35 | 
 36 | #define k1E9 1000000000ULL
 37 | 
 38 | #define kKB(a) ( ((u64)a)*1024ULL)
 39 | #define kMB(a) ( ((u64)a)*1024ULL*1024ULL)
 40 | #define kGB(a) ( ((u64)a)*1024ULL*1024ULL*1024ULL)
 41 | #define kTB(a) ( ((u64)a)*1024ULL*1024ULL*1024ULL*1024ULL)
 42 | 
 43 | // time utils
 44 | 
 45 | typedef struct
 46 | {
 47 | 	int		year;
 48 | 	int		month;
 49 | 	int		day;
 50 | 	int		hour;
 51 | 	int		sec;
 52 | 	int		min;
 53 | } clock_date_t;
 54 | 
 55 | static clock_date_t  clock_date(void)
 56 | {
 57 | 	struct timeval tv;
 58 | 	gettimeofday(&tv, NULL);
 59 | 
 60 | 	struct tm* t = localtime(&tv.tv_sec);
 61 | 
 62 | 	clock_date_t c; 
 63 | 
 64 | 	c.year		= 1900 + t->tm_year;
 65 | 	c.month		= 1 + t->tm_mon;
 66 | 	c.day		= t->tm_mday;
 67 | 	c.hour		= t->tm_hour;
 68 | 	c.min		= t->tm_min;
 69 | 	c.sec		= t->tm_sec;
 70 | 
 71 | 	return c;
 72 | }
 73 | 
 74 | // 0 - Sunday
 75 | // 1 - Monday 
 76 | // ...
 77 | // http://en.wikipedia.org/wiki/Determination_of_the_day_of_the_week#Implementation-dependent_methods_of_Sakamoto.2C_Lachman.2C_Keith_and_Craver 
 78 | static inline int dayofweek(int d, int m, int y)
 79 | {
 80 |     static int t[] = { 0, 3, 2, 5, 0, 3, 5, 1, 4, 6, 2, 4 };
 81 |     y -= m < 3;
 82 |     return ( y + y/4 - y/100 + y/400 + t[m-1] + d) % 7;
 83 | }
 84 | 
 85 | // generates date in web format RFC1123
 86 | static inline void  clock_rfc1123(u8* Str, clock_date_t c)
 87 | {
 88 | 	const char *DayStr[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
 89 | 	const char *MonthStr[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
 90 | 
 91 | 	struct tm t;
 92 | 	t.tm_year		= c.year - 1900;
 93 | 	t.tm_mon		= c.month - 1;
 94 | 	t.tm_mday		= c.day;
 95 | 	t.tm_hour		= c.hour;
 96 | 	t.tm_min		= c.min;
 97 | 	t.tm_sec		= c.sec;
 98 | 
 99 | 	int wday		= dayofweek(c.day, c.month, c.year);
100 | 
101 |     const int RFC1123_TIME_LEN = 29;
102 | 	strftime(Str, RFC1123_TIME_LEN+1, "---, %d --- %Y %H:%M:%S GMT", &t);
103 | 	memcpy(Str, 	DayStr	[wday], 3);
104 |     memcpy(Str+8, 	MonthStr[c.month - 1], 3);
105 | }
106 | 
107 | static inline void  clock_str(u8* Str, clock_date_t c)
108 | {
109 | 	sprintf(Str, "%04i%02i%02i_%02i-%02i-%02i", c.year, c.month, c.day, c.hour, c.min, c.sec);
110 | }
111 | static inline void  ns_str(u8* Str, u64 NS) 
112 | {
113 | 	u64 sec = NS % k1E9;	
114 | 	int msec = sec / 1000000ULL; 
115 | 	int usec = (sec - msec*1000000ULL)/ 1000ULL; 
116 | 	int nsec = (sec - msec*1000000ULL- usec*1000ULL);
117 | 
118 | 	sprintf(Str, "%03i.%03i.%03i", msec, usec, nsec);
119 | }
120 | 
121 | // epoch nanos -> year, mont, day, ..
122 | static clock_date_t  ns2clock(u64 ts)
123 | {
124 | 	time_t t0 = ts / 1e9;
125 | 
126 | 	struct tm* t = localtime(&t0);
127 | 	clock_date_t c; 
128 | 
129 | 	c.year		= 1900 + t->tm_year;
130 | 	c.month		= 1 + t->tm_mon;
131 | 	c.day		= t->tm_mday;
132 | 	c.hour		= t->tm_hour;
133 | 	c.min		= t->tm_min;
134 | 	c.sec		= t->tm_sec;
135 | 
136 | 	return c;
137 | }
138 | 
139 | // verbose -> nanos since epoch
140 | static u64 clock2ns(int year, int month, int day, int hour, int min, int sec)
141 | {
142 | 	struct tm t;
143 | 
144 | 	t.tm_year 	= year - 1900;
145 | 	t.tm_mon	= month-1;
146 | 	t.tm_mday	= day;
147 | 	t.tm_hour	= hour;
148 | 	t.tm_min	= min;
149 | 	t.tm_sec	= sec;
150 | 
151 | 	time_t epoch = mktime(&t);
152 | 	return (u64)epoch * (u64)1e9; 
153 | }
154 | 
155 | static u64 clock_date2ns(clock_date_t d)
156 | {
157 | 	struct tm t;
158 | 
159 | 	t.tm_year 	= d.year - 1900;
160 | 	t.tm_mon	= d.month-1;
161 | 	t.tm_mday	= d.day;
162 | 	t.tm_hour	= d.hour;
163 | 	t.tm_min	= d.min;
164 | 	t.tm_sec	= d.sec;
165 | 
166 | 	time_t epoch = mktime(&t);
167 | 	return (u64)epoch * (u64)1e9; 
168 | }
169 | 
170 | // returns the first day of the week
171 | static clock_date_t clock_startofweek(clock_date_t d)
172 | {
173 | 	struct tm t;
174 | 
175 | 	int wday		= dayofweek(d.day, d.month, d.year);
176 | 
177 | 	t.tm_year 	= d.year - 1900;
178 | 	t.tm_mon	= d.month-1;
179 | 	t.tm_mday	= d.day - wday;
180 | 	t.tm_hour	= d.hour;
181 | 	t.tm_min	= d.min;
182 | 	t.tm_sec	= d.sec;
183 | 
184 | 	mktime(&t);
185 | 
186 | 	clock_date_t r;
187 | 	r.year		= 1900 + t.tm_year;
188 | 	r.month		= 1 + t.tm_mon;
189 | 	r.day		= t.tm_mday;
190 | 	r.hour		= t.tm_hour;
191 | 	r.min		= t.tm_min;
192 | 	r.sec		= t.tm_sec;
193 | 
194 | 	return r;
195 | }
196 | 
197 | 
198 | // epoch in nanos 
199 | static u64 clock_ns(void)
200 | {
201 | 	struct timeval tv;
202 | 	gettimeofday(&tv, NULL);
203 | 
204 | 	return (u64)tv.tv_sec *(u64)1e9 +(u64)tv.tv_usec * (u64)1e3;
205 | }
206 | 
207 | static inline volatile u64 rdtsc(void)
208 | {
209 | 	u32 hi, lo;
210 | 	__asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi) );
211 | 	return (((u64)hi)<<32ULL) | (u64)lo;
212 | }
213 | 
214 | extern double TSC2Nano;
215 | static inline volatile u64 rdtsc_ns(void)
216 | {
217 | 	u32 hi, lo;
218 | 	__asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi) );
219 | 
220 | 	u64 ts = (((u64)hi)<<32ULL) | (u64)lo;
221 | 	return ts * TSC2Nano; 
222 | }
223 | 
224 | static inline volatile u64 rdtsc2ns(u64 ts)
225 | {
226 | 	return ts * TSC2Nano; 
227 | }
228 | 
229 | static inline volatile u64 tsc2ns(u64 ts)
230 | {
231 | 	return ts * TSC2Nano; 
232 | }
233 | 
234 | static inline u64 ns2tsc(u64 ns)
235 | {
236 | 	return (u64)( (double)ns / TSC2Nano);
237 | }
238 | 
239 | static void ndelay(u64 ns)
240 | {
241 | 	u64 NextTS = rdtsc() + ns2tsc(ns);
242 | 	while (rdtsc() < NextTS)
243 | 	{
244 | 		__asm__ volatile("pause");
245 | 		__asm__ volatile("pause");
246 | 		__asm__ volatile("pause");
247 | 		__asm__ volatile("pause");
248 | 	}
249 | }
250 | 
251 | static inline void prefetchnta(void* ptr)
252 | {
253 | 	__asm__ volatile("prefetchnta (%0)" :  : "r"(ptr));
254 | }
255 | 
256 | static inline u32 swap32(const u32 a)
257 | {
258 | 	return (((a>>24)&0xFF)<<0) | (((a>>16)&0xFF)<<8) | (((a>>8)&0xFF)<<16) | (((a>>0)&0xFF)<<24);
259 | }
260 | 
261 | static inline u16 swap16(const u16 a)
262 | {
263 | 	return (((a>>8)&0xFF)<<0) | (((a>>0)&0xFF)<<8);
264 | }
265 | 
266 | static inline u64 swap64(const u64 a)
267 | {
268 | 	return swap32(a>>32ULL) | ( (u64)swap32(a) << 32ULL); 
269 | }
270 | 
271 | static inline u32 min32(const u32 a, const u32 b)
272 | {
273 | 	return (a < b) ? a : b;
274 | }
275 | 
276 | static inline s32 min32s(const s32 a, const s32 b)
277 | {
278 | 	return (a < b) ? a : b;
279 | }
280 | 
281 | static inline u32 max32(const u32 a, const u32 b)
282 | {
283 | 	return (a > b) ? a : b;
284 | }
285 | static inline s32 max32s(const s32 a, const s32 b)
286 | {
287 | 	return (a > b) ? a : b;
288 | }
289 | 
290 | 
291 | static inline s32 sign32(const s32 a)
292 | {
293 | 	if (a == 0) return 0;
294 | 	return (a > 0) ? 1 : -1;
295 | }
296 | 
297 | static inline u64 min64(const u64 a, const u64 b)
298 | {
299 | 	return (a < b) ? a : b;
300 | }
301 | 
302 | static inline u64 max64(const u64 a, const u64 b)
303 | {
304 | 	return (a > b) ? a : b;
305 | }
306 | 
307 | static inline double maxf(const double a, const double b)
308 | {
309 | 	return (a > b) ? a : b;
310 | }
311 | 
312 | static inline double minf(const double a, const double b)
313 | {
314 | 	return (a < b) ? a : b;
315 | }
316 | static inline double clampf(const double min, const double v, const double max)
317 | {
318 | 	return maxf(min, minf(v,  max)); 
319 | }
320 | 
321 | static inline double inverse(const double a)
322 | {
323 | 	if (a == 0) return 0;
324 | 	return 1.0 / a;
325 | }
326 | 
327 | static inline double fSqrt(const double a)
328 | {
329 | 	if (a <= 0) return 0;
330 | 	return sqrtf(a);
331 | }
332 | 
333 | static inline double signf(const double a)
334 | {
335 | 	if (a > 0) return  1.0;
336 | 	if (a < 0) return -1.0;
337 | 
338 | 	// keep it simple..
339 | 	return 1;
340 | }
341 | 
342 | static inline double alog(const double a)
343 | {
344 | 	if (a == 0) return 0;
345 | 	if (a < 0) return -logf(-a);
346 | 	return -logf(a);
347 | }
348 | 
349 | static inline char* FormatTS(u64 ts)
350 | {
351 | 	u64 usec = ts / 1000ULL;
352 | 	u64 msec = usec / 1000ULL;
353 | 	u64 sec = msec / 1000ULL;
354 | 	u64 min = sec / 60ULL;
355 | 	u64 hour = min / 60ULL;
356 | 
357 | 	u64 nsec = ts - usec*1000ULL;
358 | 	usec = usec - msec*1000ULL;
359 | 	msec = msec - sec*1000ULL;
360 | 	sec = sec - min*60ULL;
361 | 	min = min - hour*60ULL;
362 | 
363 | 	static char List[16][128];
364 | 	static int Pos = 0;
365 | 
366 | 	char* S = List[Pos];
367 | 	Pos = (Pos + 1) & 0xf;
368 | 
369 | 	sprintf(S, "%02lli:%02lli:%02lli.%03lli.%03lli.%03lli", hour % 24, min, sec, msec,usec, nsec);
370 | 
371 | 	return S;
372 | }
373 | 
374 | static inline void CycleCalibration(void)
375 | {
376 |     fprintf(stderr, "calibrating...\n");
377 |     u64 StartTS[16];
378 |     u64 EndTS[16];
379 | 
380 |     u64 CyclesSum   = 0;
381 |     u64 CyclesSum2  = 0;
382 |     u64 CyclesCnt   = 0;
383 |     for (int i=0; i < 1; i++)
384 |     {
385 |         u64 NextTS = clock_ns() + 1e9;
386 |         u64 StartTS = rdtsc();
387 |         while (clock_ns() < NextTS)
388 |         {
389 |         }
390 |         u64 EndTS  = rdtsc();
391 | 
392 |         u64 Cycles = EndTS - StartTS;
393 |         CyclesSum += Cycles;
394 |         CyclesSum2 += Cycles*Cycles;
395 |         CyclesCnt++;
396 | 
397 |         fprintf(stderr, "%i : %016llx %16.4f cycles/nsec\n", i, Cycles, Cycles / 1e9);
398 |     }
399 | 
400 |     double CyclesSec = CyclesSum / CyclesCnt;
401 |     double CyclesStd = sqrt(CyclesCnt *CyclesSum2 - CyclesSum *CyclesSum) / CyclesCnt;
402 |     fprintf(stderr, "Cycles/Sec %12.4f Std:%8.fcycle std(%12.8f)\n", CyclesSec, CyclesStd, CyclesStd / CyclesSec);
403 | 
404 | 	// set global
405 | 
406 | 	TSC2Nano = 1e9 / CyclesSec;
407 | }
408 | // convert pcap style sec : nsec format into pure nano
409 | static inline u64 nsec2ts(u32 sec, u32 nsec)
410 | {
411 | 	return (u64)sec * 1000000000ULL + (u64)nsec;
412 | }
413 | 
414 | // ethernet header
415 | typedef struct
416 | {
417 | 	u8		Dst[6];
418 | 	u8		Src[6];
419 | 	u16		Proto;
420 | 
421 | } fEther_t;
422 | 
423 | typedef struct
424 | {
425 | 	u16			VIDhi	: 4;
426 | 	u16			DEI		: 1;
427 | 	u16			PCP		: 3;
428 | 	u16			VIDlo	: 8;
429 | 
430 | } __attribute__((packed)) VLANTag_t;
431 | #define VLANTag_ID(a) (( a->VIDhi << 8 ) | a->VIDlo )
432 | 
433 | 
434 | 
435 | #define ETHER_PROTO_IPV4		0x0800 
436 | #define ETHER_PROTO_VLAN		0x8100	
437 | typedef struct
438 | {
439 | 	union
440 | 	{
441 | 		u32		IP4;	
442 | 		u8		IP[4];
443 | 	};
444 | 
445 | } IP4_t;
446 | 
447 | typedef struct
448 | {
449 | 	u8		Version;
450 | 	u8		Service;
451 | 	u16		Len;
452 | 	u16		Ident;
453 | 	u16		Frag;
454 | 	u8		TTL;
455 | 	u8		Proto;
456 | 	u16		CSum;
457 | 
458 | 	IP4_t	Src;
459 | 	IP4_t	Dst;
460 | 
461 | } __attribute__((packed)) IP4Header_t;
462 | 
463 | #define IPv4_PROTO_TCP			6
464 | #define IPv4_PROTO_UDP			17	
465 | 
466 | #define TCP_FLAG_SYN(a) ((a >>(8+1))&1)
467 | #define TCP_FLAG_ACK(a) ((a >>(8+4))&1)
468 | #define TCP_FLAG_FIN(a) ((a >>(8+0))&1)
469 | 
470 | typedef struct
471 | {
472 | 	u16			PortSrc;
473 | 	u16			PortDst;
474 | 	u32			SeqNo;
475 | 	u32			AckNo;
476 | 	u16			Flags;
477 | 	u16			Window;
478 | 	u16			CSUM;
479 | 	u16			Urgent;
480 | 
481 | } __attribute__((packed)) TCPHeader_t;
482 | 
483 | typedef struct
484 | {
485 | 	u16			PortSrc;
486 | 	u16			PortDst;
487 | 	u16			Length;
488 | 	u16			CSUM;
489 | 
490 | } __attribute__((packed)) UDPHeader_t;
491 | 
492 | // pcap headers
493 | 
494 | #define PCAPHEADER_MAGIC_NANO		0xa1b23c4d
495 | #define PCAPHEADER_MAGIC_USEC		0xa1b2c3d4
496 | #define PCAPHEADER_MAJOR			2
497 | #define PCAPHEADER_MINOR			4
498 | #define PCAPHEADER_LINK_ETHERNET	1
499 | 
500 | typedef struct
501 | {
502 | 	u32				Sec;				// time stamp sec since epoch 
503 | 	u32				NSec;				// nsec fraction since epoch
504 | 
505 | 	u32				LengthCapture;		// captured length, inc trailing / aligned data
506 | 	u32				Length;				// length on the wire
507 | 
508 | } __attribute__((packed)) PCAPPacket_t;
509 | 
510 | // per file header
511 | 
512 | typedef struct
513 | {
514 | 
515 | 	u32				Magic;
516 | 	u16				Major;
517 | 	u16				Minor;
518 | 	u32				TimeZone;
519 | 	u32				SigFlag;
520 | 	u32				SnapLen;
521 | 	u32				Link;
522 | 
523 | } __attribute__((packed)) PCAPHeader_t;
524 | 
525 | 
526 | // meta mako packet format
527 | typedef struct Metamako_t 
528 | {
529 | 	u32             OrigFCS;            // orignial FCS
530 | 	u32             Sec;                // timestamp secconds     (big endian)
531 | 	u32             NSec;               // timestamp nanoseconds  (big endian)
532 | 	u8              Flag;               // flags
533 | 	                                    // bit 0 : orig FCS is correct
534 | 
535 | 	u16             DeviceID;           // metamako device id
536 | 	u8              PortID;             // metamako port number
537 | 	u32             MMakoFCS;           // packets new FCS generated by mmako
538 | 
539 | } __attribute__((packed)) Metamako_t;
540 | 
541 | #endif
542 | 


--------------------------------------------------------------------------------
/main.c:
--------------------------------------------------------------------------------
   1 | //---------------------------------------------------------------------------------------------
   2 | //
   3 | // Copyright (c) 2015-2019, fmad engineering llc 
   4 | //
   5 | // The MIT License (MIT) see LICENSE file for details 
   6 | // 
   7 | // pcap flow exporter 
   8 | //
   9 | //---------------------------------------------------------------------------------------------
  10 | 
  11 | #include <stdio.h>
  12 | #include <stdint.h>
  13 | #include <stdlib.h>
  14 | #include <unistd.h>
  15 | #include <string.h>
  16 | #include <errno.h>
  17 | #include <sys/stat.h>
  18 | #include <sys/mman.h>
  19 | #include <sys/shm.h>
  20 | #include <fcntl.h>
  21 | #include <locale.h>
  22 | #include <linux/sched.h>
  23 | #include <pthread.h>
  24 | 
  25 | #include "fTypes.h"
  26 | #include "fProfile.h"
  27 | #include "fFile.h"
  28 | #include "tcpstream.h"
  29 | #include "udpstream.h"
  30 | 
  31 | //---------------------------------------------------------------------------------------------
  32 | 
  33 | typedef struct
  34 | {
  35 | 	char* Path;				// path to the file
  36 | 	char	Name[128];		// short name
  37 | 	FILE*	F;				// bufferd io file handle
  38 | 	int		fd;				// file handler of the mmap attached data
  39 | 	u64		Length;			// exact file length
  40 | 	u64		MapLength;		// 4KB aligned mmap length
  41 | 	u8*		Map;			// raw mmap ptr
  42 | 
  43 | 	u64		ReadPos;		// current read pointer
  44 | 	u64		PktCnt;			// number of packets processed
  45 | 
  46 | 	u8*		PacketBuffer;	// temp read buffer
  47 | 	bool	Finished;		// read completed
  48 | 
  49 | 	u64		TS;				// last TS processed
  50 | 
  51 | } PCAPFile_t;
  52 | 
  53 | #define FLOW_TYPE_TCP		1	
  54 | #define FLOW_TYPE_UDP		2	
  55 | 
  56 | typedef struct
  57 | {
  58 | 	u64		Data[64/8];			// protocol specific unique hash
  59 | 
  60 | 	u32		Hash;				// copy of the flow hash
  61 | 	u32		Type;				// what kind of flow is this
  62 | 
  63 | 	u64		PktCnt;				// number of packets in this flow
  64 | 	u64		Bytes;				// number of bytes in this flow
  65 | 	u64		TSFirst;			// first packet of the flow
  66 | 	u64		TSLast;				// last packet of the flow 
  67 | 			
  68 | 								// for duplex matching
  69 | 	u32		TCPSeqNo;			// tcp syn/synack seq no
  70 | 
  71 | 	u32		Next;				// next flow has index for this hash
  72 | 
  73 | } FlowHash_t;
  74 | 
  75 | // specific protocol hash info
  76 | 
  77 | typedef struct
  78 | {
  79 | 	u8		MACSrc[6];
  80 | 	u8		MACDst[6];
  81 | 	IP4_t	IPSrc;
  82 | 	IP4_t	IPDst;
  83 | 
  84 | 	u16		PortSrc;
  85 | 	u16		PortDst;
  86 | 
  87 | 	u16		DeviceID;			// packet broker device ID 
  88 | 	u16		DevicePort;			// packet broker device port 
  89 | 
  90 | } __attribute__((packed)) TCPHash_t;
  91 | 
  92 | typedef struct
  93 | {
  94 | 	u8		MACSrc[6];
  95 | 	u8		MACDst[6];
  96 | 	IP4_t	IPSrc;
  97 | 	IP4_t	IPDst;
  98 | 
  99 | 	u16		PortSrc;
 100 | 	u16		PortDst;
 101 | 
 102 | 	u16		DeviceID;			// packet broker device ID 
 103 | 	u16		DevicePort;			// packet broker device port 
 104 | 
 105 | } UDPHash_t;
 106 | 
 107 | double TSC2Nano = 0;
 108 | 
 109 | //---------------------------------------------------------------------------------------------
 110 | // tunables
 111 | static u64		s_MaxPackets			= (1ULL<<63);			// max number of packets to process
 112 | static bool		s_TimeZoneEnable		= true;					// enable timezone adjustment by default
 113 | static s64		s_TimeZoneOffset		= 0;					// local timezone
 114 | u64				g_TotalMemory			= 0;					// total memory consumption
 115 | u64				g_TotalMemoryTCP		= 0;					// total memory of keeping out of order tcp packets 
 116 | bool			g_Verbose				= false;				// verbose print mode
 117 | 
 118 | //---------------------------------------------------------------------------------------------
 119 | // first level index 
 120 | 
 121 | static u32*					s_FlowIndex;						// 24b index into first level list 
 122 | static u64					s_FlowIndexBits = 24;				// bit depth of the hash index, default to 24b / 64MB
 123 | static u64					s_FlowIndexMask;					// bit mask for for the hash depth 
 124 | static u32					s_FlowIndexDepthMax;				// current max depth of an index 
 125 | static u64					s_FlowIndexDepthS0;					// calculate mean depth 
 126 | static u64					s_FlowIndexDepthS1;					// calculate mean depth 
 127 | 
 128 | static FlowHash_t*			s_FlowList;							// statically allocated max number of flows 
 129 | static u32					s_FlowListPos = 1;					// current allocated flow
 130 | static u32					s_FlowListMax;						// max number of flows
 131 | 
 132 | static u64					s_FlowListPacketMin 	= 0;		// minimum number of packets to show entry for
 133 | 
 134 | static u8*					s_FlowExtract 			= NULL;		// boolean to extract the specified flow id
 135 | static bool					s_FlowExtractEnable 	= false;	// indicaes flow extraction 
 136 | 
 137 | static u32					s_ExtractTCPEnable 		= false;	// request extraction of tcp stream
 138 | static u8*					s_ExtractTCPFlow		= NULL;		// boolean to extract the specified flow id
 139 | 
 140 | static bool					s_ExtractTCPPortEnable 	= false;	// extract all tcp flows with the specified port number
 141 | static u32					s_ExtractTCPPortMin		= 0;
 142 | static u32					s_ExtractTCPPortMax		= 0;
 143 | static struct TCPStream_t** s_ExtractTCP			= NULL; 	// list of tcp stream extraction objects 
 144 | 
 145 | static u32					s_DisableTCPPortCnt 	= 0;		// extract all tcp flows with the specified port number
 146 | static u32					s_DisableTCPPortCntMax 	= 32;		// max list of port ranges 
 147 | static u32					s_DisableTCPPortMin[32];
 148 | static u32					s_DisableTCPPortMax[32];
 149 | 
 150 | static bool					s_ExtractUDPPortEnable 	= false;	// extract all UDP flows within the specified port range 
 151 | static u32					s_ExtractUDPPortMin		= 0;
 152 | static u32					s_ExtractUDPPortMax		= 0;
 153 | static struct UDPStream_t**	s_ExtractUDP			= NULL;
 154 | 
 155 | static u32					s_DisableUDPPortCnt 	= 0;		// extract all tcp flows with the specified port number
 156 | static u32					s_DisableUDPPortCntMax 	= 32;		// max list of port ranges 
 157 | static u32					s_DisableUDPPortMin[32];
 158 | static u32					s_DisableUDPPortMax[32];
 159 | 
 160 | static bool					s_ExtractIPEnable		= false;	// extract an IP range into a seperate pcap file
 161 | static u32					s_ExtractIPMask			= 0;		// /32 mask
 162 | static u32					s_ExtractIPMatch		= 0;		// match 
 163 | 
 164 | static bool					s_ExtractPortEnable		= false;	// extract TCP/UDP port range to a seperate pcap file 
 165 | static u32					s_ExtractPortMin		= 0;		// min port range 
 166 | static u32					s_ExtractPortMax		= 0;		// max port range 
 167 | 
 168 | static bool					s_EnableFlowDisplay 	= true;		// print full flow information
 169 | bool						g_EnableTCPHeader 		= false;	// output packet header in tcp stream
 170 | bool						g_EnableUDPHeader 		= false;	// output packet header in udp stream
 171 | 
 172 | static bool					s_EnableFlowLog			= true;		// write flow log in realtime
 173 | static char					s_FlowLogPath[128];					// where to store the flow log
 174 | static FILE*				s_FlowLogFile			= NULL;		// file handle where to write flows
 175 | 
 176 | static u64					s_PCAPTimeScale			= 1;		// timescale all raw pcap time stamps
 177 | 
 178 | bool						g_EnableMetamako		= false;	// enable metamako timestamp decoding 
 179 | bool						g_EnableMetamako2T		= false;	// double metamako tags 
 180 | s32							g_MetamakoOffset		= 0;		// bugix for old stream_cat 
 181 | bool						g_EnableVLAN			= false;	// enable VLAN de-encapsulation		
 182 | 
 183 | static bool					s_Flow5Tuple			= false;	// use 5-tuple only flow caluclation mode
 184 | 
 185 | bool						g_OutputPCAP			= false;	// output streams in pcap format
 186 | 
 187 | //---------------------------------------------------------------------------------------------
 188 | 
 189 | void sha1_compress(uint32_t state[static 5], const uint8_t block[static 64]);
 190 | 
 191 | //---------------------------------------------------------------------------------------------
 192 | // mmaps a pcap file in full
 193 | static PCAPFile_t* OpenPCAP(char* Path, bool EnableStdin)
 194 | {
 195 | 	PCAPFile_t* F = (PCAPFile_t*)malloc( sizeof(PCAPFile_t) );
 196 | 	assert(F != NULL);
 197 | 	memset(F, 0, sizeof(PCAPFile_t));
 198 | 	F->Path		= Path;
 199 | 
 200 | 	if (!EnableStdin)
 201 | 	{
 202 | 		F->F = fopen(Path, "r");
 203 | 		if (F->F == NULL)
 204 | 		{
 205 | 			fprintf(stderr, "failed to open buffered file [%s]\n", Path);
 206 | 			return NULL;
 207 | 		}
 208 | 
 209 | 		struct stat fstat;	
 210 | 		if (stat(Path, &fstat) < 0)
 211 | 		{
 212 | 			fprintf(stderr, "failed to get file size [%s]\n", Path);
 213 | 			return NULL;
 214 | 		}
 215 | 		F->Length 	= fstat.st_size;
 216 | 	}
 217 | 	else
 218 | 	{
 219 | 		F->F 		= stdin;
 220 | 		F->Length 	= 1e15;		// 1PB limit
 221 | 	}
 222 | 
 223 | 	// note always map as read-only 
 224 | 	PCAPHeader_t Header1;
 225 | 	PCAPHeader_t* Header = NULL; 
 226 | 	{
 227 | 		int ret = fread(&Header1, 1, sizeof(Header1), F->F);
 228 | 		if (ret != sizeof(PCAPHeader_t))
 229 | 		{
 230 | 			fprintf(stderr, "failed to read header %i\n", ret);
 231 | 			return NULL;
 232 | 		}
 233 | 
 234 | 		Header = &Header1;
 235 | 		F->PacketBuffer	= malloc(32*1024);
 236 | 		assert(F->PacketBuffer != NULL);
 237 | 	}
 238 | 
 239 | 	switch (Header->Magic)
 240 | 	{
 241 | 	case PCAPHEADER_MAGIC_USEC: fprintf(stderr, "USec PACP\n"); s_PCAPTimeScale = 1000; break;
 242 | 	case PCAPHEADER_MAGIC_NANO: fprintf(stderr, "Nano PACP\n"); s_PCAPTimeScale = 1; break;
 243 | 	default:
 244 | 		fprintf(stderr, "invalid pcap header %08x\n", Header->Magic);
 245 | 		return NULL;
 246 | 	}
 247 | 	F->ReadPos +=  sizeof(PCAPHeader_t);
 248 | 
 249 | 	return F;
 250 | }
 251 | 
 252 | //---------------------------------------------------------------------------------------------
 253 | // get the next packet
 254 | static PCAPPacket_t* ReadPCAP(PCAPFile_t* PCAP)
 255 | {
 256 | 	int ret;
 257 | 	PCAPPacket_t* Pkt = (PCAPPacket_t*)PCAP->PacketBuffer;
 258 | 	ret = fread(Pkt, 1, sizeof(PCAPPacket_t), PCAP->F);
 259 | 	if (ret != sizeof(PCAPPacket_t))
 260 | 	{
 261 | 		fprintf(stderr, "header read failed. expect:%li got:%i errno:%i %s\n", sizeof(PCAPPacket_t), ret, errno, strerror(errno));
 262 | 		fprintf(stderr, "errno: %i\n", ferror(PCAP->F));
 263 | 		return NULL;
 264 | 	}
 265 | 
 266 | 	if (PCAP->ReadPos + sizeof(PCAPPacket_t) + Pkt->LengthCapture > PCAP->Length)
 267 | 	{
 268 | 		fprintf(stderr, "read overflow %lli %li %i > %lli\n", 
 269 | 				PCAP->ReadPos,
 270 | 				sizeof(PCAPPacket_t),
 271 | 				Pkt->LengthCapture,
 272 | 				PCAP->Length
 273 | 		);	
 274 | 		return NULL; 
 275 | 	}
 276 | 
 277 | 	ret = fread(Pkt+1, 1, Pkt->LengthCapture, PCAP->F);
 278 | 	if (ret != Pkt->LengthCapture)
 279 | 	{
 280 | 		fprintf(stderr, "payload read failed. expect:%li got:%i errno:%i %s\n", sizeof(PCAPPacket_t), ret, errno, strerror(errno));
 281 | 		fprintf(stderr, "errno: %i\n", ferror(PCAP->F));
 282 | 		return NULL;
 283 | 	}
 284 | 
 285 | 	PCAP->ReadPos += Pkt->LengthCapture;
 286 | 	return Pkt;
 287 | }
 288 | 
 289 | //---------------------------------------------------------------------------------------------
 290 | // helpers for network formating 
 291 | static u64 PCAPTimeStamp(PCAPPacket_t* Pkt)
 292 | {
 293 | 	return s_TimeZoneOffset + Pkt->Sec * k1E9 + Pkt->NSec * s_PCAPTimeScale;
 294 | }
 295 | static fEther_t * PCAPETHHeader(PCAPPacket_t* Pkt)
 296 | {
 297 | 	fEther_t* E = (fEther_t*)(Pkt+1);	
 298 | 	return E;
 299 | }
 300 | 
 301 | static IP4Header_t* PCAPIP4Header(PCAPPacket_t* Pkt)
 302 | {
 303 | 	fEther_t* E = (fEther_t*)(Pkt+1);	
 304 | 
 305 | 	IP4Header_t* IP4 = (IP4Header_t*)(E + 1);
 306 | 	u32 IPOffset = (IP4->Version & 0x0f)*4; 
 307 | 
 308 | 	return IP4;
 309 | }
 310 | 
 311 | static TCPHeader_t* PCAPTCPHeader(PCAPPacket_t* Pkt)
 312 | {
 313 | 	fEther_t* E = (fEther_t*)(Pkt+1);	
 314 | 
 315 | 	IP4Header_t* IP4 = (IP4Header_t*)(E + 1);
 316 | 	u32 IPOffset = (IP4->Version & 0x0f)*4; 
 317 | 
 318 | 	TCPHeader_t* TCP = (TCPHeader_t*)( ((u8*)IP4) + IPOffset);
 319 | 	u32 TCPOffset = ((TCP->Flags&0xf0)>>4)*4;
 320 | 
 321 | 	return TCP;
 322 | }
 323 | 
 324 | static u8* PCAPTCPPayload(PCAPPacket_t* Pkt, s32* Length)
 325 | {
 326 | 	fEther_t* E = (fEther_t*)(Pkt+1);	
 327 | 
 328 | 	IP4Header_t* IP4 = (IP4Header_t*)(E + 1);
 329 | 	u32 IPOffset = (IP4->Version & 0x0f)*4; 
 330 | 
 331 | 	TCPHeader_t* TCP = (TCPHeader_t*)( ((u8*)IP4) + IPOffset);
 332 | 	u32 TCPOffset = ((TCP->Flags&0xf0)>>4)*4;
 333 | 
 334 | 	Length[0] = swap16(IP4->Len) - IPOffset - TCPOffset;
 335 | 
 336 | 	return (u8*)TCP + TCPOffset;
 337 | }
 338 | 
 339 | static UDPHeader_t* PCAPUDPHeader(PCAPPacket_t* Pkt)
 340 | {
 341 | 	fEther_t* E = (fEther_t*)(Pkt+1);	
 342 | 
 343 | 	IP4Header_t* IP4 = (IP4Header_t*)(E + 1);
 344 | 	u32 IPOffset = (IP4->Version & 0x0f)*4; 
 345 | 
 346 | 	UDPHeader_t* UDP = (UDPHeader_t*)( ((u8*)IP4) + IPOffset);
 347 | 
 348 | 	return UDP;
 349 | }
 350 | 
 351 | static Metamako_t* PCAPMetamako(PCAPPacket_t* Pkt)
 352 | {
 353 | 	u8* Payload = (u8*)(Pkt+1);	
 354 | 
 355 | 	s32 Offset = Pkt->LengthCapture;
 356 | 
 357 | 	Offset -= sizeof(Metamako_t);
 358 | 
 359 | 	// double tagged
 360 | 	if (g_EnableMetamako2T)
 361 | 	{
 362 | 		Offset -= sizeof(Metamako_t) - 4;  
 363 | 	}
 364 | 
 365 | // temp work around
 366 | // stream_cat isnt adjusting the capture length payload size
 367 | // when stripping vlans
 368 | //Offset -= 4; 
 369 | Offset -= g_MetamakoOffset; 
 370 | 
 371 | 	// required if pcap has no FCS
 372 | 	//Offset += 4; 
 373 | 	if (Offset < 0)
 374 | 	{
 375 | 		printf("pkt length %i %i\n", Pkt->Length, Pkt->LengthCapture);
 376 | 	}
 377 | 
 378 |  	assert(Offset > 0);	
 379 | 
 380 | 	Metamako_t* M = (Metamako_t*)(Payload + Offset); 
 381 | 
 382 | 	return M;
 383 | }
 384 | 
 385 | //---------------------------------------------------------------------------------------------
 386 | 
 387 | static void PrintMAC(FILE* Out, u8* MAC)
 388 | {
 389 | 	fprintf(Out, "%02x:%02x:%02x:%02x:%02x:%02x",
 390 | 			MAC[0],
 391 | 			MAC[1],
 392 | 			MAC[2],
 393 | 			MAC[3],
 394 | 			MAC[4],
 395 | 			MAC[5]
 396 | 		  );
 397 | }
 398 | 
 399 | static void PrintIP4(FILE* Out, IP4_t IP)
 400 | {
 401 | 	fprintf(Out, "%3i.%3i.%3i.%3i", IP.IP[0], IP.IP[1], IP.IP[2], IP.IP[3]); 
 402 | }
 403 | 
 404 | //---------------------------------------------------------------------------------------------
 405 | 
 406 | static void PrintFlowTCP(FILE* Out, FlowHash_t* F, u32 FlowID, u32 FlowCnt)
 407 | {
 408 | 	TCPHash_t* TCP = (TCPHash_t*)F->Data;
 409 | 	fprintf(Out, "%5i FlowID: %8i | TCP  ", FlowCnt, FlowID); 	
 410 | 	PrintMAC(Out, TCP->MACSrc);
 411 | 	fprintf(Out, " -> ");
 412 | 	PrintMAC(Out, TCP->MACDst);
 413 | 
 414 | 	fprintf(Out, " | ");
 415 | 	PrintIP4(Out, TCP->IPSrc);
 416 | 	fprintf(Out, " -> ");
 417 | 	PrintIP4(Out, TCP->IPDst);
 418 | 
 419 | 	fprintf(Out, " | %6i -> %6i ", TCP->PortSrc, TCP->PortDst);
 420 | 
 421 | 	fprintf(Out, " | ");
 422 | 	fprintf(Out, " %'16lld Pkts ", F->PktCnt);
 423 | 	fprintf(Out, " %'16lli Bytes ", F->Bytes);
 424 | 
 425 | 	fprintf(Out, " | ");
 426 | 	fprintf(Out, " %s -> %s", FormatTS(F->TSFirst), FormatTS(F->TSLast) ); 
 427 | 	fprintf(Out, " : %s", FormatTS(F->TSLast - F->TSFirst));
 428 | 
 429 | 	fprintf(Out, " | ");
 430 | 	fprintf(Out, " Seq:%08x", F->TCPSeqNo);
 431 | 
 432 | 	// generate TCP specific stats
 433 | 	struct TCPStream_t* Stream = s_ExtractTCP[FlowID];
 434 | 	fTCPStream_FlowStats(Stream, Out);
 435 | 
 436 | 	fprintf(Out, "\n");
 437 | }
 438 | 
 439 | //---------------------------------------------------------------------------------------------
 440 | 
 441 | static void PrintFlowUDP(FILE* Out, FlowHash_t* F, u32 FlowID, u32 FlowCnt)
 442 | {
 443 | 
 444 | 	UDPHash_t* UDP = (UDPHash_t*)F->Data;
 445 | 	fprintf(Out, "%5i FlowID: %8i | UDP  ", FlowCnt, FlowID); 	
 446 | 	PrintMAC(Out, UDP->MACSrc);
 447 | 	fprintf(Out, " -> ");
 448 | 	PrintMAC(Out, UDP->MACDst);
 449 | 
 450 | 	fprintf(Out, " | ");
 451 | 	PrintIP4(Out, UDP->IPSrc);
 452 | 	fprintf(Out, " -> ");
 453 | 	PrintIP4(Out, UDP->IPDst);
 454 | 
 455 | 	fprintf(Out, " | %6i -> %6i ", UDP->PortSrc, UDP->PortDst);
 456 | 
 457 | 	fprintf(Out, " | ");
 458 | 	fprintf(Out, " %'16lld Pkts ", F->PktCnt);
 459 | 	fprintf(Out, " %'16lli Bytes ", F->Bytes);
 460 | 
 461 | 	fprintf(Out, " | ");
 462 | 	fprintf(Out, " %s -> %s", FormatTS(F->TSFirst), FormatTS(F->TSLast) ); 
 463 | 	fprintf(Out, " : %s", FormatTS(F->TSLast - F->TSFirst));
 464 | 
 465 | 	fprintf(Out, "\n");
 466 | }
 467 | 
 468 | //---------------------------------------------------------------------------------------------
 469 | 
 470 | static u32 FlowHash(u32 Type, u8* Payload, u32 Length)
 471 | {
 472 | 	// generate SHA1
 473 | 	u32 SHA1State[5] = { 0, 0, 0, 0, 0 };
 474 | 
 475 | 	// hash the first 64B
 476 | 	sha1_compress(SHA1State, (u8*)Payload);
 477 | 
 478 | 	u8* Data8 = (u8*)SHA1State;
 479 | 
 480 | 	// FNV1a 80b hash 
 481 | 	const u32 Prime  = 0x01000193; //   16777619
 482 | 	const u32  Seed  = 0x811C9DC5; // 2166136261
 483 | 
 484 | 	u32 Hash = Seed;
 485 | 	Hash = ((u32)Data8[ 0] ^ Hash) * Prime;
 486 | 	Hash = ((u32)Data8[ 1] ^ Hash) * Prime;
 487 | 	Hash = ((u32)Data8[ 2] ^ Hash) * Prime;
 488 | 	Hash = ((u32)Data8[ 3] ^ Hash) * Prime;
 489 | 
 490 | 	Hash = ((u32)Data8[ 4] ^ Hash) * Prime;
 491 | 	Hash = ((u32)Data8[ 5] ^ Hash) * Prime;
 492 | 	Hash = ((u32)Data8[ 6] ^ Hash) * Prime;
 493 | 	Hash = ((u32)Data8[ 7] ^ Hash) * Prime;
 494 | 
 495 | 	Hash = ((u32)Data8[ 8] ^ Hash) * Prime;
 496 | 	Hash = ((u32)Data8[ 9] ^ Hash) * Prime;
 497 | 	Hash = ((u32)Data8[10] ^ Hash) * Prime;
 498 | 	Hash = ((u32)Data8[11] ^ Hash) * Prime;
 499 | 
 500 | 	Hash = ((u32)Data8[12] ^ Hash) * Prime;
 501 | 	Hash = ((u32)Data8[13] ^ Hash) * Prime;
 502 | 	Hash = ((u32)Data8[14] ^ Hash) * Prime;
 503 | 	Hash = ((u32)Data8[15] ^ Hash) * Prime;
 504 | 
 505 | 	Hash = ((u32)Data8[16] ^ Hash) * Prime;
 506 | 	Hash = ((u32)Data8[17] ^ Hash) * Prime;
 507 | 	Hash = ((u32)Data8[18] ^ Hash) * Prime;
 508 | 	Hash = ((u32)Data8[19] ^ Hash) * Prime;
 509 | 
 510 | 	/*
 511 | 	// DEK packets usually have enough entropy for this to be enough 
 512 | 	u32 Hash = Type; 
 513 | 	for (int i=0; i < Length; i++)
 514 | 	{
 515 | 		Hash = ((Hash << 5ULL) ^ (Hash >> (32-5))) ^ (u64)Payload[i];
 516 | 	}
 517 | 	*/
 518 | 
 519 | 	// reduce to a 32b hash
 520 | 	return Hash;
 521 | }
 522 | 
 523 | //---------------------------------------------------------------------------------------------
 524 | 
 525 | static u32 FlowAdd(FlowHash_t* Flow, u32 PktLength, u64 TS) 
 526 | {
 527 | 	if (s_FlowListPos >= s_FlowListMax) return 0;
 528 | 
 529 | 	FlowHash_t* F 	= NULL; 
 530 | 
 531 | 	// first level has is 24b index, followed by list of leaf nodes
 532 | 	Flow->Hash 		= FlowHash(Flow->Type, (u8*)Flow->Data, 64);
 533 | 	u32 Index 		= Flow->Hash & s_FlowIndexMask;
 534 | 
 535 | 	u32 FlowIndex 	= 0;
 536 | 	bool IsFlowNew 	= false;
 537 | 
 538 | 	u32 HashDepth	= 0;
 539 | 
 540 | 	if (s_FlowIndex[Index] != 0)
 541 | 	{
 542 | 		F = s_FlowList +  s_FlowIndex[Index];
 543 | 		bool Found = false;
 544 | 		for (int t=0; t < 1e6; t++)
 545 | 		{
 546 | 			HashDepth++;
 547 | 
 548 | 			// flow matched
 549 | 			if (memcmp(F->Data, Flow->Data, 64) == 0)
 550 | 			{
 551 | 				Found = true;
 552 | 				break;
 553 | 			}
 554 | 
 555 | 			if (F->Next == 0) break;
 556 | 			F = s_FlowList + F->Next;
 557 | 			assert(t < 99e3);
 558 | 		}
 559 | 
 560 | 		// keep stats on the max depth
 561 | 		if (s_FlowIndexDepthMax  < HashDepth)
 562 | 		{
 563 | 			fprintf(stderr, "Hash New Max Depth: %i : %08x : ", HashDepth, Flow->Hash); 
 564 | 			for (int i=0; i < 64/8; i++) fprintf(stderr, "%016llx ", Flow->Data[i]);
 565 | 			fprintf(stderr, "\n");
 566 | 
 567 | 			s_FlowIndexDepthMax = HashDepth;
 568 | 		}
 569 | 		s_FlowIndexDepthS0	+= 1; 
 570 | 		s_FlowIndexDepthS1	+= HashDepth; 
 571 | 
 572 | 		// new flow
 573 | 		if (Found)
 574 | 		{
 575 | 		}
 576 | 		else
 577 | 		{
 578 | 			F = &s_FlowList[ s_FlowListPos++ ];
 579 | 			assert(s_FlowListPos < s_FlowListMax);
 580 | 
 581 | 			memcpy(F, Flow, sizeof(FlowHash_t));
 582 | 			F->Next = s_FlowIndex[Index];
 583 | 
 584 | 			s_FlowIndex[Index] = F - s_FlowList;
 585 | 
 586 | 			IsFlowNew = true; 
 587 | 		}
 588 | 	}
 589 | 	else
 590 | 	{
 591 | 		F = &s_FlowList[ s_FlowListPos++ ];
 592 | 		assert(s_FlowListPos < s_FlowListMax);
 593 | 
 594 | 		memcpy(F, Flow, sizeof(FlowHash_t));
 595 | 		F->Next = 0;
 596 | 
 597 | 		s_FlowIndex[Index] = F - s_FlowList; 
 598 | 		IsFlowNew = true; 
 599 | 	}
 600 | 
 601 | 	// update stats
 602 | 
 603 | 	F->PktCnt++;
 604 | 	F->Bytes += PktLength;
 605 | 	F->TSFirst	= IsFlowNew ? TS : F->TSFirst;
 606 | 	F->TSLast	= TS;
 607 | 
 608 | 	// update flow log
 609 | 	if (IsFlowNew && s_EnableFlowLog)
 610 | 	{
 611 | 		u32 ID = F - s_FlowList;
 612 | 		switch (F->Type)
 613 | 		{
 614 | 		case FLOW_TYPE_TCP: PrintFlowTCP(s_FlowLogFile, F, ID, s_FlowListPos); break;
 615 | 		case FLOW_TYPE_UDP: PrintFlowUDP(s_FlowLogFile, F, ID, s_FlowListPos); break;
 616 | 		}
 617 | 	}
 618 | 	return F - s_FlowList;
 619 | }
 620 | 
 621 | //---------------------------------------------------------------------------------------------
 622 | 
 623 | static void PrintHumanFlows(void)
 624 | {
 625 | 	u64 PktMax = 0;
 626 | 	/*
 627 | 	for (int i=1; i < s_FlowListPos; i++)
 628 | 	{
 629 | 		FlowHash_t* F = &s_FlowList[i];
 630 | 		if (PktMax < F->PktCnt) PktMax = F->PktCnt;
 631 | 	}
 632 | 	*/
 633 | 
 634 | 	u32 FlowCnt = 0;
 635 | 	s32 Remain = s_FlowListPos-1;
 636 | 	while (Remain > 0)
 637 | 	{
 638 | 		u64 NextMax = 1e16;
 639 | 		for (int i=1; i < s_FlowListPos; i++)
 640 | 		{
 641 | 			FlowHash_t* F = &s_FlowList[i];
 642 | 			if (F->PktCnt == PktMax)
 643 | 			{
 644 | 				if (F->PktCnt >= s_FlowListPacketMin)
 645 | 				{
 646 | 					switch (F->Type)
 647 | 					{
 648 | 					case FLOW_TYPE_TCP: PrintFlowTCP(stdout, F, i, FlowCnt); break;
 649 | 					case FLOW_TYPE_UDP: PrintFlowUDP(stdout, F, i, FlowCnt); break;
 650 | 					}
 651 | 				}
 652 | 				
 653 | 				Remain--;
 654 | 				FlowCnt++;
 655 | 				assert(Remain >= 0);
 656 | 			}
 657 | 			else if (F->PktCnt > PktMax)
 658 | 			{
 659 | 				if (NextMax > F->PktCnt) NextMax = F->PktCnt;
 660 | 			}
 661 | 		}
 662 | 
 663 | 		//printf("%i -> %i : %i\n", PktMax, NextMax, Remain);
 664 | 		PktMax = NextMax;
 665 | 	}
 666 | }
 667 | 
 668 | //---------------------------------------------------------------------------------------------
 669 | 
 670 | static void print_usage(void)
 671 | {
 672 | 	fprintf(stderr, "pcap_flows: <pcap>>\n");
 673 | 	fprintf(stderr, "\n");
 674 | 	fprintf(stderr, "Version: %s %s\n", __DATE__, __TIME__);
 675 | 	fprintf(stderr, "Contact: support at fmad.io\n"); 
 676 | 	fprintf(stderr, "\n");
 677 | 	fprintf(stderr, "Options:\n");
 678 | 	fprintf(stderr, "  --output-tcp <filename>                  | write TCP output to the specified file name\n");
 679 | 	fprintf(stderr, "  --output-udp <filename>                  | write UDP output to the specified file name\n");
 680 | 	fprintf(stderr, "\n");
 681 | 	fprintf(stderr, "  --packet-max  <number>                   | only process the first <number> packets\n");
 682 | 	fprintf(stderr, "  --flow-max  <number>                     | sets max flow count to <number> packets\n");
 683 | 	fprintf(stderr, "  --flow-hash-bits  <number>               | sets number of bits to use for the flow hash index\n");
 684 | 	fprintf(stderr, "  --extract <number>                       | extract FlowID <number> into the output PCAP file\n");
 685 | 	fprintf(stderr, "  --extract-port <min port>  <max port>    | extract ports between min/max\n");
 686 | 	fprintf(stderr, "  --extract-ip <address/netmask>           | extract only a subnet\n");
 687 | 	fprintf(stderr, "  --extract-tcp <number>                   | extract FlowID <number> as a TCP stream to the output file name\n"); 
 688 | 	fprintf(stderr, "  --extract-tcp-port <min port> <max port> | extract all TCP flows with the specified port in src or dest\n");
 689 | 	fprintf(stderr, "  --extract-tcp-all                        | extract all TCP flows\n");
 690 | 	fprintf(stderr, "  --disable-tcp-port <min port> <max port> | do not extract TCP ports within this range\n");
 691 | 	fprintf(stderr, "  --stdin                                  | read pcap from stdin. e.g. zcat capture.pcap | pcap_flow --stdin\n"); 
 692 | 	fprintf(stderr, "  --flow-packet-min <number>               | minimum packet count to display flow info\n"); 
 693 | 	fprintf(stderr, "  --disable-display                        | do not display flow information to stdout\n");
 694 | 	fprintf(stderr, "  --cpu <number>                           | pin thread to a specific CPU\n"); 
 695 | 	fprintf(stderr, "  --flow-size-min <bytes>                  | minium file size to flow creation\n"); 
 696 | 	fprintf(stderr, "  --metamako                               | decode metamako footer\n"); 
 697 | 	fprintf(stderr, "  --metamako-double                        | decode double tagged metamako footer\n"); 
 698 | 	fprintf(stderr, "  --metamako-offset <bytes>                | manual offset for metamako pcap footer\n"); 
 699 | 	fprintf(stderr, "  --tcpheader                              | include TCP header in output\n"); 
 700 | 	fprintf(stderr, "  --udpheader                              | include UDP header in output\n"); 
 701 | 	fprintf(stderr, "\n");
 702 | }
 703 | 
 704 | //---------------------------------------------------------------------------------------------
 705 | static void FlowAlloc(u32 FlowMax)
 706 | {
 707 | 	s_FlowListMax 		= FlowMax;
 708 | 
 709 | 	if (s_FlowExtract)		free(s_FlowExtract);
 710 | 	if (s_ExtractTCPFlow)	free(s_ExtractTCPFlow);
 711 | 	if (s_ExtractTCP) 		free(s_ExtractTCP);
 712 | 	if (s_ExtractUDP) 		free(s_ExtractUDP);
 713 | 
 714 | 	s_FlowExtract 		= (u8*)malloc( s_FlowListMax * sizeof(u8) );
 715 | 	s_ExtractTCPFlow 	= (u8*)malloc( s_FlowListMax * sizeof(u8) );
 716 | 	s_ExtractTCP	 	= (struct TCPStream_t**)malloc( s_FlowListMax * sizeof(void*) );
 717 | 	s_ExtractUDP	 	= (struct UDPStream_t**)malloc( s_FlowListMax * sizeof(void*) );
 718 | 
 719 | 	memset(s_FlowExtract,		0, s_FlowListMax * sizeof(u8) );
 720 | 	memset(s_ExtractTCPFlow,	0, s_FlowListMax * sizeof(u8) );
 721 | 	memset(s_ExtractTCP, 		0, s_FlowListMax * sizeof(void*) );
 722 | 	memset(s_ExtractUDP, 		0, s_FlowListMax * sizeof(void*) );
 723 | }
 724 | 
 725 | //---------------------------------------------------------------------------------------------
 726 | 
 727 | int main(int argc, char* argv[])
 728 | {
 729 | 	int 	FileNameListPos = 0;
 730 | 	char 	FileNameList[16][256];
 731 | 	int		FileStdin = false;
 732 | 
 733 | 	char* 	UDPOutputFileName = NULL;
 734 | 	char* 	TCPOutputFileName = NULL;
 735 | 
 736 | 	// allocate flow lists
 737 | 	FlowAlloc(100e3);							// default flow count
 738 | 
 739 | 	for (int i=1; i < argc; i++)
 740 | 	{
 741 | 		fprintf(stderr, "[%s]\n", argv[i]);
 742 | 		if (argv[i][0] != '-')
 743 | 		{
 744 | 			strcpy(FileNameList[FileNameListPos], argv[i]);
 745 | 			FileNameListPos++;
 746 | 		}
 747 | 		else
 748 | 		{
 749 | 			if (strcmp(argv[i], "--packet-max") == 0)
 750 | 			{
 751 | 				s_MaxPackets = atoll(argv[i+1]); 
 752 | 				i+= 1;
 753 | 				fprintf(stderr, "    setting maximum number of packets to %lli\n", s_MaxPackets);
 754 | 			}
 755 | 			// set the maximum number of flows
 756 | 			else if (strcmp(argv[i], "--flow-max") == 0)
 757 | 			{
 758 | 				u32 FlowMax = (u32)atof(argv[i+1]); 
 759 | 				i++;
 760 | 				fprintf(stderr, "    set max flow count to %i\n", FlowMax);
 761 | 				FlowAlloc(FlowMax);							// default flow count
 762 | 			}
 763 | 			// set the number of bits for the hash index 
 764 | 			else if (strcmp(argv[i], "--flow-hash-bits") == 0)
 765 | 			{
 766 | 				u32 HashBits = (u32)atoi(argv[i+1]); 
 767 | 				i++;
 768 | 				fprintf(stderr, "    set Hash Bit Depth to %i\n", HashBits);
 769 | 				s_FlowIndexBits = HashBits;
 770 | 			}
 771 | 
 772 | 			else if (strcmp(argv[i], "--extract") == 0)
 773 | 			{
 774 | 				u32 FlowID = atoi(argv[i+1]); 
 775 | 				if (FlowID >= sizeof(s_FlowExtract))
 776 | 				{
 777 | 					fprintf(stderr, "    flow overflow\n");
 778 | 					return 0;
 779 | 				}
 780 | 				s_FlowExtract[ FlowID ] = 1<<7;
 781 | 				s_FlowExtractEnable 	= true;
 782 | 				i++;
 783 | 			}
 784 | 			// extract the specified ip range into a seperate pcap 
 785 | 			else if (strcmp(argv[i], "--extract-ip") == 0)
 786 | 			{
 787 | 				// in the form of 192.168.1.1/255.255.255.255
 788 | 				char* IPRange 		= argv[i+1];
 789 | 				i++;
 790 | 
 791 | 				u8 Segment[8][256];
 792 | 				u32 SegmentPos = 0;
 793 | 				u32 SegmentLen = 0;
 794 | 				for (int p=0; p < strlen(IPRange); p++)
 795 | 				{
 796 | 					u8 c = IPRange[p];
 797 | 					if ((c == '.') || (c == '/'))
 798 | 					{
 799 | 						Segment[SegmentPos][SegmentLen] = 0;
 800 | 						//printf("seg len: %i %i : %s\n", SegmentPos, SegmentLen, Segment[SegmentPos]);
 801 | 
 802 | 						SegmentPos++;
 803 | 						SegmentLen = 0;
 804 | 					}
 805 | 					else
 806 | 					{
 807 | 						Segment[SegmentPos][SegmentLen] = c;
 808 | 						SegmentLen++;
 809 | 					}
 810 | 				}
 811 | 
 812 | 				u32 IP[4];
 813 | 				u32 Mask[4];
 814 | 
 815 | 				IP[0] = atoi(Segment[0]);
 816 | 				IP[1] = atoi(Segment[1]);
 817 | 				IP[2] = atoi(Segment[2]);
 818 | 				IP[3] = atoi(Segment[3]);
 819 | 
 820 | 				Mask[0] = atoi(Segment[4]);
 821 | 				Mask[1] = atoi(Segment[5]);
 822 | 				Mask[2] = atoi(Segment[6]);
 823 | 				Mask[3] = atoi(Segment[7]);
 824 | 
 825 | 				fprintf(stderr, "    extract ip range %i.%i.%i.%i/%i.%i.%i.%i\n", 
 826 | 						IP[0],
 827 | 						IP[1],
 828 | 						IP[2],
 829 | 						IP[3],
 830 | 
 831 | 						Mask[0],
 832 | 						Mask[1],
 833 | 						Mask[2],
 834 | 						Mask[3]);
 835 | 
 836 | 				s_ExtractIPEnable 	= true;
 837 | 				s_ExtractIPMatch	= (IP[0] << 0) | (IP[1] << 8) | (IP[2] << 16) | (IP[3] << 24);
 838 | 				s_ExtractIPMask		= (Mask[0] << 0) | (Mask[1] << 8) | (Mask[2] << 16) | (Mask[3] << 24);
 839 | 			}
 840 | 			// extract all packets with the specified udp port 
 841 | 			else if (strcmp(argv[i], "--extract-port") == 0)
 842 | 			{
 843 | 				s_ExtractPortEnable 	= true;
 844 | 				s_ExtractPortMin		= atoi(argv[i+1]); 
 845 | 				s_ExtractPortMax		= atoi(argv[i+2]); 
 846 | 				i+= 2;
 847 | 
 848 | 				fprintf(stderr, "    extract port range: %i-%i\n", s_ExtractPortMin, s_ExtractPortMax);
 849 | 			}
 850 | 			// extract the specified flow as tcp stream
 851 | 			else if (strcmp(argv[i], "--extract-tcp") == 0)
 852 | 			{
 853 | 				u32 FlowID 			= atoi(argv[i+1]); 
 854 | 				s_ExtractTCPEnable 	= true;					
 855 | 				s_ExtractTCPFlow[ FlowID ] = 1;
 856 | 				i++;
 857 | 
 858 | 				fprintf(stderr, "     extract tcp flow %i\n", FlowID);
 859 | 			}
 860 | 			// extract all tcp flows with the matching port 
 861 | 			else if (strcmp(argv[i], "--extract-tcp-port") == 0)
 862 | 			{
 863 | 				u32 PortMin 			= atoi(argv[i+1]);
 864 | 				u32 PortMax 			= atoi(argv[i+2]);
 865 | 				s_ExtractTCPPortEnable 	= true;					
 866 | 				s_ExtractTCPPortMin 	= PortMin; 
 867 | 				s_ExtractTCPPortMax 	= PortMax; 
 868 | 				i += 2;	
 869 | 
 870 | 				fprintf(stderr, "     extract all tcp flow with port %i-%i\n", PortMin, PortMax);
 871 | 			}
 872 | 			// extract all tcp flows 
 873 | 			else if (strcmp(argv[i], "--extract-tcp-all") == 0)
 874 | 			{
 875 | 				s_ExtractTCPPortEnable 	= true;					
 876 | 				s_ExtractTCPPortMin 	= 0; 
 877 | 				s_ExtractTCPPortMax 	= 0x10000; 
 878 | 				fprintf(stderr, "    extract all tcp flow with port %i-%i\n", s_ExtractTCPPortMin, s_ExtractTCPPortMax);
 879 | 			}
 880 | 			// disable port range 
 881 | 			else if (strcmp(argv[i], "--disable-tcp-port") == 0)
 882 | 			{
 883 | 				u32 PortMin 			= atoi(argv[i+1]);
 884 | 				u32 PortMax 			= atoi(argv[i+2]);
 885 | 				s_DisableTCPPortMin[s_DisableTCPPortCnt] = PortMin; 
 886 | 				s_DisableTCPPortMax[s_DisableTCPPortCnt] = PortMax; 
 887 | 				s_DisableTCPPortCnt++;
 888 | 				assert(s_DisableTCPPortCnt < s_DisableTCPPortCntMax);
 889 | 
 890 | 				i += 2;	
 891 | 
 892 | 				fprintf(stderr, "    disable tcp extraction on ports [%i] %i-%i\n", s_DisableTCPPortCnt-1, PortMin, PortMax);
 893 | 			}
 894 | 			// extract udp flows within the specified range to individual files
 895 | 			else if (strcmp(argv[i], "--extract-udp-port") == 0)
 896 | 			{
 897 | 				u32 PortMin 			= atoi(argv[i+1]);
 898 | 				u32 PortMax 			= atoi(argv[i+2]);
 899 | 				s_ExtractUDPPortEnable 	= true;					
 900 | 				s_ExtractUDPPortMin 	= PortMin; 
 901 | 				s_ExtractUDPPortMax 	= PortMax; 
 902 | 			 	i += 2;	
 903 | 
 904 | 				fprintf(stderr, "    extract all udp flow`s with port %i-%i\n", PortMin, PortMax);
 905 | 			}
 906 | 			// extract udp all ports 
 907 | 			else if (strcmp(argv[i], "--extract-udp-all") == 0)
 908 | 			{
 909 | 				s_ExtractUDPPortEnable 	= true;					
 910 | 				s_ExtractUDPPortMin 	= 0; 
 911 | 				s_ExtractUDPPortMax 	= 65535; 
 912 | 
 913 | 				fprintf(stderr, "    extract all udp flows\n");
 914 | 			}
 915 | 			// disable port range 
 916 | 			else if (strcmp(argv[i], "--disable-udp-port") == 0)
 917 | 			{
 918 | 				u32 PortMin 			= atoi(argv[i+1]);
 919 | 				u32 PortMax 			= atoi(argv[i+2]);
 920 | 				s_DisableUDPPortMin[s_DisableUDPPortCnt] = PortMin; 
 921 | 				s_DisableUDPPortMax[s_DisableUDPPortCnt] = PortMax; 
 922 | 				s_DisableUDPPortCnt++;
 923 | 				assert(s_DisableUDPPortCnt < s_DisableUDPPortCntMax);
 924 | 
 925 | 				i += 2;	
 926 | 
 927 | 				fprintf(stderr, "    disable UDP extraction on ports [%i] %i-%i\n", s_DisableUDPPortCnt-1, PortMin, PortMax);
 928 | 			}
 929 | 
 930 | 			// input is from stdin 
 931 | 			else if (strcmp(argv[i], "--stdin") == 0)
 932 | 			{
 933 | 				TCPOutputFileName 	= "stdin"; 
 934 | 				UDPOutputFileName 	= "stdin"; 
 935 | 				FileStdin 		= true;
 936 | 				fprintf(stderr, "    reading PCAP from stdin\n");
 937 | 			}
 938 | 			// minimum number of packets 
 939 | 			else if (strcmp(argv[i], "--flow-packet-min") == 0)
 940 | 			{
 941 | 				s_FlowListPacketMin = atoi(argv[i+1]);
 942 | 				fprintf(stderr, "    minimum packet count %lli\n", s_FlowListPacketMin);
 943 | 			}
 944 | 			// display flow info
 945 | 			else if (strcmp(argv[i], "--disable-display") == 0)
 946 | 			{
 947 | 				s_EnableFlowDisplay = false;
 948 | 			}
 949 | 			// enable tcp header output 
 950 | 			else if (strcmp(argv[i], "--tcpheader") == 0)
 951 | 			{
 952 | 				g_EnableTCPHeader =true;
 953 | 				fprintf(stderr, "    enabling output TCP header\n");
 954 | 			}
 955 | 			// enable udp header output 
 956 | 			else if (strcmp(argv[i], "--udpheader") == 0)
 957 | 			{
 958 | 				g_EnableUDPHeader =true;
 959 | 				fprintf(stderr, "    enabling output UDP header\n");
 960 | 			}
 961 | 
 962 | 			// UDP output file
 963 | 			else if (strcmp(argv[i], "--output-udp") == 0)
 964 | 			{
 965 | 				UDPOutputFileName = argv[i+1];
 966 | 				i++;
 967 | 				fprintf(stderr, "    writing UDP data to [%s]\n", UDPOutputFileName);
 968 | 			}
 969 | 			// TCP output file
 970 | 			else if (strcmp(argv[i], "--output-tcp") == 0)
 971 | 			{
 972 | 				TCPOutputFileName = argv[i+1];
 973 | 				i++;
 974 | 				fprintf(stderr, "    writing TCP data to [%s]\n", TCPOutputFileName);
 975 | 			}
 976 | 			// Ouput in PCAP format 
 977 | 			else if (strcmp(argv[i], "--output-pcap") == 0)
 978 | 			{
 979 | 				g_OutputPCAP	= true;
 980 | 				fprintf(stderr, "    Output in PCAP format\n", g_OutputPCAP); 
 981 | 			}
 982 | 
 983 | 			// pin to a specific CPU
 984 | 			else if (strcmp(argv[i], "--cpu") == 0)
 985 | 			{
 986 | 				u32 CPU 			= atoi(argv[i+1]);
 987 | 				i++;
 988 | 
 989 | 				// pin to a thread
 990 | 				cpu_set_t	MainCPUS;
 991 | 				CPU_ZERO(&MainCPUS);
 992 | 				CPU_SET(CPU, &MainCPUS);
 993 | 				pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &MainCPUS);
 994 | 			}
 995 | 			// minimum flow file size creation 
 996 | 			else if (strcmp(argv[i], "--flow-size-min") == 0)
 997 | 			{
 998 | 				u32 FlowFileSizeMin	= atoi(argv[i+1]);
 999 | 				i++;
1000 | 
1001 | 				fFile_SizeMin(FlowFileSizeMin);
1002 | 
1003 | 				fprintf(stderr, "    minimum flow file size %i B\n", FlowFileSizeMin);
1004 | 			}
1005 | 			// force flushing after each write 
1006 | 			else if (strcmp(argv[i], "--flow-flush") == 0)
1007 | 			{
1008 | 				fFile_ForceFlush();
1009 | 				fprintf(stderr, "    flow force flushing\n"); 
1010 | 			}
1011 | 			else if (strcmp(argv[i], "--verbose") == 0)
1012 | 			{
1013 | 				fprintf(stderr, "    enable verbose mode\n");
1014 | 				g_Verbose = true;
1015 | 			}
1016 | 			else if (strcmp(argv[i], "--metamako") == 0)
1017 | 			{
1018 | 				fprintf(stderr, "    enable metamako timestamping\n");
1019 | 				g_EnableMetamako = true;
1020 | 			}
1021 | 			else if (strcmp(argv[i], "--metamako-double") == 0)
1022 | 			{
1023 | 				fprintf(stderr, "    enable metamako double tags\n");
1024 | 				g_EnableMetamako 	= true;
1025 | 				g_EnableMetamako2T 	= true;
1026 | 			}
1027 | 			else if (strcmp(argv[i], "--metamako-offset") == 0)
1028 | 			{
1029 | 				g_MetamakoOffset 	= atoi(argv[i+1]);
1030 | 				i++;
1031 | 				fprintf(stderr, "    metamako footer offset: %i\n", g_MetamakoOffset);
1032 | 			}
1033 | 			else if (strcmp(argv[i], "--vlan") == 0)
1034 | 			{
1035 | 				fprintf(stderr, "    enable vlan de-encapsulation\n");
1036 | 				g_EnableVLAN = true;
1037 | 			}
1038 | 			else if (strcmp(argv[i], "--disable-timezone") == 0)
1039 | 			{
1040 | 				fprintf(stderr, "    disable timezone adjustment\n");
1041 | 				s_TimeZoneEnable = false;
1042 | 			}
1043 | 			else if (strcmp(argv[i], "--flow-5tuple") == 0)
1044 | 			{
1045 | 				fprintf(stderr, "    flow 5 tuple only\n");
1046 | 				s_Flow5Tuple = true;
1047 | 			}
1048 | 			else if (strcmp(argv[i], "--uid") == 0)
1049 | 			{
1050 | 				fprintf(stderr, "    UID [%s]\n", argv[i+1]);
1051 | 				i++;
1052 | 			}
1053 | 			else
1054 | 			{
1055 | 				fprintf(stderr, "    unknown option [%s]\n", argv[i]);
1056 | 				return 0;
1057 | 			}
1058 | 		}
1059 | 	}
1060 | 
1061 | 	// needs atleast 2 files
1062 | 	if (!(FileStdin) && (FileNameListPos <= 0))
1063 | 	{
1064 | 		print_usage();
1065 | 		return 0;
1066 | 	}
1067 | 
1068 | 	if (FileStdin)	strcpy(s_FlowLogPath, "capture"); 
1069 | 	else			strcpy(s_FlowLogPath, FileNameList[0]); 
1070 | 
1071 | 	if (s_FlowExtractEnable && s_ExtractTCPEnable)
1072 | 	{
1073 | 		fprintf(stderr, "can not extract flow and tcp at the same time\n");
1074 | 		return 0;
1075 | 	}
1076 | 	if (s_ExtractTCPEnable && (!TCPOutputFileName))
1077 | 	{
1078 | 		fprintf(stderr, "specify tcp extract path --output-tcp <path>\n");
1079 | 		return 0;
1080 | 	}
1081 | 
1082 | 	// open output file
1083 | 	FILE* OutPCAP = NULL;
1084 | 	if (TCPOutputFileName && (!s_ExtractTCPEnable) )
1085 | 	{
1086 | 		OutPCAP = fopen(TCPOutputFileName, "w");
1087 | 		if (OutPCAP == NULL)
1088 | 		{
1089 | 			fprintf(stderr, "failed to open output file [%s]\n", TCPOutputFileName);
1090 | 			return 0;
1091 | 		}
1092 | 
1093 | 		PCAPHeader_t Header;
1094 | 		Header.Magic 	= PCAPHEADER_MAGIC_NANO;
1095 | 		Header.Major 	= PCAPHEADER_MAJOR;
1096 | 		Header.Minor 	= PCAPHEADER_MINOR;
1097 | 		Header.TimeZone = 0;
1098 | 		Header.SigFlag	= 0;
1099 | 		Header.SnapLen	= 8192;
1100 | 		Header.Link		= PCAPHEADER_LINK_ETHERNET;
1101 | 
1102 | 		fwrite(&Header, sizeof(Header), 1, OutPCAP);
1103 | 	}
1104 | 	
1105 | 	// open flow log file 
1106 | 	if (s_EnableFlowLog)
1107 | 	{
1108 | 		char Path[1024];
1109 | 		sprintf(Path, "%s.flow", s_FlowLogPath);
1110 | 		s_FlowLogFile = fopen(Path, "w");
1111 | 		if (!s_FlowLogFile )
1112 | 		{
1113 | 			fprintf(stderr, "failed to create flow log\n");
1114 | 			return 0; 
1115 | 		}
1116 | 	}
1117 | 
1118 | 	// calcuate tsc frequency
1119 | 	CycleCalibration();
1120 | 	setlocale(LC_NUMERIC, "");
1121 | 
1122 | 	// get timezone offset
1123 |   	time_t t = time(NULL);
1124 | 	struct tm lt = {0};
1125 | 
1126 | 	localtime_r(&t, &lt);
1127 | 	if (s_TimeZoneEnable)
1128 | 	{
1129 | 		s_TimeZoneOffset = lt.tm_gmtoff * 1e9;
1130 | 		fprintf(stderr, "Timezone Adjustment: %lli\n", s_TimeZoneOffset);
1131 | 	}
1132 | 
1133 | 	s_FlowIndex = (u32*)malloc( sizeof(u32)*(1ULL<<s_FlowIndexBits));
1134 | 	assert(s_FlowIndex  != NULL);
1135 | 	memset(s_FlowIndex, 0, sizeof(u32)*(1ULL<<s_FlowIndexBits));
1136 | 	g_TotalMemory 		+= sizeof(u32)*(1ULL<<s_FlowIndexBits);
1137 | 	s_FlowIndexMask		= (1<< s_FlowIndexBits) - 1;
1138 | 
1139 | 	s_FlowList 			= (FlowHash_t*)malloc( sizeof(FlowHash_t) * s_FlowListMax ); 
1140 | 	memset(s_FlowList, 0, sizeof(FlowHash_t) * s_FlowListMax );
1141 | 	assert(s_FlowList != NULL);
1142 | 	g_TotalMemory 		+= sizeof(FlowHash_t) * s_FlowListMax;
1143 | 
1144 | 	// set number of tcp flows
1145 | 	fTCPStream_MaxFlow(s_FlowListMax);	
1146 | 	
1147 | 	// open pcap diff files
1148 | 	PCAPFile_t* PCAPFile = OpenPCAP(FileNameList[0], FileStdin);	
1149 | 	if (!PCAPFile) return 0;
1150 | 
1151 | 	// init tcp reassembly
1152 | 	/*
1153 | 	if (s_ExtractTCPEnable)
1154 | 	{
1155 | 		for (int i=0; i < s_FlowExtractMax; i++)
1156 | 		{
1157 | 			if (s_ExtractTCP[i]j
1158 | 			{
1159 | 				char Path[256];
1160 | 				u32 FlowID = i; 
1161 | 
1162 | 				sprintf(Path, "%s.tcpflow.%i", TCPOutputFileName, FlowID);
1163 | 				s_ExtractTCP[i] = fTCPStream_Init(kMB(128), Path, FlowID, 0);
1164 | 			}
1165 | 		}
1166 | 	}
1167 | 	*/
1168 | 
1169 | 	// clear tcp output
1170 | 	memset(s_ExtractTCP, 0, sizeof(s_ExtractTCP));
1171 | 	printf("[%30s] FileSize: %lliGB\n", PCAPFile->Path, PCAPFile->Length / kGB(1));
1172 | 
1173 | 	u64 TotalByte 		= 0;
1174 | 	u64 TotalPkt 		= 0;
1175 | 	u64 NextPrintTSC 	= 0;
1176 | 	u64 StartTSC		= rdtsc();	
1177 | 	u64 OutputByte		= 0;
1178 | 	u64 OutputTCPByte	= 0;
1179 | 	u64 OutputUDPByte	= 0;
1180 | 	while (true)
1181 | 	{
1182 | 		fProfile_Start(0, "Top");
1183 | 
1184 | 		fProfile_Start(1, "Fetch");
1185 | 
1186 | 		PCAPPacket_t* Pkt = ReadPCAP(PCAPFile); 
1187 | 		if (!Pkt)
1188 | 		{
1189 | 			fprintf(stderr, "no more packets exiting\n");
1190 | 			break;
1191 | 		}
1192 | 
1193 | 		// invalid packet
1194 | 		if (Pkt->Length 		== 0) break;
1195 | 		if (Pkt->LengthCapture 	== 0) break;
1196 | 
1197 | 		fProfile_Stop(1);
1198 | 		fProfile_Start(2, "Decode");
1199 | 
1200 | 		PCAPFile->TS = PCAPTimeStamp(Pkt);
1201 | 
1202 | 		u32 HashLength = 0;
1203 | 		FlowHash_t  Flow;
1204 | 
1205 | 		fEther_t * Ether = PCAPETHHeader(Pkt);
1206 | 
1207 | 		u32 EtherProto = swap16(Ether->Proto);
1208 | 
1209 | 		u32 DeviceID		= 0;
1210 | 		u32 DevicePort		= 0;
1211 | 
1212 | 		// VLAN tagging as device ID. e.g. Arista DANZ
1213 | 		if (g_EnableVLAN && (swap16(Ether->Proto) == ETHER_PROTO_VLAN))
1214 | 		{
1215 | 			VLANTag_t* Header 	= (VLANTag_t*)(Ether+1);
1216 | 			u16* Proto 			= (u16*)(Header + 1);
1217 | 
1218 | 			DeviceID 	=  VLANTag_ID(Header);		
1219 | 			DevicePort	= 0;
1220 | 
1221 | 			// set next ethernet protocol 
1222 | 			EtherProto = swap16(Proto[0]); 
1223 | 	
1224 | 			// nasty... chomp the packet
1225 | 			// need a better way to do this 
1226 | 			memmove(Ether + 1, Proto + 1, Pkt->Length - sizeof(VLANTag_t) - 2  - sizeof(fEther_t) );
1227 | 		}
1228 | 
1229 | 		// update timestamp
1230 | 		Metamako_t* MFooter = NULL; 
1231 | 		if (g_EnableMetamako)
1232 | 		{
1233 | 			// metamako footer
1234 | 			MFooter = PCAPMetamako(Pkt); 
1235 | 
1236 | 			// update with Metamako timestamp
1237 | 			PCAPFile->TS = ((u64)swap32(MFooter->Sec)) * 1000000000ULL + (u64)swap32(MFooter->NSec);
1238 | 		}
1239 | 
1240 | 		switch (EtherProto)
1241 | 		{
1242 | 		case ETHER_PROTO_IPV4:
1243 | 		{
1244 | 			IP4Header_t* IP4 = PCAPIP4Header(Pkt); 
1245 | 			u32 IPOffset = (IP4->Version & 0x0f)*4; 
1246 | 			switch (IP4->Proto)
1247 | 			{
1248 | 			case IPv4_PROTO_TCP:
1249 | 			{
1250 | 				if (s_ExtractTCPPortEnable)
1251 | 				{
1252 | 					TCPHeader_t* TCP = (TCPHeader_t*)( ((u8*)IP4) + IPOffset);
1253 | 
1254 | 					memset(&Flow, 0, sizeof(Flow));
1255 | 
1256 | 					Flow.Type = FLOW_TYPE_TCP;
1257 | 
1258 | 					TCPHash_t* TCPHash = (TCPHash_t*)Flow.Data;
1259 | 					memset(TCPHash, 0, 64);
1260 | 
1261 | 					memcpy(TCPHash->MACDst, Ether->Dst, 6);
1262 | 					memcpy(TCPHash->MACSrc, Ether->Src, 6);
1263 | 
1264 | 					TCPHash->IPSrc = IP4->Src;
1265 | 					TCPHash->IPDst = IP4->Dst;
1266 | 
1267 | 					TCPHash->PortSrc = swap16(TCP->PortSrc); 
1268 | 					TCPHash->PortDst = swap16(TCP->PortDst); 
1269 | 
1270 | 					if (!s_Flow5Tuple)
1271 | 					{
1272 | 						TCPHash->DeviceID 	= DeviceID; 
1273 | 						TCPHash->DevicePort = DevicePort; 
1274 | 
1275 | 						if (g_EnableMetamako)
1276 | 						{
1277 | 							TCPHash->DeviceID 	= swap16(MFooter->DeviceID);
1278 | 							TCPHash->DevicePort = MFooter->PortID;
1279 | 						}
1280 | 					}
1281 | 
1282 | 					HashLength = 64; 
1283 | 
1284 | 					// mark tcp SYN/SYNACK sequence numbers for duplex matching
1285 | 					if (TCP_FLAG_SYN(TCP->Flags) & (!TCP_FLAG_ACK(TCP->Flags)))
1286 | 					{
1287 | 						// syn seq no 
1288 | 						Flow.TCPSeqNo = swap32(TCP->SeqNo);
1289 | 					}
1290 | 					if (TCP_FLAG_SYN(TCP->Flags) & (TCP_FLAG_ACK(TCP->Flags)))
1291 | 					{
1292 | 						// syn.ack ack no (syn.ack bumps it by 1)
1293 | 						Flow.TCPSeqNo = swap32(TCP->AckNo) -1;
1294 | 					}
1295 | 				}
1296 | 			}
1297 | 			break;
1298 | 
1299 | 			case IPv4_PROTO_UDP: 
1300 | 			{
1301 | 				if (s_ExtractUDPPortEnable)
1302 | 				{
1303 | 					UDPHeader_t* UDP = (UDPHeader_t*)( ((u8*)IP4) + IPOffset);
1304 | 
1305 | 					memset(&Flow, 0, sizeof(Flow));
1306 | 
1307 | 					Flow.Type = FLOW_TYPE_UDP;
1308 | 
1309 | 					UDPHash_t* UDPHash = (UDPHash_t*)Flow.Data;
1310 | 					memset(UDPHash, 0, 64);
1311 | 
1312 | 					memcpy(UDPHash->MACDst, Ether->Dst, 6);
1313 | 					memcpy(UDPHash->MACSrc, Ether->Src, 6);
1314 | 
1315 | 					UDPHash->IPSrc = IP4->Src;
1316 | 					UDPHash->IPDst = IP4->Dst;
1317 | 
1318 | 					UDPHash->PortSrc = swap16(UDP->PortSrc); 
1319 | 					UDPHash->PortDst = swap16(UDP->PortDst); 
1320 | 
1321 | 					if (!s_Flow5Tuple)
1322 | 					{
1323 | 						UDPHash->DeviceID 	= DeviceID; 
1324 | 						UDPHash->DevicePort = DevicePort; 
1325 | 
1326 | 						if (g_EnableMetamako)
1327 | 						{
1328 | 							UDPHash->DeviceID 	= swap16(MFooter->DeviceID);
1329 | 							UDPHash->DevicePort = MFooter->PortID;
1330 | 						}
1331 | 					}
1332 | 					HashLength = 64; 
1333 | 				}
1334 | 			}
1335 | 			break;
1336 | 
1337 | 			default:
1338 | 				//printf("ipv4 %x\n", IP4->Proto);
1339 | 				break;
1340 | 			}
1341 | 		}
1342 | 		break;
1343 | 
1344 | 		default:
1345 | 			//printf("proto: %08x\n", swap16(Ether->Proto) );
1346 | 			break;
1347 | 		}
1348 | 
1349 | 		fProfile_Stop(2);
1350 | 
1351 | 		// if its valid TCP or UDP data
1352 | 		if (HashLength != 0)
1353 | 		{
1354 | 			fProfile_Start(3, "FlowAdd");
1355 | 			u32 FlowID = 0;
1356 | 			if (HashLength > 0)
1357 | 			{
1358 | 				FlowID = FlowAdd(&Flow, Pkt->Length, PCAPFile->TS);
1359 | 			}
1360 | 
1361 | 			if ((FlowID != 0) && s_FlowExtract[ FlowID ])
1362 | 			{
1363 | 				if (OutPCAP)
1364 | 				{
1365 | 					fwrite(Pkt, sizeof(PCAPPacket_t) + Pkt->LengthCapture, 1, OutPCAP);
1366 | 					OutputByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1367 | 				}
1368 | 			}
1369 | 			fProfile_Stop(3);
1370 | 
1371 | 			// extract all tcp flows with the specified port range
1372 | 			fProfile_Start(4, "TCP Flow");
1373 | 			if (Flow.Type == FLOW_TYPE_TCP)
1374 | 			{
1375 | 				TCPHeader_t* TCPHeader 	= PCAPTCPHeader(Pkt);
1376 | 				TCPHash_t* TCP 			= (TCPHash_t*)Flow.Data;
1377 | 
1378 | 				bool Output = false; 
1379 | 				// tcp port ranges
1380 | 				if (s_ExtractTCPPortEnable)
1381 | 				{
1382 | 					Output |= (TCP->PortSrc >= s_ExtractTCPPortMin) && (TCP->PortSrc <= s_ExtractTCPPortMax);
1383 | 					Output |= (TCP->PortDst >= s_ExtractTCPPortMin) && (TCP->PortDst <= s_ExtractTCPPortMax);
1384 | 
1385 | 					// disable port range
1386 | 					for (int d=0; d < s_DisableTCPPortCnt; d++)
1387 | 					{
1388 | 						if ((TCP->PortSrc >= s_DisableTCPPortMin[d]) && (TCP->PortSrc <= s_DisableTCPPortMax[d]))
1389 | 						{
1390 | 							Output = false;	
1391 | 						}
1392 | 						if ((TCP->PortDst >= s_DisableTCPPortMin[d]) && (TCP->PortDst <= s_DisableTCPPortMax[d]))
1393 | 						{
1394 | 							Output = false;	
1395 | 						}
1396 | 					}
1397 | 				}
1398 | 				// specific flow id`s
1399 | 				if (s_ExtractTCPEnable && s_ExtractTCPFlow[FlowID])
1400 | 				{
1401 | 					Output = true;
1402 | 				}
1403 | 					
1404 | 				if (Output)
1405 | 				{
1406 | 					// new flow ? 	
1407 | 					struct TCPStream_t* Stream = s_ExtractTCP[FlowID];
1408 | 					if (Stream == NULL)
1409 | 					{
1410 | 						char FileName[1024];
1411 | 
1412 | 						// 5-tuple flow only mode
1413 | 						if (s_Flow5Tuple)
1414 | 						{
1415 | 							sprintf(FileName, "%s_%02x:%02x:%02x:%02x:%02x:%02x->%02x:%02x:%02x:%02x:%02x:%02x_%3i.%3i.%3i.%3i->%3i.%3i.%3i.%3i_%6i->%6i",
1416 | 								TCPOutputFileName,
1417 | 								
1418 | 								TCP->MACSrc[0],	
1419 | 								TCP->MACSrc[1],	
1420 | 								TCP->MACSrc[2],	
1421 | 								TCP->MACSrc[3],	
1422 | 								TCP->MACSrc[4],	
1423 | 								TCP->MACSrc[5],	
1424 | 		
1425 | 								TCP->MACDst[0],	
1426 | 								TCP->MACDst[1],	
1427 | 								TCP->MACDst[2],	
1428 | 								TCP->MACDst[3],	
1429 | 								TCP->MACDst[4],	
1430 | 								TCP->MACDst[5],	
1431 | 
1432 | 								TCP->IPSrc.IP[0],
1433 | 								TCP->IPSrc.IP[1],
1434 | 								TCP->IPSrc.IP[2],
1435 | 								TCP->IPSrc.IP[3],
1436 | 
1437 | 								TCP->IPDst.IP[0],
1438 | 								TCP->IPDst.IP[1],
1439 | 								TCP->IPDst.IP[2],
1440 | 								TCP->IPDst.IP[3],
1441 | 
1442 | 								TCP->PortSrc,
1443 | 								TCP->PortDst
1444 | 							);
1445 | 
1446 | 						}
1447 | 						// add device/port info from the packet broker
1448 | 						else
1449 | 						{
1450 | 							sprintf(FileName, "%s_%02x:%02x:%02x:%02x:%02x:%02x->%02x:%02x:%02x:%02x:%02x:%02x_%3i.%3i.%3i.%3i->%3i.%3i.%3i.%3i_%6i->%6i_%05i_%02i",
1451 | 								TCPOutputFileName,
1452 | 								
1453 | 								TCP->MACSrc[0],	
1454 | 								TCP->MACSrc[1],	
1455 | 								TCP->MACSrc[2],	
1456 | 								TCP->MACSrc[3],	
1457 | 								TCP->MACSrc[4],	
1458 | 								TCP->MACSrc[5],	
1459 | 		
1460 | 								TCP->MACDst[0],	
1461 | 								TCP->MACDst[1],	
1462 | 								TCP->MACDst[2],	
1463 | 								TCP->MACDst[3],	
1464 | 								TCP->MACDst[4],	
1465 | 								TCP->MACDst[5],	
1466 | 
1467 | 								TCP->IPSrc.IP[0],
1468 | 								TCP->IPSrc.IP[1],
1469 | 								TCP->IPSrc.IP[2],
1470 | 								TCP->IPSrc.IP[3],
1471 | 
1472 | 								TCP->IPDst.IP[0],
1473 | 								TCP->IPDst.IP[1],
1474 | 								TCP->IPDst.IP[2],
1475 | 								TCP->IPDst.IP[3],
1476 | 
1477 | 								TCP->PortSrc,
1478 | 								TCP->PortDst,
1479 | 
1480 | 								TCP->DeviceID,
1481 | 								TCP->DevicePort
1482 | 							);
1483 | 						}
1484 | 
1485 | 						Stream = fTCPStream_Init(kMB(128), FileName, FlowID, Flow.Hash, PCAPFile->TS);
1486 | 						s_ExtractTCP[FlowID] = Stream;
1487 | 					}
1488 | 					if (Stream == NULL)
1489 | 					{
1490 | 						printf("invalid flwo: %i\n", FlowID);
1491 | 					}
1492 | 					assert(Stream != NULL);
1493 | 
1494 | 					// add packet to the stream
1495 | 			fProfile_Start(6, "TCP PktAdd");
1496 | 
1497 | 					u32 TCPPayloadLength = 0;
1498 | 					u8*	TCPPayload	= PCAPTCPPayload(Pkt, &TCPPayloadLength); 
1499 | 
1500 | 					fTCPStream_PacketAdd(Stream, PCAPFile->TS, TCPHeader, TCPPayloadLength, TCPPayload, Pkt);
1501 | 
1502 | 			fProfile_Stop(6);
1503 | 
1504 | 					OutputTCPByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1505 | 				}
1506 | 			}
1507 | 			fProfile_Stop(4);
1508 | 
1509 | 			// extract all udp flows  
1510 | 			if (s_ExtractUDPPortEnable && (Flow.Type == FLOW_TYPE_UDP))
1511 | 			{
1512 | 				UDPHeader_t* UDPHeader 	= PCAPUDPHeader(Pkt);
1513 | 				UDPHash_t* UDP 			= (UDPHash_t*)Flow.Data;
1514 | 
1515 | 				bool Output = false; 
1516 | 				Output |= (UDP->PortSrc >= s_ExtractUDPPortMin) && (UDP->PortSrc <= s_ExtractUDPPortMax);
1517 | 				Output |= (UDP->PortDst >= s_ExtractUDPPortMin) && (UDP->PortDst <= s_ExtractUDPPortMax);
1518 | 
1519 | 				// disable port range
1520 | 				for (int d=0; d < s_DisableUDPPortCnt; d++)
1521 | 				{
1522 | 					if ((UDP->PortSrc >= s_DisableUDPPortMin[d]) && (UDP->PortSrc <= s_DisableUDPPortMax[d]))
1523 | 					{
1524 | 						Output = false;	
1525 | 					}
1526 | 					if ((UDP->PortDst >= s_DisableUDPPortMin[d]) && (UDP->PortDst <= s_DisableUDPPortMax[d]))
1527 | 					{
1528 | 						Output = false;	
1529 | 					}
1530 | 				}
1531 | 
1532 | 				if (Output)
1533 | 				{
1534 | 					// new flow ? 	
1535 | 					struct UDPStream_t* Stream = s_ExtractUDP[FlowID];
1536 | 					if (Stream == NULL)
1537 | 					{
1538 | 						char FileName[257];
1539 | 
1540 | 						if (s_Flow5Tuple)
1541 | 						{
1542 | 							sprintf(FileName, "%s_%02x:%02x:%02x:%02x:%02x:%02x->%02x:%02x:%02x:%02x:%02x:%02x_%3i.%3i.%3i.%3i->%3i.%3i.%3i.%3i_%6i->%6i",
1543 | 								UDPOutputFileName,
1544 | 
1545 | 								UDP->MACSrc[0],	
1546 | 								UDP->MACSrc[1],	
1547 | 								UDP->MACSrc[2],	
1548 | 								UDP->MACSrc[3],	
1549 | 								UDP->MACSrc[4],	
1550 | 								UDP->MACSrc[5],	
1551 | 		
1552 | 								UDP->MACDst[0],	
1553 | 								UDP->MACDst[1],	
1554 | 								UDP->MACDst[2],	
1555 | 								UDP->MACDst[3],	
1556 | 								UDP->MACDst[4],	
1557 | 								UDP->MACDst[5],	
1558 | 
1559 | 								UDP->IPSrc.IP[0],
1560 | 								UDP->IPSrc.IP[1],
1561 | 								UDP->IPSrc.IP[2],
1562 | 								UDP->IPSrc.IP[3],
1563 | 
1564 | 								UDP->IPDst.IP[0],
1565 | 								UDP->IPDst.IP[1],
1566 | 								UDP->IPDst.IP[2],
1567 | 								UDP->IPDst.IP[3],
1568 | 
1569 | 								UDP->PortSrc,
1570 | 								UDP->PortDst
1571 | 							);
1572 | 						}
1573 | 						// inlcude packet broker device id/port info
1574 | 						else
1575 | 						{
1576 | 							sprintf(FileName, "%s_%02x:%02x:%02x:%02x:%02x:%02x->%02x:%02x:%02x:%02x:%02x:%02x_%3i.%3i.%3i.%3i->%3i.%3i.%3i.%3i_%6i->%6i_%05i_%02i",
1577 | 								UDPOutputFileName,
1578 | 
1579 | 								UDP->MACSrc[0],	
1580 | 								UDP->MACSrc[1],	
1581 | 								UDP->MACSrc[2],	
1582 | 								UDP->MACSrc[3],	
1583 | 								UDP->MACSrc[4],	
1584 | 								UDP->MACSrc[5],	
1585 | 		
1586 | 								UDP->MACDst[0],	
1587 | 								UDP->MACDst[1],	
1588 | 								UDP->MACDst[2],	
1589 | 								UDP->MACDst[3],	
1590 | 								UDP->MACDst[4],	
1591 | 								UDP->MACDst[5],	
1592 | 
1593 | 								UDP->IPSrc.IP[0],
1594 | 								UDP->IPSrc.IP[1],
1595 | 								UDP->IPSrc.IP[2],
1596 | 								UDP->IPSrc.IP[3],
1597 | 
1598 | 								UDP->IPDst.IP[0],
1599 | 								UDP->IPDst.IP[1],
1600 | 								UDP->IPDst.IP[2],
1601 | 								UDP->IPDst.IP[3],
1602 | 
1603 | 								UDP->PortSrc,
1604 | 								UDP->PortDst,
1605 | 
1606 | 								UDP->DeviceID,
1607 | 								UDP->DevicePort
1608 | 							);
1609 | 						}
1610 | 						Stream = fUDPStream_Init(FileName, FlowID, PCAPFile->TS);
1611 | 						s_ExtractUDP[FlowID] = Stream;
1612 | 					}
1613 | 					assert(Stream != NULL);
1614 | 
1615 | 					fUDPStream_Add(Stream, PCAPFile->TS, Pkt, UDPHeader);
1616 | 					OutputUDPByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1617 | 				}
1618 | 			}
1619 | 
1620 | 			// extract all IP`s that match the mask
1621 | 			if (s_ExtractIPEnable)
1622 | 			{
1623 | 				IP4Header_t* IP4 = PCAPIP4Header(Pkt); 
1624 | 
1625 | 				bool Extract = false;
1626 | 
1627 | 				if ((IP4->Src.IP4 & s_ExtractIPMask) == s_ExtractIPMatch)
1628 | 				{
1629 | 					Extract = true;
1630 | 				}
1631 | 				if ((IP4->Dst.IP4 & s_ExtractIPMask) == s_ExtractIPMatch)
1632 | 				{
1633 | 					Extract = true;
1634 | 				}
1635 | 
1636 | 				if (Extract && OutPCAP)
1637 | 				{
1638 | 					fwrite(Pkt, sizeof(PCAPPacket_t) + Pkt->LengthCapture, 1, OutPCAP);
1639 | 					OutputByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1640 | 				}
1641 | 			}
1642 | 
1643 | 			// extract UDP port
1644 | 			if (s_ExtractPortEnable)
1645 | 			{
1646 | 				bool Extract = false;
1647 | 				u32 PortSrc = 0;
1648 | 				u32 PortDst = 0;
1649 | 				if (Flow.Type == FLOW_TYPE_UDP)
1650 | 								
1651 | 				{
1652 | 					UDPHeader_t* UDP = PCAPUDPHeader(Pkt); 
1653 | 
1654 | 					PortSrc = swap16(UDP->PortSrc);
1655 | 					PortDst = swap16(UDP->PortDst);
1656 | 				}
1657 | 				if (Flow.Type == FLOW_TYPE_TCP)
1658 | 				{
1659 | 					TCPHeader_t* TCP = PCAPTCPHeader(Pkt); 
1660 | 
1661 | 					PortSrc = swap16(TCP->PortSrc);
1662 | 					PortDst = swap16(TCP->PortDst);
1663 | 				}
1664 | 
1665 | 				if ((s_ExtractPortMin <= PortSrc) && 
1666 | 					(PortSrc <= s_ExtractPortMax))
1667 | 				{
1668 | 					Extract = true;
1669 | 				}
1670 | 
1671 | 				if ((s_ExtractPortMin <= PortDst) && 
1672 | 					(PortDst <= s_ExtractPortMax))
1673 | 				{
1674 | 					Extract = true;
1675 | 				}
1676 | 				if (Extract && OutPCAP)
1677 | 				{
1678 | 					fwrite(Pkt, sizeof(PCAPPacket_t) + Pkt->LengthCapture, 1, OutPCAP);
1679 | 					OutputByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1680 | 				}
1681 | 			}
1682 | 		}
1683 | 
1684 | 		TotalPkt++;
1685 | 		TotalByte += sizeof(PCAPPacket_t) + Pkt->LengthCapture;
1686 | 
1687 | 		fProfile_Stop(0);
1688 | 
1689 | 		if (rdtsc() > NextPrintTSC)
1690 | 		{
1691 | 			u64 TSC = rdtsc();
1692 | 			NextPrintTSC = TSC + 3e9;
1693 | 
1694 | 			static u64 LastTSC = 0;
1695 | 			double dT = tsc2ns(TSC - LastTSC) / 1e9;
1696 | 			LastTSC = TSC;
1697 | 
1698 | 			static u64 LastByte = 0;
1699 | 			double Bps = (TotalByte - LastByte) / dT;
1700 | 			LastByte = TotalByte;
1701 | 
1702 | 			double TotalTime = tsc2ns(TSC - StartTSC);
1703 | 
1704 | 			double MeanHashDepth = s_FlowIndexDepthS1 * inverse(s_FlowIndexDepthS0);
1705 | 
1706 | 			u64 TSf = PCAPFile->TS; 
1707 | 			fprintf(stderr, "[%s %.3f%%] ", FormatTS(TSf), PCAPFile->ReadPos / (double)PCAPFile->Length); 
1708 | 			fprintf(stderr, "%5.f Min ", TotalTime / 60e9);
1709 | 			fprintf(stderr, "Flows:%i ", s_FlowListPos);
1710 | 			fprintf(stderr, "%lli Pkts %8.3fGbps : %.2fGB ",
1711 | 					TotalPkt,
1712 | 					(8.0*Bps) / 1e9,
1713 | 					TotalByte / 1e9
1714 | 			);
1715 | 			fprintf(stderr, "Out:%.2fGB ", OutputByte / 1e9);
1716 | 			fprintf(stderr, "OutTCP:%.2fGB ", OutputTCPByte / 1e9);
1717 | 			fprintf(stderr, "OutUDP:%.2fGB ", OutputUDPByte / 1e9);
1718 | 			fprintf(stderr, "Memory:%.2fMB ", g_TotalMemory / 1e6); 
1719 | 			fprintf(stderr, "MemoryTCP:%.2fMB ", g_TotalMemoryTCP / 1e6); 
1720 | 			fprintf(stderr, "HashDepth:%i (%.3f)  ", s_FlowIndexDepthMax, MeanHashDepth);
1721 | 
1722 | 			fprintf(stderr, "\n");
1723 | 
1724 | 			// push everything out (e.g. for long runs constantly push to log file)
1725 | 			fflush(stdout);
1726 | 			fflush(stderr);
1727 | 			if (TotalPkt > s_MaxPackets)
1728 | 			{
1729 | 				fprintf(stderr, "Maxpackets reached exiting: %lli %lli\n", TotalPkt, s_MaxPackets);
1730 | 				break;
1731 | 			}
1732 | 
1733 | 			static int cnt = 0;
1734 | 			if (cnt++ > 10)
1735 | 			{
1736 | 				cnt = 0;
1737 | 				fProfile_Dump(0);
1738 | 
1739 | 				// dump stats
1740 | 				fTCPStream_Dump(PCAPFile->TS);
1741 | 			}
1742 | /*
1743 | 			// flush tcp/streams to disk
1744 | 			static u32 FlowFlush = 0;
1745 | 
1746 | 			// only flush 5K flows at a time
1747 | 			u32 FlushCnt = (s_FlowListMax > 5000) ? 5000 : s_FlowListMax;
1748 | 			for (int i=0; i < FlushCnt; i++)
1749 | 			{
1750 | 				if (s_ExtractTCP[FlowFlush])
1751 | 				{
1752 | 					//fprintf(stderr, "[%i] TCP Flush\n", i);
1753 | 					fTCPStream_Flush(s_ExtractTCP[FlowFlush]);
1754 | 				}
1755 | 				FlowFlush++;
1756 | 				if (FlowFlush >= s_FlowListMax) FlowFlush = 0;
1757 | 			}
1758 | */
1759 | 		}
1760 | 	}
1761 | 	fprintf(stderr, "parse done TotalPkts:%lli\n", TotalPkt);
1762 | 	fflush(stderr);
1763 | 
1764 | 	// need print stats before closing streams
1765 | 	if (s_EnableFlowDisplay) PrintHumanFlows();	
1766 | 
1767 | 	// close output streams
1768 | 	for (int i=0; i < s_FlowListMax; i++)
1769 | 	{
1770 | 		if (s_ExtractTCP[i])
1771 | 		{
1772 | 			//fprintf(stderr, "[%i] TCP Close\n", i);
1773 | 			fTCPStream_Close(s_ExtractTCP[i]);
1774 | 		}
1775 | 	}
1776 | 
1777 | 	if (OutPCAP) fclose(OutPCAP);
1778 | }
1779 | 
1780 | /* vim: set ts=4 sts=4 */
1781 | 


--------------------------------------------------------------------------------
/sha1.c:
--------------------------------------------------------------------------------
  1 | /* 
  2 |  * SHA-1 hash in C
  3 |  * 
  4 |  * Copyright (c) 2017 Project Nayuki. (MIT License)
  5 |  * https://www.nayuki.io/page/fast-sha1-hash-implementation-in-x86-assembly
  6 |  * 
  7 |  * Permission is hereby granted, free of charge, to any person obtaining a copy of
  8 |  * this software and associated documentation files (the "Software"), to deal in
  9 |  * the Software without restriction, including without limitation the rights to
 10 |  * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 11 |  * the Software, and to permit persons to whom the Software is furnished to do so,
 12 |  * subject to the following conditions:
 13 |  * - The above copyright notice and this permission notice shall be included in
 14 |  *   all copies or substantial portions of the Software.
 15 |  * - The Software is provided "as is", without warranty of any kind, express or
 16 |  *   implied, including but not limited to the warranties of merchantability,
 17 |  *   fitness for a particular purpose and noninfringement. In no event shall the
 18 |  *   authors or copyright holders be liable for any claim, damages or other
 19 |  *   liability, whether in an action of contract, tort or otherwise, arising from,
 20 |  *   out of or in connection with the Software or the use or other dealings in the
 21 |  *   Software.
 22 |  */
 23 | 
 24 | #include <stdint.h>
 25 | 
 26 | 
 27 | void sha1_compress(uint32_t state[static 5], const uint8_t block[static 64]) {
 28 | 	#define ROTL32(x, n)  (((0U + (x)) << (n)) | ((x) >> (32 - (n))))  // Assumes that x is uint32_t and 0 < n < 32
 29 | 	
 30 | 	#define LOADSCHEDULE(i)  \
 31 | 		schedule[i] = (uint32_t)block[i * 4 + 0] << 24  \
 32 | 		            | (uint32_t)block[i * 4 + 1] << 16  \
 33 | 		            | (uint32_t)block[i * 4 + 2] <<  8  \
 34 | 		            | (uint32_t)block[i * 4 + 3] <<  0;
 35 | 	
 36 | 	#define SCHEDULE(i)  \
 37 | 		temp = schedule[(i - 3) & 0xF] ^ schedule[(i - 8) & 0xF] ^ schedule[(i - 14) & 0xF] ^ schedule[(i - 16) & 0xF];  \
 38 | 		schedule[i & 0xF] = ROTL32(temp, 1);
 39 | 	
 40 | 	#define ROUND0a(a, b, c, d, e, i)  LOADSCHEDULE(i)  ROUNDTAIL(a, b, e, ((b & c) | (~b & d))         , i, 0x5A827999)
 41 | 	#define ROUND0b(a, b, c, d, e, i)  SCHEDULE(i)      ROUNDTAIL(a, b, e, ((b & c) | (~b & d))         , i, 0x5A827999)
 42 | 	#define ROUND1(a, b, c, d, e, i)   SCHEDULE(i)      ROUNDTAIL(a, b, e, (b ^ c ^ d)                  , i, 0x6ED9EBA1)
 43 | 	#define ROUND2(a, b, c, d, e, i)   SCHEDULE(i)      ROUNDTAIL(a, b, e, ((b & c) ^ (b & d) ^ (c & d)), i, 0x8F1BBCDC)
 44 | 	#define ROUND3(a, b, c, d, e, i)   SCHEDULE(i)      ROUNDTAIL(a, b, e, (b ^ c ^ d)                  , i, 0xCA62C1D6)
 45 | 	
 46 | 	#define ROUNDTAIL(a, b, e, f, i, k)  \
 47 | 		e = 0U + e + ROTL32(a, 5) + f + UINT32_C(k) + schedule[i & 0xF];  \
 48 | 		b = ROTL32(b, 30);
 49 | 	
 50 | 	uint32_t a = state[0];
 51 | 	uint32_t b = state[1];
 52 | 	uint32_t c = state[2];
 53 | 	uint32_t d = state[3];
 54 | 	uint32_t e = state[4];
 55 | 	
 56 | 	uint32_t schedule[16];
 57 | 	uint32_t temp;
 58 | 	ROUND0a(a, b, c, d, e,  0)
 59 | 	ROUND0a(e, a, b, c, d,  1)
 60 | 	ROUND0a(d, e, a, b, c,  2)
 61 | 	ROUND0a(c, d, e, a, b,  3)
 62 | 	ROUND0a(b, c, d, e, a,  4)
 63 | 	ROUND0a(a, b, c, d, e,  5)
 64 | 	ROUND0a(e, a, b, c, d,  6)
 65 | 	ROUND0a(d, e, a, b, c,  7)
 66 | 	ROUND0a(c, d, e, a, b,  8)
 67 | 	ROUND0a(b, c, d, e, a,  9)
 68 | 	ROUND0a(a, b, c, d, e, 10)
 69 | 	ROUND0a(e, a, b, c, d, 11)
 70 | 	ROUND0a(d, e, a, b, c, 12)
 71 | 	ROUND0a(c, d, e, a, b, 13)
 72 | 	ROUND0a(b, c, d, e, a, 14)
 73 | 	ROUND0a(a, b, c, d, e, 15)
 74 | 	ROUND0b(e, a, b, c, d, 16)
 75 | 	ROUND0b(d, e, a, b, c, 17)
 76 | 	ROUND0b(c, d, e, a, b, 18)
 77 | 	ROUND0b(b, c, d, e, a, 19)
 78 | 	ROUND1(a, b, c, d, e, 20)
 79 | 	ROUND1(e, a, b, c, d, 21)
 80 | 	ROUND1(d, e, a, b, c, 22)
 81 | 	ROUND1(c, d, e, a, b, 23)
 82 | 	ROUND1(b, c, d, e, a, 24)
 83 | 	ROUND1(a, b, c, d, e, 25)
 84 | 	ROUND1(e, a, b, c, d, 26)
 85 | 	ROUND1(d, e, a, b, c, 27)
 86 | 	ROUND1(c, d, e, a, b, 28)
 87 | 	ROUND1(b, c, d, e, a, 29)
 88 | 	ROUND1(a, b, c, d, e, 30)
 89 | 	ROUND1(e, a, b, c, d, 31)
 90 | 	ROUND1(d, e, a, b, c, 32)
 91 | 	ROUND1(c, d, e, a, b, 33)
 92 | 	ROUND1(b, c, d, e, a, 34)
 93 | 	ROUND1(a, b, c, d, e, 35)
 94 | 	ROUND1(e, a, b, c, d, 36)
 95 | 	ROUND1(d, e, a, b, c, 37)
 96 | 	ROUND1(c, d, e, a, b, 38)
 97 | 	ROUND1(b, c, d, e, a, 39)
 98 | 	ROUND2(a, b, c, d, e, 40)
 99 | 	ROUND2(e, a, b, c, d, 41)
100 | 	ROUND2(d, e, a, b, c, 42)
101 | 	ROUND2(c, d, e, a, b, 43)
102 | 	ROUND2(b, c, d, e, a, 44)
103 | 	ROUND2(a, b, c, d, e, 45)
104 | 	ROUND2(e, a, b, c, d, 46)
105 | 	ROUND2(d, e, a, b, c, 47)
106 | 	ROUND2(c, d, e, a, b, 48)
107 | 	ROUND2(b, c, d, e, a, 49)
108 | 	ROUND2(a, b, c, d, e, 50)
109 | 	ROUND2(e, a, b, c, d, 51)
110 | 	ROUND2(d, e, a, b, c, 52)
111 | 	ROUND2(c, d, e, a, b, 53)
112 | 	ROUND2(b, c, d, e, a, 54)
113 | 	ROUND2(a, b, c, d, e, 55)
114 | 	ROUND2(e, a, b, c, d, 56)
115 | 	ROUND2(d, e, a, b, c, 57)
116 | 	ROUND2(c, d, e, a, b, 58)
117 | 	ROUND2(b, c, d, e, a, 59)
118 | 	ROUND3(a, b, c, d, e, 60)
119 | 	ROUND3(e, a, b, c, d, 61)
120 | 	ROUND3(d, e, a, b, c, 62)
121 | 	ROUND3(c, d, e, a, b, 63)
122 | 	ROUND3(b, c, d, e, a, 64)
123 | 	ROUND3(a, b, c, d, e, 65)
124 | 	ROUND3(e, a, b, c, d, 66)
125 | 	ROUND3(d, e, a, b, c, 67)
126 | 	ROUND3(c, d, e, a, b, 68)
127 | 	ROUND3(b, c, d, e, a, 69)
128 | 	ROUND3(a, b, c, d, e, 70)
129 | 	ROUND3(e, a, b, c, d, 71)
130 | 	ROUND3(d, e, a, b, c, 72)
131 | 	ROUND3(c, d, e, a, b, 73)
132 | 	ROUND3(b, c, d, e, a, 74)
133 | 	ROUND3(a, b, c, d, e, 75)
134 | 	ROUND3(e, a, b, c, d, 76)
135 | 	ROUND3(d, e, a, b, c, 77)
136 | 	ROUND3(c, d, e, a, b, 78)
137 | 	ROUND3(b, c, d, e, a, 79)
138 | 	
139 | 	state[0] = 0U + state[0] + a;
140 | 	state[1] = 0U + state[1] + b;
141 | 	state[2] = 0U + state[2] + c;
142 | 	state[3] = 0U + state[3] + d;
143 | 	state[4] = 0U + state[4] + e;
144 | }
145 | 


--------------------------------------------------------------------------------
/tcpstream.c:
--------------------------------------------------------------------------------
  1 | //---------------------------------------------------------------------------------------------
  2 | //
  3 | // Copyright (c) 2015 fmad engineering llc 
  4 | //
  5 | // The MIT License (MIT) see LICENSE file for details 
  6 | //
  7 | // tcp stream exporter 
  8 | //
  9 | //---------------------------------------------------------------------------------------------
 10 | 
 11 | #include <stdio.h>
 12 | #include <stdint.h>
 13 | #include <stdlib.h>
 14 | #include <unistd.h>
 15 | #include <string.h>
 16 | #include <errno.h>
 17 | #include <sys/stat.h>
 18 | #include <sys/mman.h>
 19 | #include <sys/shm.h>
 20 | #include <fcntl.h>
 21 | #include <locale.h>
 22 | #include <assert.h>
 23 | #include <errno.h>
 24 | 
 25 | #include "fTypes.h"
 26 | #include "fProfile.h"
 27 | #include "tcpstream.h"
 28 | #include "fFile.h"
 29 | 
 30 | //---------------------------------------------------------------------------------------------
 31 | 
 32 | #define MAX_TCPSEGMENT		(1024*1024)
 33 | typedef struct 
 34 | {
 35 | 	u64			TS;
 36 | 	u32			SeqNo;
 37 | 	u32			Length;
 38 | 
 39 | 	u8*			Payload;
 40 | 
 41 | } TCPBuffer_t;
 42 | 
 43 | typedef struct TCPStream_t 
 44 | {
 45 | 	u32				ID;						// unique ID for this stream
 46 | 	struct fFile_t*	File;
 47 | 	u32				FlowID;
 48 | 	u32				FlowHash;				// use the same has to index the tcp stream cache
 49 | 
 50 | 	u32				SeqNo;					// current/next expected tcp seq number
 51 | 
 52 | 	u32				BufferListPos;
 53 | 	u32				BufferListMax;
 54 | 	TCPBuffer_t*	BufferList[1*1024];
 55 | 
 56 | 	char			Path[1024];				// full path 
 57 | 	u64				LastTSC;				// cycle counter when this file was last accessed 
 58 | 	u64				LastTS;
 59 | 
 60 | 	bool			IsFileCreate;			// has the output file been created yet. e.g. for
 61 | 											// file truncation
 62 | 
 63 | 	u64				WritePos;				// number of bytes written to output (including stream headers)
 64 | 	u32				WindowScale;			// tcp window scaling value
 65 | 
 66 | 	bool			IsTrace;				// debug this stream
 67 | 
 68 | 	u32				PktDrop;				// number of packets dropped
 69 | 	u32				PktOOO;					// number of packets out of order 
 70 | 	u32				PktTotal;				// total number of packets added
 71 | 
 72 | } TCPStream_t;
 73 | 
 74 | // public header output for each stream
 75 | 
 76 | #define TCPHEADER_FLAG_RESET		(1<<15)		// indicates tcp stream has been reset
 77 | #define TCPHEADER_FLAG_REASSEMBLE	(1<<14)		// indicates this payload has been re-assembled 
 78 | 
 79 | typedef struct
 80 | {
 81 | 	u64		TS;					// timestamp of last byte	
 82 | 	u16		Length;				// number of bytes in this packet
 83 | 	u16		StreamID;			// unique id per flow
 84 | 
 85 | 	u32		SeqNo;				// current tcp seq no
 86 | 	u32		AckNo;				// current tcp ack no
 87 | 
 88 | 	u32		Window;				// tcp window size in bytes
 89 | 	u16		Flag;				// flags
 90 | 	u16		CRC;				// checksum
 91 | 
 92 | } TCPOutputHeader_t;
 93 | 
 94 | //---------------------------------------------------------------------------------------------
 95 | 
 96 | extern u64			g_TotalMemory;
 97 | extern u64			g_TotalMemoryTCP;
 98 | extern bool			g_Verbose;
 99 | extern bool			g_OutputPCAP;
100 | 
101 | //---------------------------------------------------------------------------------------------
102 | 
103 | static int			s_StreamCnt				= 0;
104 | static int			s_StreamMax				= 4*1024*1024;
105 | static TCPStream_t**s_Stream				= NULL;
106 | 
107 | extern bool			g_EnableTCPHeader;
108 | 
109 | static u64			s_TCPBufferMemoryTotal	= 0;				// total amount of gapped tcp data
110 | static u64			s_TCPBufferPacketTotal	= 0;				// total number of packets unclaimed
111 | static u32			s_TCPBufferMaxDepth		= 0;				// max depth of an OOO buffer
112 | 
113 | static u64			s_TotalByte 			= 0;				// total number of bytes outputed in streams
114 | static u64			s_TotalPkt 				= 0;				// total number of packets output 
115 | 
116 | //---------------------------------------------------------------------------------------------
117 | 
118 | void fTCPStream_MaxFlow(u32 MaxFlow)
119 | {
120 | 	s_StreamMax = MaxFlow;
121 | 	s_Stream    = (TCPStream_t**)malloc( s_StreamMax * sizeof(TCPStream_t*));
122 | 	memset(s_Stream, 0, s_StreamMax * sizeof(TCPStream_t*));
123 | 	fprintf(stderr, "TCP Flow Max: %i\n", MaxFlow);
124 | }
125 | 
126 | //---------------------------------------------------------------------------------------------
127 | // converts tcpflags to our internal version
128 | static u32 StreamTCPFlag(u32 TCPFlag)
129 | {
130 | 	u32 Flag = 0;
131 | /*
132 | 	// syn
133 | 	if (TCP_FLAG_SYN(TCPFlag) && (!TCP_FLAG_ACK(TCPFlag)))
134 | 	{
135 | 		Flag = TCPHEADER_FLAG_SYN;
136 | 	}
137 | 	// syn.ack
138 | 	if (TCP_FLAG_SYN(TCPFlag) && (TCP_FLAG_ACK(TCPFlag)))
139 | 	{
140 | 		Flag = TCPHEADER_FLAG_SYN | TCPHEADER_FLAG_ACK;
141 | 	}
142 | 	if (TCP_FLAG_FIN(TCPFlag))
143 | 	{
144 | 		Flag = TCPHEADER_FLAG_FIN;
145 | 	}
146 | */
147 | 
148 | 	Flag = swap16(TCPFlag) & 0xFF;
149 | 	return Flag;
150 | }
151 | 
152 | //---------------------------------------------------------------------------------------------
153 | 
154 | TCPStream_t* fTCPStream_Init(u64 MemorySize, char* OutputName, u32 FlowID, u32 FlowHash, u64 TS)
155 | {
156 | 	TCPStream_t* TCPStream = malloc( sizeof( TCPStream_t) );
157 | 	assert(TCPStream != NULL);
158 | 	memset(TCPStream, 0, sizeof( TCPStream_t) );
159 | 	g_TotalMemory += sizeof( TCPStream_t);
160 | 
161 | 	TCPStream->ID				= s_StreamCnt++;
162 | 	assert(s_StreamCnt < s_StreamMax);
163 | 
164 | 	s_Stream[TCPStream->ID] 	= TCPStream;
165 | 
166 | 	TCPStream->SeqNo 			= 0;
167 | 	TCPStream->BufferListPos 	= 0;
168 | 	TCPStream->BufferListMax 	= 1*1024;
169 | 
170 | 	strncpy(TCPStream->Path, OutputName, sizeof(TCPStream->Path));
171 | 	//TCPStream->fd = -1; 
172 | 	TCPStream->File 			= NULL;
173 | 
174 | 	TCPStream->FlowID 			= FlowID;
175 | 	TCPStream->FlowHash	 		= FlowHash;
176 | 
177 | 	TCPStream->IsFileCreate		= false;
178 | 
179 | 	TCPStream->WindowScale		= 1;
180 | 
181 | 	// if want raw PCAP only
182 | 	if (g_OutputPCAP)
183 | 	{
184 | 		sprintf(TCPStream->Path, "%s.pcap", OutputName);
185 | 
186 | 		TCPStream->File = fFile_Open(TCPStream->Path, "w");
187 | 		if (!TCPStream->File)
188 | 		{
189 | 			fprintf(stderr, "failed to create file [%s]\n", TCPStream->Path);
190 | 			return 0;
191 | 		}
192 | 
193 | 		PCAPHeader_t Header;
194 | 		Header.Magic 	= PCAPHEADER_MAGIC_NANO;
195 | 		Header.Major 	= PCAPHEADER_MAJOR;
196 | 		Header.Minor 	= PCAPHEADER_MINOR;
197 | 		Header.TimeZone = 0;
198 | 		Header.SigFlag 	= 0;
199 | 		Header.SnapLen 	= 0;
200 | 		Header.Link		= PCAPHEADER_LINK_ETHERNET;
201 | 
202 | 		fFile_Write(TCPStream->File, &Header, sizeof(Header), true);
203 | 	}
204 | 
205 | 
206 | 
207 | 	//printf("[%s] FlowID:%i TCP Stream: [%s]\n", FormatTS(TS), FlowID, OutputName);
208 | 	return TCPStream; 
209 | }
210 | 
211 | //---------------------------------------------------------------------------------------------
212 | 
213 | static void fTCPStream_Open(TCPStream_t* S)
214 | {
215 | 	fProfile_Start(10, "TCP PktAdd Open");
216 | 
217 | 	if (S->File == NULL)
218 | 	{
219 | 		fProfile_Start(15, "TCP IO Open");
220 | 
221 | 		//S->File = fopen64(S->Path, "w+"); 
222 | 		S->File = fFile_Open(S->Path, NULL); 
223 | 		fProfile_Stop(15);
224 | 
225 | 		if (S->File == NULL) 
226 | 		{
227 | 			fprintf(stderr, "failed to create output file [%s] errno:%i (%s)\n", S->Path, errno, strerror(errno));
228 | 			exit(-1);
229 | 		}
230 | 	}
231 | 	fProfile_Stop(10);
232 | }
233 | 
234 | //---------------------------------------------------------------------------------------------
235 | 
236 | void fTCPStream_Close(struct TCPStream_t* S)
237 | {
238 | 	if(!S) return;
239 | 
240 | 	//close(S->fd);
241 | 	//fclose(S->File);
242 | 	fFile_Close(S->File);
243 | 	S->File = NULL;
244 | 
245 | //	printf("[%s] OOO PacketList: %i/%i\n", S->Path, S->BufferListPos, S->BufferListMax);
246 | 
247 | 	memset(S, 0, sizeof(TCPStream_t));
248 | 	free(S);
249 | }
250 | 
251 | //---------------------------------------------------------------------------------------------
252 | // periodically flush to disk
253 | void fTCPStream_Flush(struct TCPStream_t* S)
254 | {
255 | 	if (!S) return;
256 | 	//fflush(S->File);
257 | 	fFile_Flush(S->File);
258 | }
259 | 
260 | //---------------------------------------------------------------------------------------------
261 | // write tcp header only to disk
262 | void fTCPStream_OutputHeader(TCPStream_t* S, TCPOutputHeader_t* Header) 
263 | {
264 | 	// ensure file is open
265 | 	fTCPStream_Open(S);	
266 | 
267 | 	fProfile_Start(13, "TCP PktAdd OutputHeader");
268 | 
269 | 	//int wlen = write(S->fd, &Header, sizeof(Header));
270 | 	//int wlen = fwrite(&Header, sizeof(Header), 1, S->File);
271 | 	fFile_Write(S->File, Header, sizeof(TCPOutputHeader_t), false);
272 | 
273 | 	S->WritePos += sizeof(TCPOutputHeader_t);
274 | 
275 | 	S->LastTSC = rdtsc();
276 | 
277 | 	fProfile_Stop(13);
278 | }
279 | 
280 | //---------------------------------------------------------------------------------------------
281 | 
282 | void fTCPStream_OutputPayload(TCPStream_t* S, u64 TS, u32 Length, u8* Payload, u32 Flag, u32 SeqNo, u32 AckNo, u32 WindowSize)
283 | {
284 | 
285 | 	// ensure file is open 
286 | 	fTCPStream_Open(S);	
287 | 
288 | 	fProfile_Start(12, "TCP PktAdd OutputPayload");
289 | 	S->SeqNo += Length;
290 | 
291 | 	int rlen;
292 | 	if (g_EnableTCPHeader)
293 | 	{
294 | 		TCPOutputHeader_t Header;
295 | 		Header.TS 		= TS;
296 | 		Header.Length 	= Length;
297 | 		Header.StreamID	= S->FlowID;
298 | 		Header.SeqNo	= SeqNo; 
299 | 		Header.AckNo	= AckNo; 
300 | 		Header.Window	= WindowSize; 
301 | 		Header.Flag		= Flag; 
302 | 		Header.CRC		= 0;
303 | 		Header.CRC		+= Header.TS;
304 | 		Header.CRC		+= Header.Length;
305 | 		Header.CRC		+= Header.StreamID;
306 | 		Header.CRC		+= Header.SeqNo;
307 | 		Header.CRC		+= Header.AckNo;
308 | 		Header.CRC		+= Header.Window;
309 | 		Header.CRC		+= Header.Flag;
310 | 
311 | 		//rlen = write(S->fd, &Header, sizeof(Header));
312 | 		//rlen = fwrite(&Header, sizeof(Header), 1, S->File);
313 | 		fFile_Write(S->File, &Header, sizeof(TCPOutputHeader_t), false);
314 | 
315 | 		S->WritePos += sizeof(TCPOutputHeader_t);
316 | 	}
317 | 
318 | 	/*
319 | 	if (S->IsTrace)
320 | 	{
321 | 		for (int i=0; i < Length; i++)
322 | 		{
323 | 			if (i % 16 == 0) printf("\n");
324 | 			printf("%02x ", Payload[i]);
325 | 		}
326 | 		printf("\n");
327 | 	}
328 | 	*/
329 | 
330 | 
331 | 	//rlen = write(S->fd, Payload, Length);
332 | 	//rlen = fwrite(Payload, Length, 1, S->File);
333 | 	fFile_Write(S->File, Payload, Length, true);
334 | 
335 | 	S->WritePos += Length; 
336 | 
337 | 	S->LastTSC = rdtsc();
338 | 
339 | 	// total TCP bytes output
340 | 	s_TotalByte += Length;
341 | 	s_TotalPkt	+= 1;
342 | 
343 | 	fProfile_Stop(12);
344 | }
345 | 
346 | //---------------------------------------------------------------------------------------------
347 | 
348 | static void fTCPStream_Reassembly(TCPStream_t* S, u64 TS, u32 Flag)
349 | {
350 | 	fProfile_Start(11, "TCP PktAdd Reassembly");
351 | 	while (true)
352 | 	{
353 | 		bool Hit = false;
354 | 		for (int i=0; i < S->BufferListPos; i++)
355 | 		{
356 | 			TCPBuffer_t* Buffer = S->BufferList[i];
357 | 
358 | 			s32 dSeqStart = Buffer->SeqNo - S->SeqNo;
359 | 			s32 dSeqEnd   = Buffer->SeqNo - S->SeqNo + Buffer->Length;
360 | 
361 | 			// perfectly aligned start packet
362 | 			if (dSeqStart == 0)
363 | 			{
364 | 				if (g_Verbose) printf("[%s] [%s] reassembly hit Seq:%i:%i : %i\n", FormatTS(Buffer->TS), S->Path, S->SeqNo, S->SeqNo + Buffer->Length, S->BufferListPos);
365 | 				fTCPStream_OutputPayload(S, TS, Buffer->Length, Buffer->Payload, Flag | TCPHEADER_FLAG_REASSEMBLE, S->SeqNo, 0, 0);
366 | 				Hit = true;
367 | 			}
368 | 			// redundant packet fully before the current packet 
369 | 			else if ((dSeqStart < 0) && (dSeqEnd < 0))
370 | 			{
371 | 				if (g_Verbose) printf("[%s] [%s] redundant packet hit Seq:%i BufferSeq:%i : %i\n", FormatTS(Buffer->TS), S->Path, S->SeqNo, Buffer->SeqNo, S->BufferListPos);
372 | 				Hit = true;
373 | 			}
374 | 			// partialy straddled packet. started before the current Seq number but adds to the seq no
375 | 			else if ((dSeqStart < 0) && (dSeqEnd > 0))
376 | 			{
377 | 				if (g_Verbose) printf("[%s] [%s] partial packet hit Seq:%i BufferSeq:%i : %i %i : %i\n", FormatTS(Buffer->TS), S->Path, S->SeqNo, Buffer->SeqNo, dSeqStart, dSeqEnd, S->BufferListPos);
378 | 				fTCPStream_OutputPayload(S, TS, dSeqEnd, Buffer->Payload  - dSeqStart, Flag | TCPHEADER_FLAG_REASSEMBLE, S->SeqNo, 0, 0);
379 | 
380 | 				Hit = true;
381 | 			}
382 | 
383 | 			// free and remove buffer 
384 | 			if (Hit)
385 | 			{
386 | 				free(Buffer->Payload);
387 | 				Buffer->Payload = NULL;	
388 | 				free(Buffer);
389 | 
390 | 				s_TCPBufferMemoryTotal -= sizeof(TCPBuffer_t);
391 | 
392 | 				for (int j=i; j < S->BufferListPos; j++)
393 | 				{
394 | 					S->BufferList[j] = S->BufferList[j+1];
395 | 				}
396 | 				S->BufferListPos--;
397 | 				break;
398 | 			}
399 | 		}
400 | 		if (!Hit) break;
401 | 	}
402 | 	fProfile_Stop(11);
403 | }
404 | 
405 | //---------------------------------------------------------------------------------------------
406 | 
407 | void fTCPStream_PacketAdd(TCPStream_t* S, u64 TS, TCPHeader_t* TCP, s32 Length, u8* Payload, PCAPPacket_t* Pkt)
408 | {
409 | 	// raw PCAP output only 
410 | 	if (g_OutputPCAP)
411 | 	{
412 | 		// closed so re-open
413 | 		if (!S->File)
414 | 		{
415 | 			S->File = fFile_Open(S->Path, NULL); 
416 | 			if (S->File == NULL)
417 | 			{
418 | 				fprintf(stderr, "failed to open output file [%s]\n", S->Path);
419 | 			}
420 | 		}
421 | 		fFile_Write(S->File, Pkt, sizeof(PCAPPacket_t) + Pkt->LengthCapture, true);
422 | 		return;
423 | 	}
424 | 
425 | 	// if length is invalid then drop the packet
426 | 	if (Length < 0)
427 | 	{
428 | 		printf("ERROR: TCP Packet Length Negative!?: %s %s : %i\n", FormatTS(TS), S->Path, Length);
429 | 		return;
430 | 	}
431 | 
432 | 	//printf("tcp len: %i %08x %08x\n", Length, swap32(TCP->SeqNo), swap32(TCP->AckNo));
433 | 	u32 WindowSize 	= swap16( TCP->Window) * S->WindowScale;
434 | 	u32 SeqNo 		= swap32( TCP->SeqNo );
435 | 	u32 AckNo 		= swap32( TCP->AckNo );
436 | 
437 | 	// last packet TS	
438 | 	S->LastTS 		= TS; 
439 | 
440 | 	u32 Flag 		= StreamTCPFlag(TCP->Flags); 
441 | 	if (TCP_FLAG_SYN(TCP->Flags))
442 | 	{
443 | 		if (g_Verbose) printf("[%s] [%s] got syn\n", FormatTS(TS), S->Path);
444 | 		S->SeqNo = SeqNo + 1;
445 | 
446 | 		fProfile_Start(9, "TCP PktAdd release");
447 | 
448 | 		// release all reassembly buffer data . assumption is this is now a new tcp stream
449 | 		for (int i=0; i < S->BufferListPos; i++)
450 | 		{
451 | 			TCPBuffer_t* Buffer = S->BufferList[i];
452 | 
453 | 			free(Buffer->Payload);
454 | 			Buffer->Payload = NULL;
455 | 
456 | 			free(Buffer);
457 | 
458 | 			s_TCPBufferMemoryTotal  -= sizeof(TCPBuffer_t) + Length;
459 | 			s_TCPBufferPacketTotal 	-= 1;
460 | 		}
461 | 		S->BufferListPos = 0;
462 | 
463 | 		fProfile_Stop(9);
464 | 
465 | 		// parse options
466 | 		u32 OLen = (4*swap16(TCP->Flags) >> 12) - 20;
467 | 		u8* Option = (u8*)(TCP + 1);
468 | 		for (int o=0; o < OLen; o++)
469 | 		{
470 | 			// end of options
471 | 			if (Option[o] == 0) break;
472 | 
473 | 			// nop
474 | 			if (Option[o] == 1) continue;
475 | 
476 | 			// MSS
477 | 			if ((Option[o+0] == 2) && (Option[o+1] == 4))
478 | 			{
479 | 				u32 MSS = ((u32)Option[o+2]<< 8) | (u32)Option[o+3];
480 | 				if (g_Verbose) printf("TCP MSS: %i\n", MSS);
481 | 				o+= 2;
482 | 			}
483 | 			// Window Scale 
484 | 			if ((Option[o+0] == 3) && (Option[o+1] == 3))
485 | 			{
486 | 				u32 Scale = Option[o+2];
487 | 				if (g_Verbose) printf("TCP WindowScale: %i\n", Scale);
488 | 				o+= 1;
489 | 				S->WindowScale = Scale;
490 | 			}
491 | 			// Selective Ack 
492 | 			if ((Option[o+0] == 4) && (Option[o+1] == 2))
493 | 			{
494 | 				if (g_Verbose) printf("TCP SelAck\n");
495 | 			}
496 | 			//printf("tcp offset: [%4i] %02x\n", o, Option[o]);
497 | 		}
498 | 
499 | 		// indicate stream reset
500 | 		if (g_EnableTCPHeader)
501 | 		{
502 | 			TCPOutputHeader_t Header;
503 | 			Header.TS 		= TS;
504 | 			Header.Length 	= 0;
505 | 			Header.StreamID	= S->FlowID;
506 | 			Header.SeqNo	= 0; 
507 | 			Header.AckNo	= 0; 
508 | 			Header.Window	= 0; 
509 | 			Header.Flag		= Flag | TCPHEADER_FLAG_RESET; 
510 | 			Header.CRC		= 0;
511 | 			Header.CRC		+= Header.TS;
512 | 			Header.CRC		+= Header.Length;
513 | 			Header.CRC		+= Header.StreamID;
514 | 			Header.CRC		+= Header.SeqNo;
515 | 			Header.CRC		+= Header.AckNo;
516 | 			Header.CRC		+= Header.Window;
517 | 			Header.CRC		+= Header.Flag;
518 | 
519 | 			fTCPStream_OutputHeader(S, &Header);
520 | 		}
521 | 	}
522 | 
523 | 	if (Length == 0)
524 | 	{
525 | 		// usuall ack with no payload	
526 | 		if (g_EnableTCPHeader)
527 | 		{
528 | 			TCPOutputHeader_t Header;
529 | 			Header.TS 		= TS;
530 | 			Header.Length 	= Length;
531 | 			Header.StreamID	= S->FlowID;
532 | 			Header.SeqNo	= SeqNo; 
533 | 			Header.AckNo	= AckNo; 
534 | 			Header.Window	= WindowSize; 
535 | 			Header.Flag		= Flag; 
536 | 			Header.CRC		= 0;
537 | 			Header.CRC		+= Header.TS;
538 | 			Header.CRC		+= Header.Length;
539 | 			Header.CRC		+= Header.StreamID;
540 | 			Header.CRC		+= Header.SeqNo;
541 | 			Header.CRC		+= Header.AckNo;
542 | 			Header.CRC		+= Header.Window;
543 | 			Header.CRC		+= Header.Flag;
544 | 
545 | 			fTCPStream_OutputHeader(S, &Header);
546 | 		}
547 | 	}
548 | 	else
549 | 	{
550 | 		s32 dSeqNo = SeqNo - S->SeqNo;
551 | 		if (dSeqNo == 0)
552 | 		{
553 | 			fTCPStream_OutputPayload(S, TS, Length, Payload, Flag, SeqNo, AckNo, WindowSize);
554 | 			if (S->BufferListPos > 0)
555 | 			{
556 | 				if (g_Verbose) printf("[%s] [%s] resend hit Seq:%08x : %i\n", FormatTS(TS), S->Path, SeqNo, S->BufferListPos);
557 | 
558 | 				// check for reassembly
559 | 				fTCPStream_Reassembly(S, TS, Flag);
560 | 			}
561 | 		}
562 | 		else
563 | 		{
564 | 			fProfile_Start(7, "TCP PktAdd OOO");
565 | 
566 | 			// reassembly packet thats not aligned but close to the current seq no
567 | 			s32 dRemain = (SeqNo + Length) - S->SeqNo;
568 | 			if ((SeqNo < S->SeqNo) && (dRemain > 0) && (dRemain < 8*1024))
569 | 			{
570 | 				if (g_Verbose) 
571 | 				{
572 | 					printf("[%s] [%s] unaligned seq resend: %i [%llx:%llx]\n",
573 | 						FormatTS(TS),
574 | 						S->Path,
575 | 						dRemain,
576 | 						S->WritePos,
577 | 						S->WritePos + dRemain
578 | 					   );
579 | 				}
580 | 
581 | 				s32 PayloadOffset = S->SeqNo - SeqNo; 
582 | 				if (PayloadOffset <= 0)
583 | 				{
584 | 					printf("[%s] [%s] ERROR: TCP OOO reassembly PayloadOffset:%i SeqNo:%08x %08x dRemain:%i Length:%i\n", FormatTS(TS), S->Path, PayloadOffset, S->SeqNo, SeqNo, dRemain, Length);
585 | 				}
586 | 				else
587 | 				{
588 | 					fTCPStream_OutputPayload(S, TS, dRemain, Payload + PayloadOffset, Flag | TCPHEADER_FLAG_REASSEMBLE, S->SeqNo, 0, 0);
589 | 
590 | 					// check re-assembly buffer 
591 | 					fTCPStream_Reassembly(S, TS, Flag);
592 | 				}
593 | 			}
594 | 
595 | 			// stop processing if too many gaps 
596 | 			else if (S->BufferListPos < S->BufferListMax)
597 | 			{
598 | 				TCPBuffer_t* B = (TCPBuffer_t*)malloc( sizeof(TCPBuffer_t) );
599 | 				assert(B != NULL);
600 | 
601 | 				s_TCPBufferMemoryTotal  += sizeof(TCPBuffer_t) + Length;
602 | 				s_TCPBufferPacketTotal 	+= 1;
603 | 				g_TotalMemory 			+= sizeof(TCPBuffer_t) + Length;
604 | 				g_TotalMemoryTCP 		+= sizeof(TCPBuffer_t) + Length;
605 | 
606 | 				memset(B, 0, sizeof(TCPBuffer_t));
607 | 				if (Length >= MAX_TCPSEGMENT)
608 | 				{
609 | 					printf("TCP length overflow: %i %i\n", Length,  MAX_TCPSEGMENT);
610 | 					return;
611 | 				}
612 | 				//assert(Length < MAX_TCPSEGMENT);
613 | 
614 | 				B->TS 		= TS;
615 | 				B->SeqNo 	= SeqNo;
616 | 				B->Length 	= Length;
617 | 				B->Payload	= malloc( Length );
618 | 
619 | 				assert(B->Payload != NULL);
620 | 				memcpy(B->Payload, Payload, Length);
621 | 
622 | 				S->BufferList[ S->BufferListPos++ ] = B;	
623 | 
624 | 				if (g_Verbose)
625 | 				{
626 | 					printf("[%s] [%s] tcp gap Seq:%08x PktSeq:%08x delta %i OOPkts:%i SEnd:%08x\n", 
627 | 						FormatTS(TS), 
628 | 						S->Path, 
629 | 						S->SeqNo, 
630 | 						SeqNo, 
631 | 						dSeqNo,
632 | 						S->BufferListPos,
633 | 						SeqNo + Length);
634 | 					printf("[%s] TCP Buffer: %lliMB %.fK Pkts\n", FormatTS(TS), s_TCPBufferMemoryTotal / kMB(1), s_TCPBufferPacketTotal / 1e3 ); 
635 | 				}
636 | 
637 | 				// special case when capture begins mid-stream
638 | 				if (S->SeqNo == 0)
639 | 				{
640 | 					if (g_Verbose) printf("[%s] %s midstream start Len:%i\n", FormatTS(TS), S->Path, Length); 
641 | 					S->SeqNo = swap32(TCP->SeqNo) + Length;
642 | 				}
643 | 			}
644 | 			else
645 | 			{
646 | 				// no space for re-assembly so drop it
647 | 				S->PktDrop++;
648 | 			}
649 | 
650 | 			// number of packets out of order
651 | 			S->PktOOO++;;
652 | 
653 | 			fProfile_Stop(7);
654 | 		}
655 | 	}
656 | 
657 | 	// toatl number of packets
658 | 	S->PktTotal++;
659 | }
660 | 
661 | //---------------------------------------------------------------------------------------------
662 | // dump tcp stream export stats
663 | void fTCPStream_Dump(u64 TS)
664 | {
665 | 	static u64 LastStreamCnt = 0;
666 | 
667 | 	fprintf(stderr, "TCPStream:\n");
668 | 	fprintf(stderr, "  TotalPkts     : %16lli\n", s_TotalPkt);
669 | 	fprintf(stderr, "  TotalByte     : %16lli\n", s_TotalByte);
670 | 
671 | 	fprintf(stderr, "  OOO Packets   : %16lli\n", s_TCPBufferPacketTotal);
672 | 
673 | 	s64 NewStreamCnt = s_StreamCnt - LastStreamCnt;
674 | 	fprintf(stderr, "  StreamCnt     : %16i\n", s_StreamCnt);
675 | 	fprintf(stderr, "  StreamNew     : %16lli\n", NewStreamCnt);
676 | 
677 | 	LastStreamCnt = s_StreamCnt;
678 | 
679 | 	u32 StreamCnt 		= 0;
680 | 	u32 StreamInactive 	= 0;
681 | 	u32 StreamClose 	= 0;
682 | 	u64 TSC				= rdtsc();
683 | 
684 | 	u32 Histo[1024];
685 | 	memset(Histo, 0, sizeof(Histo) );
686 | 
687 | 	// garbage collect old streams
688 | 	for (int i=0; i < s_StreamCnt; i++)
689 | 	{
690 | 		TCPStream_t* S = s_Stream[i];
691 | 		assert(S != NULL);
692 | 
693 | 		StreamCnt++;
694 | 
695 | 		// stream is inactive
696 | 		if (!S->File)
697 | 		{
698 | 			StreamInactive++;
699 | 			continue;
700 | 		}
701 | 
702 | 		u32 Index = S->WritePos / (16*1024);
703 | 		if (Index > 1023) Index = 1023;
704 | 		Histo[Index]++;
705 | 
706 | 		// candidate for closing
707 | 		// no activity in the last 5 min 
708 | 		s64 dTS = TS - S->LastTS; 
709 | 		if (dTS > 60e9 * 5)
710 | 		{
711 | 			fProfile_Start(14, "TCP IO Close");
712 | 			fFile_Close(S->File);
713 | 			fProfile_Stop(14);
714 | 
715 | 			S->File = NULL;
716 | 
717 | 			// release 
718 | 			StreamClose++;
719 | 		}
720 | 	}
721 | 
722 | 	/*
723 | 	for (int i=0; i < 1024; i++)
724 | 	{
725 | 		if (Histo[i] == 0) continue;
726 | 		printf("%8i B : Cnt %8i\n", i*16*1024, Histo[i]);
727 | 	}
728 | 	*/
729 | 
730 | 	fprintf(stderr, "  StreamFound   : %16i\n", StreamCnt);
731 | 	fprintf(stderr, "  StreamInactive: %16i (%.3f)\n", StreamInactive, StreamInactive / (float)StreamCnt);
732 | 	fprintf(stderr, "  StreamActive  : %16i\n", StreamCnt - StreamInactive - StreamClose);
733 | 	fprintf(stderr, "  StreamClose   : %16i\n", StreamClose); 
734 | }
735 | 
736 | //---------------------------------------------------------------------------------------------
737 | // print per flow stats 
738 | void fTCPStream_FlowStats(TCPStream_t* S, FILE* Out)
739 | {
740 | 	if (!S) return;
741 | 
742 | 	fprintf(Out, " BufPos: %8i WritePos:%8i PktTotal:%8i PktOOO:%8i PktDrop:%8i", 
743 | 					S->BufferListPos, 
744 | 					S->WritePos,
745 | 					S->PktTotal,
746 | 					S->PktOOO,
747 | 					S->PktDrop
748 | 
749 | 	);
750 | }
751 | 


--------------------------------------------------------------------------------
/tcpstream.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FLOW_TCPSTREAM_H__
 2 | #define __FLOW_TCPSTREAM_H__
 3 | 
 4 | struct TCPStream_t;
 5 | 
 6 | void fTCPStream_MaxFlow				(u32 MaxFlow);
 7 | struct TCPStream_t* fTCPStream_Init	(u64 MemorySize, char* OutputName, u32 FlowID, u32 FlowHash, u64 TS);
 8 | void fTCPStream_PacketAdd			(struct TCPStream_t* S, u64 TS, TCPHeader_t* TCP, s32 PayloadLength, u8* Payload, PCAPPacket_t* Pkt);
 9 | void fTCPStream_Close				(struct TCPStream_t* S);
10 | void fTCPStream_Flush				(struct TCPStream_t* S);
11 | void fTCPStream_Dump				(u64 TS);
12 | void fTCPStream_FlowStats			(struct TCPStream_t* S, FILE* Out);
13 | 
14 | #endif
15 | 


--------------------------------------------------------------------------------
/udpstream.c:
--------------------------------------------------------------------------------
  1 | //---------------------------------------------------------------------------------------------
  2 | //
  3 | // Copyright (c) 2015, fmad engineering llc 
  4 | //
  5 | // The MIT License (MIT) see LICENSE file for details 
  6 | //
  7 | // udp stream exporter. kinda silly as UDP isnt really a stream. the utility of this
  8 | // is a single flow is extracted into a single file
  9 | //
 10 | //---------------------------------------------------------------------------------------------
 11 | 
 12 | #include <stdio.h>
 13 | #include <stdint.h>
 14 | #include <stdlib.h>
 15 | #include <unistd.h>
 16 | #include <string.h>
 17 | #include <errno.h>
 18 | #include <sys/stat.h>
 19 | #include <sys/mman.h>
 20 | #include <sys/shm.h>
 21 | #include <fcntl.h>
 22 | #include <locale.h>
 23 | #include <assert.h>
 24 | #include <errno.h>
 25 | 
 26 | #include "fTypes.h"
 27 | #include "fFile.h"
 28 | #include "udpstream.h"
 29 | 
 30 | //---------------------------------------------------------------------------------------------
 31 | 
 32 | extern bool			g_EnableUDPHeader;
 33 | extern bool			g_OutputPCAP;	
 34 | 
 35 | //---------------------------------------------------------------------------------------------
 36 | 
 37 | // public header output for each stream
 38 | 
 39 | typedef struct
 40 | {
 41 | 	u64		TS;							// timestamp of last byte	
 42 | 	u16		Length;						// number of bytes in this packet
 43 | 	u16		StreamID;					// unique id per flow
 44 | 
 45 | } OutputHeader_t;
 46 | 
 47 | typedef struct UDPStream_t
 48 | {
 49 | 	struct fFile_t*	F;					// output file handle
 50 | 	char			FileName[1024];		// file handle name
 51 | 	u32				FlowID;				// master flowid for this stream
 52 | 
 53 | } UDPStream_t;
 54 | 
 55 | //---------------------------------------------------------------------------------------------
 56 | 
 57 | UDPStream_t* fUDPStream_Init(char* FileName, u32 FlowID, u64 TS)
 58 | {
 59 | 	UDPStream_t* Stream = malloc(sizeof(UDPStream_t));
 60 | 	memset(Stream, 0, sizeof(UDPStream_t));
 61 | 
 62 | 	// append .pcap
 63 | 	u8 FilenamePCAP[2048];
 64 | 	if (g_OutputPCAP)
 65 | 	{
 66 | 		sprintf(FilenamePCAP, "%s.pcap", FileName);
 67 | 		FileName = FilenamePCAP;
 68 | 	}
 69 | 
 70 | 	Stream->F = fFile_Open(FileName, "w");
 71 | 	if (!Stream->F)
 72 | 	{
 73 | 		fprintf(stderr, "failed to create file [%s]\n", Stream->FileName);
 74 | 		return 0;
 75 | 	}
 76 | 	Stream->FlowID = FlowID;
 77 | 	printf("[%s] FlowID:%i UDP Stream: [%s]\n", FormatTS(TS), FlowID, FileName);
 78 | 
 79 | 	// if PCAP then write pcap header
 80 | 	if (g_OutputPCAP)
 81 | 	{
 82 | 		PCAPHeader_t Header;
 83 | 		Header.Magic 	= PCAPHEADER_MAGIC_NANO;
 84 | 		Header.Major 	= PCAPHEADER_MAJOR;
 85 | 		Header.Minor 	= PCAPHEADER_MINOR;
 86 | 		Header.TimeZone = 0;
 87 | 		Header.SigFlag 	= 0;
 88 | 		Header.SnapLen 	= 0;
 89 | 		Header.Link		= PCAPHEADER_LINK_ETHERNET;
 90 | 
 91 | 		fFile_Write(Stream->F, &Header, sizeof(Header), true);
 92 | 	}
 93 | 
 94 | 	return Stream;
 95 | }
 96 | 
 97 | //---------------------------------------------------------------------------------------------
 98 | 
 99 | void fUDPStream_Add(UDPStream_t* Stream, u64 TS, PCAPPacket_t* Pkt, UDPHeader_t* UDPHeader)
100 | {
101 | 	// output in PCAP format
102 | 	if (g_OutputPCAP)
103 | 	{
104 | 		fFile_Write(Stream->F, Pkt, sizeof(PCAPPacket_t) + Pkt->LengthCapture, true);
105 | 		return;
106 | 	}
107 | 
108 | 	// write only the UDP payload
109 | 	u32 Length = swap16(UDPHeader->Length);	
110 | 	if (Length >= 16*1024)
111 | 	{
112 | 		fprintf(stderr, "udp packet length invalid UDP Length:%i PacketLength:%i\n", Length, Pkt->Length);
113 | 		return;
114 | 	}
115 | 
116 | 	// ensure its caped at the packet length
117 | 	if (Length > Pkt->Length)
118 | 	{
119 | 		fprintf(stderr, "udp packet length truncated: Header %i Capture %i\n", Length, Pkt->Length);
120 | 		Length = Pkt->Length;
121 | 	}	
122 | 
123 | 	if (g_EnableUDPHeader)
124 | 	{
125 | 		OutputHeader_t Header;
126 | 		Header.TS 		= TS;
127 | 		Header.Length 	= sizeof(UDPHeader_t) + Length; 
128 | 		Header.StreamID	= Stream->FlowID;
129 | 		fFile_Write(Stream->F, &Header, sizeof(Header), false );
130 | 	}
131 | 
132 | 	// write the UDP header + payload 
133 | 	fFile_Write(Stream->F, UDPHeader, sizeof(UDPHeader_t) + Length, true);
134 | }
135 | 


--------------------------------------------------------------------------------
/udpstream.h:
--------------------------------------------------------------------------------
 1 | #ifndef __FLOW_UDPSTREAM_H__
 2 | #define __FLOW_UDPSTREAM_H__
 3 | 
 4 | struct UDPStream_t;
 5 | 
 6 | 
 7 | struct UDPStream_t* 	fUDPStream_Init(char* OutputName, u32 FlowID, u64 TS);
 8 | void 					fUDPStream_Add(struct UDPStream_t* S, u64 TS, PCAPPacket_t* Pkt, UDPHeader_t* UDPHeader);
 9 | #endif
10 | 


--------------------------------------------------------------------------------