├── .gitignore
├── Makefile
├── CMakeLists.txt
├── README
├── LICENSE
└── seqtest.c


/.gitignore:
--------------------------------------------------------------------------------
1 | seqtest
2 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # This is a Makefile - it works worth GNUmake on Linux, Solaris, and illumos.
 2 | # Use of cmake instaed is recommended, but if you lack cmake, this can help.
 3 | # To use legacy (non-GNU) make, do "make UNAME=`uname`"
 4 | 
 5 | UNAME		=$(shell uname)
 6 | 
 7 | CFLAGS_COMMON	=-std=gnu99 -Wall -Werror
 8 | CFLAGS_Linux	=-D _GNU_SOURCE -D _XOPEN_SOURCE=700
 9 | CFLAGS_SunOS	=-D __EXTENSIONS__ -D _XOPEN_SOURCE=600
10 | CFLAGS		+=$(CFLAGS_COMMON) $(CFLAGS_$(UNAME))
11 | 
12 | LDFLAGS_Linux	=-lrt -lm
13 | LDFLAGS_SunOS	=-lnsl -lsocket -lm -lrt -lpthread
14 | LDFLAGS		+=$(LDFLAGS_$(UNAME))
15 | 
16 | all: seqtest
17 | 
18 | seqtest: seqtest.c
19 | 	$(CC) $(CFLAGS) seqtest.c -o $@ $(LDFLAGS)
20 | 
21 | clean:
22 | 	$(RM) seqtest
23 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | #
 2 | # Copyright 2016 Lucera Financial Infrastructures, Inc.
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use file except in compliance with the License.
 6 | # You may obtain a copy of the license at
 7 | #
 8 | #  http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | # This program is used to stress test TCP connections, verifying that
16 | # ordering constraints are preserved across a connection.  The intent
17 | # is to validate correct function of a TCP proxy.
18 | #
19 | 
20 | cmake_minimum_required(VERSION 2.8)
21 | 
22 | project(seqtest C)
23 | 
24 | include(CheckLibraryExists)
25 | include(CheckFunctionExists)
26 | 
27 | add_executable(seqtest seqtest.c)
28 | 
29 | check_library_exists(socket getaddrinfo "" HAVE_LIBSOCKET)
30 | if (HAVE_LIBSOCKET)
31 |     target_link_libraries(seqtest socket)
32 |     LIST(APPEND CMAKE_REQUIRED_LIBRARIES socket)
33 | endif (HAVE_LIBSOCKET)
34 | 
35 | check_library_exists(nsl gethostbyname "" HAVE_LIBNSL)
36 | if (HAVE_LIBNSL)
37 |     target_link_libraries(seqtest nsl)
38 |     LIST(APPEND CMAKE_REQUIRED_LIBRARIES nsl)
39 | endif (HAVE_LIBNSL)
40 | 
41 | check_library_exists(rt clock_gettime "" HAVE_LIBRT)
42 | if (HAVE_LIBRT)
43 |     target_link_libraries(seqtest rt)
44 |     LIST(APPEND CMAKE_REQUIRED_LIBRARIES rt)
45 | endif (HAVE_LIBRT)
46 | 
47 | check_library_exists(pthread pthread_create "" HAVE_LIBPTHREAD)
48 | if (HAVE_LIBPTHREAD)
49 |     target_link_libraries(seqtest pthread)
50 |     LIST(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
51 | endif (HAVE_LIBPTHREAD)
52 | 
53 | check_library_exists(m sqrt "" HAVE_LIBM)
54 | if (HAVE_LIBM)
55 |     target_link_libraries(seqtest m)
56 |     LIST(APPEND CMAKE_REQUIRED_LIBRARIES m)
57 | endif (HAVE_LIBM)
58 | 
59 | check_function_exists(strlcpy HAVE_STRLCPY)
60 | if (HAVE_STRLCPY)
61 |     add_definitions(-DHAVE_STRLCPY)
62 | endif (HAVE_STRLCPY)
63 | 
64 | check_function_exists(gethrtime HAVE_GETHRTIME)
65 | if (HAVE_GETHRTIME)
66 |     add_definitions(-DHAVE_GETHRTIME)
67 | endif (HAVE_GETHRTIME)
68 | 
69 | check_function_exists(clock_gettime HAVE_CLOCK_GETTIME)
70 | if (HAVE_CLOCK_GETTIME)
71 |     add_definitions(-DHAVE_CLOCK_GETTIME)
72 | endif (HAVE_CLOCK_GETTIME)
73 | 
74 | check_function_exists(gettimeofday HAVE_GETTIMEOFDAY)
75 | if (HAVE_GETTIMEOFDAY)
76 |     add_definitions(-DHAVE_GETTIMEOFDAY)
77 | endif (HAVE_GETTIMEOFDAY)
78 | 
79 | install(TARGETS seqtest DESTINATION bin)
80 | 


--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
  1 | seqtest -- test TCP under load
  2 | 
  3 | seqtest is a program that is designed to verify correct end-to-end TCP
  4 | operation.  It can also do some limited performance measurement.  The biggest
  5 | advantage it has over alternative programs is that it allows one to alter
  6 | the traffic patterns and inject an element of randomness.  This can be used
  7 | to help uncover race conditions and other difficult-to-reproduce bugs.
  8 | 
  9 | When run with a fixed number of iterations (see the count option below), and
 10 | with replies, it will report the round trip latency between each message sent,
 11 | and when its reply is received.
 12 | 
 13 | It runs in three modes:
 14 | 
 15 | * sender mode.  In this mode seqtest sends a stream of packets, at some
 16 |   (randomized, possibly zero) interval to the replier.  It also runs a thread
 17 |   to wait for replies from the replier, and verifies that messages are
 18 |   delivered in order.  Most of the customization is done when running the
 19 |   sender side.
 20 | 
 21 | * synchoronous sender mode.  THis mode is just like sender mode, except that
 22 |   each thread waits for the reply before sending the next message.  This is
 23 |   a better measurement of latency.
 24 | 
 25 | * replier mode.  In this mode, seqtest accepts incoming TCP connections,
 26 |   analyzes them for correctness (in order delivery, etc.) and optionally
 27 |   sends a reply.  The size of the reply can vary, as well as the frequency
 28 |   with which replies are made.  A random delay can also be inserted before
 29 |   sending the reply.
 30 | 
 31 | 
 32 | In sender mode, the synopsis is as follows:
 33 | 
 34 |     seqtest -s [-d] [-o <option>=<value>[,<option>=<value>...] <address>...
 35 | 
 36 | Synchronous sender juse replaces -s for -S:
 37 | 
 38 |     seqtest -S [-d] [-o <option>=<value>[,<option>=<value>...] <address>...
 39 | 
 40 | Where <address> has one of two forms:
 41 | 
 42 |     <remote_addr>:<remote_port>
 43 |     <local_addr>,<remote_addr>:<remote_port> (bind to local_addr)
 44 | 
 45 | The options are name value pairs; the following are defined:
 46 | 
 47 |     ssize=<num>		The size of message payload to send.  Will be
 48 | 			rounded up to 40 bytes if less than that is specified,
 49 | 			as seqtest needs 40 bytes of header information on
 50 | 			each message.  Currently messages are limited to 8000
 51 | 			bytes maximum, as well.
 52 | 
 53 |     ssize_min=<num>	A minimum value to use for send payload sizes.  If
 54 | 			this is specified, then each sent message will have a
 55 | 			random payload size of at least this size.
 56 | 
 57 |     ssize_max=<num>	A maximum value to use for send paylod size.
 58 | 
 59 |     rsize=<num>		The size of reply payloads to send.  As with ssize,
 60 | 			the value must be between 40 and 8000, inclusive.
 61 | 
 62 |     rsize_min=<num>	A minimum reply payload size, used when randomly
 63 | 			choosing reply payload sizes.  Each reply's size
 64 | 			is generated randomly.
 65 | 
 66 |     rsize_max=<num>	A maximum reply payload sized, used when randomly
 67 | 			choosing reply payload sizes.
 68 | 
 69 |     threads=<num>	The number of parallel sending threads to spawn.
 70 | 			There actually be twice this number of threads
 71 | 			spawned. One for each connection is spawned for
 72 | 			sending messages and another for receiving replies.
 73 | 			This corresponds to the number of concurrent TCP
 74 | 			flows to process.
 75 | 
 76 |     sdelay=<ns>		A number of nanoseconds (can be zero) to wait
 77 | 			between sending messages.  If the number is at least
 78 | 			1000000 then a sleeping delay will be used, otherwise
 79 | 			the sender will spin on the CPU.
 80 | 
 81 |     sdelay_min=<ns>	Allows the delay before sending a message to be
 82 | 			randomized each time.  This is a minimum number
 83 | 			of nsec.
 84 | 
 85 |     sdelay_max=<ns>	Maximum time to wait before sending.
 86 | 
 87 |     rdelay=<ns>		As with sdelay, this is a wait time, but it is the
 88 | 			time that a replier should wait before sending
 89 | 			a reply.  This can be randomized for each
 90 | 			reply with rdelay_min and rdelay_max.  This simulates
 91 | 			work being done before sending a reply.
 92 | 
 93 |     rinterval=<num>     The interval between replies, as a number of messages
 94 | 			received.  For example, if 2, then a reply will only
 95 | 			be sent every other message.  Defaults to 1.  If 0,
 96 | 			then no replies are sent at all.
 97 | 
 98 |     count=<num>		The number of messages each sending thread should send.
 99 | 
100 | The address(es) are IP address (or hostname) and port pairs separated by
101 | a colon to use for connecting.  If a name resolves to multiple IP addresses,
102 | then multiple senders will be spawned by default, one for each resolved IP.
103 | (The exception would be if a value of threads is specified in above.  In
104 | that case exactly that many senders would be spawned, balancing the load
105 | amongst resolved addreses.)  For example, to send traffic to localhost port 88,
106 | use 127.0.0.1:88 or "localhost:88".  Note that the addresses can be either IPv4
107 | or IPv6.
108 | 
109 | The receiver synopsis is simpler:
110 | 
111 |     seqtest -r <address>...
112 | 
113 | In this case the list of addresses is specified just as with the sender, but
114 | the addresses must be addresses local to the host where seqtest is run.  The
115 | program will bind and listen for incoming connections on these addresses,
116 | and reply according to the specifications of received messages.
117 | 
118 | 
119 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "[]"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright [yyyy] [name of copyright owner]
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 


--------------------------------------------------------------------------------
/seqtest.c:
--------------------------------------------------------------------------------
   1 | /*
   2 |  * Copyright 2016 Lucera Financial Infrastructures, Inc.
   3 |  *
   4 |  * Licensed under the Apache License, Version 2.0 (the "License");
   5 |  * you may not use file except in compliance with the License.
   6 |  * You may obtain a copy of the license at
   7 |  *
   8 |  *  http://www.apache.org/licenses/LICENSE-2.0
   9 |  *
  10 |  * Unless required by applicable law or agreed to in writing, software
  11 |  * distributed under the License is distributed on an "AS IS" BASIS,
  12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 |  * See the License for the specific language governing permissions and
  14 |  * limitations under the License.
  15 |  * This program is used to stress test TCP connections, verifying that
  16 |  * ordering constraints are preserved across a connection.  The intent
  17 |  * is to validate correct function of a TCP proxy.
  18 |  */
  19 | #include <errno.h>
  20 | #include <stdio.h>
  21 | #include <stdlib.h>
  22 | #include <stdint.h>
  23 | #include <inttypes.h>
  24 | #include <unistd.h>
  25 | #include <assert.h>
  26 | #include <string.h>
  27 | #include <sys/socket.h>
  28 | #include <time.h>
  29 | #include <sys/time.h>
  30 | #include <netdb.h>
  31 | #include <netinet/in.h>
  32 | #include <netinet/tcp.h>
  33 | #include <math.h>
  34 | #include <pthread.h>
  35 | 
  36 | #define	FLAG_REPLY	(1u << 0)
  37 | #define	FLAG_ERROR	(1u << 1)
  38 | 
  39 | #define	min(x, y) ((x) < (y) ? (x) : (y))
  40 | #define	max(x, y) ((x) > (y) ? (x) : (y))
  41 | 
  42 | static int start_wait = 0;
  43 | static int start_ready = 0;
  44 | static pthread_cond_t waitcv;
  45 | static pthread_cond_t startcv;
  46 | static pthread_mutex_t startmx;
  47 | 
  48 | typedef struct sample {
  49 | 	uint64_t	when;
  50 | 	uint64_t	lat;
  51 | 	uint16_t	ssz;
  52 | 	uint16_t	rsz;
  53 | } sample_t;
  54 | 
  55 | #ifndef HAVE_STRLCPY
  56 | int
  57 | strlcpy(char *dst, const char *src, size_t dstsize)
  58 | {
  59 | 	int len;
  60 | 	char *end;
  61 | 	/* poor mans strlcpy, not as fast as a smarter implementation */
  62 | 	(void) strncpy(dst, src, dstsize);
  63 | 	if (dstsize > 0) {
  64 | 		dst[dstsize - 1] = '\0';
  65 | 	}
  66 | 	return (strlen(src));
  67 | }
  68 | #endif /* HAVE_STRLCPY */
  69 | 
  70 | /*
  71 |  * Amazing - MacOS X doesn't have a standards conforming version
  72 |  * of high resolution timers.
  73 |  */
  74 | #ifndef	HAVE_GETHRTIME
  75 | 
  76 | #if defined(HAVE_CLOCK_GETTIME)
  77 | 
  78 | uint64_t
  79 | gethrtime(void)
  80 | {
  81 | 	struct timespec ts;
  82 | 	if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) {
  83 | 		perror("clock_gettime");
  84 | 		exit(1);
  85 | 	}
  86 | 	return (ts.tv_nsec + (ts.tv_sec * 1000000000ull));
  87 | }
  88 | 
  89 | #elif defined(HAVE_GETTIMEOFDAY)
  90 | uint64_t
  91 | gethrtime(void)
  92 | {
  93 | 	struct timeval tv;
  94 | 	uint64_t nsec;
  95 | 
  96 | 	gettimeofday(&tv, NULL);
  97 | 	nsec = tv.tv_sec * 1000000000ull + tv.tv_usec * 1000ull;
  98 | 	return (nsec);
  99 | }
 100 | #else
 101 | #error "No way to determine high resolution time"
 102 | #endif
 103 | 
 104 | #endif /* HAVE_GETHRTIME */
 105 | 
 106 | /* We probably don't want to exchange messages in excess of this. */
 107 | uint32_t maxmsg = 8000;
 108 | 
 109 | int debug = 0;
 110 | 
 111 | /*
 112 |  * Test header, used at the start of every message.
 113 |  */
 114 | typedef struct test_header {
 115 | 	uint64_t	seqno;
 116 | 	uint64_t	ts1;	/* senders send time */
 117 | 	uint64_t	ts2;	/* repliers recv time */
 118 | 	uint64_t	ts3;	/* repliers send time */
 119 | 	uint32_t	rdly;	/* reply delay (ns) */
 120 | 	uint16_t	ssz;	/* send size */
 121 | 	uint16_t	rsz;	/* reply size */
 122 | } test_header_t;
 123 | 
 124 | /*
 125 |  * Each thread in the sending system is driven by a single state.
 126 |  * This allows us to set up the test, but otherwise each thread runs
 127 |  * independent of the others, so we have no locks, nor races.
 128 |  */
 129 | typedef struct test {
 130 | 	int		sock;
 131 | 	uint64_t	sseqno;
 132 | 	uint64_t	rseqno;
 133 | 	uint32_t	rdly_min;	/* reply delay (ns) */
 134 | 	uint32_t	rdly_max;	/* reply delay (ns) */
 135 | 	uint32_t	sdly_min;	/* interpacket send delay (ns) */
 136 | 	uint32_t	sdly_max;	/* interpacket send delay (ns) */
 137 | 	uint16_t	ssz_min;	/* send size min */
 138 | 	uint16_t	ssz_max;	/* send size max */
 139 | 	uint16_t	rsz_min;	/* reply size min */
 140 | 	uint16_t	rsz_max;	/* reply size max */
 141 | 	uint32_t	rintvl;		/* reply interval (0 = none) */
 142 | 	uint64_t	count;		/* num to exchange */
 143 | 	uint64_t	replies;	/* total replies */
 144 | 	uint32_t	flags;		/* flags */
 145 | 	pthread_t	tid;		/* pthread processing this test */
 146 | 	struct sockaddr	*addr;		/* address for the socket */
 147 | 	struct addrinfo *lai;		/* local addr to bind (client only) */
 148 | 	socklen_t	addrlen;
 149 | 	sample_t	*samples;
 150 | } test_t;
 151 | 
 152 | /*
 153 |  * randtime determines the number of nsec used for each call to random().
 154 |  * The idea here is that we can use random() as a busy worker to spin.  This
 155 |  * will prevent it from being optimized away, and gives us some idea of the
 156 |  * involved with each iteration.
 157 |  */
 158 | uint64_t
 159 | randtime(void)
 160 | {
 161 | 	uint64_t now, end;
 162 | 	static uint64_t rtime;
 163 | 	int i;
 164 | 
 165 | 	if (rtime < 1) {
 166 | 		now = gethrtime();
 167 | 		for (i = 0; i < 1U << 20; i++) {
 168 | 			random();
 169 | 		}
 170 | 		end = gethrtime();
 171 | 		rtime = (end - now) >> 20;
 172 | 	}
 173 | 	if (rtime < 1) {
 174 | 		rtime = 1;
 175 | 	}
 176 | 	return rtime;
 177 | }
 178 | 
 179 | int
 180 | cmpu64(const void *u1, const void *u2)
 181 | {
 182 | 	return (*(uint64_t *)u1 - *(uint64_t *)u2);
 183 | }
 184 | 
 185 | double
 186 | pctile(uint64_t *samples, size_t nsamples, double pctile)
 187 | {
 188 | 	double i, k;
 189 | 
 190 | 	i = (nsamples * pctile / 100.0);
 191 | 	k = ceil(i);
 192 | 
 193 | 	if (k != i) {
 194 | 		return (double)samples[(int)k-1];
 195 | 	}
 196 | 
 197 | 	k = floor(i);
 198 | 	return ((samples[(int)k-1]+samples[(int)k])/2.0);
 199 | }
 200 | 
 201 | /*
 202 |  * ndelay waits a given number of nsec.  It does this by sleeping for large
 203 |  * values of nsec, but will spin when a smaller delay is required.
 204 |  */
 205 | void
 206 | ndelay(uint32_t nsec)
 207 | {
 208 | 	uint64_t now, end;
 209 | 	end = gethrtime() + nsec;
 210 | 	while ((now = gethrtime()) < end) {
 211 | 		if ((end - now) > 1000000) {
 212 | 			struct timespec ts;
 213 | 			ts.tv_sec = 0;
 214 | 			ts.tv_nsec = end - now;
 215 | 			/* we'll probably sleep too long, that's ok */
 216 | 			nanosleep(&ts, NULL);
 217 | 			continue;
 218 | 		}
 219 | 		/*
 220 | 		 * Do some work, shouldn't take long, but this eases the pressure
 221 | 		 * we put on gethrtime.
 222 | 		 */
 223 | 		random();
 224 | 	}
 225 | }
 226 | 
 227 | /*
 228 |  * range returns a value chosen at random between a min and a max.  The value
 229 |  * is chosen using rand(), so this is not suitable for cryptographic purposes.
 230 |  */
 231 | uint32_t
 232 | range(uint32_t minval, uint32_t maxval)
 233 | {
 234 | 	uint32_t val = minval;
 235 | 
 236 | 	if (maxval > minval) {
 237 | 		val += (rand() % (maxval - minval));
 238 | 	}
 239 | 	return (val);
 240 | }
 241 | 
 242 | /*
 243 |  * senderreceiver is a pthread worker that sends a single message and expects
 244 |  * a reply.
 245 |  */
 246 | void *
 247 | senderreceiver(void *arg)
 248 | {
 249 | 	test_t		*t = arg;
 250 | 	char		*sbuf, *rbuf, *sptr, *rptr;
 251 | 	uint64_t	stime, now, deltat;
 252 | 	uint32_t	nbytes = 0;
 253 | 	int		rv;
 254 | 	test_header_t	*sh, *rh;
 255 | 	int		i;
 256 | 	int		good = 0;
 257 | 	int		count;
 258 | 
 259 | 	sbuf = malloc(maxmsg);
 260 | 	rbuf = malloc(maxmsg);
 261 | 
 262 | 	pthread_mutex_lock(&startmx);
 263 | 	start_wait++;
 264 | 	pthread_cond_signal(&waitcv);
 265 | 	while (!start_ready) {
 266 | 		pthread_cond_wait(&startcv, &startmx);
 267 | 	}
 268 | 	pthread_mutex_unlock(&startmx);
 269 | 
 270 | 	rptr = rbuf;
 271 | 	count = t->count;
 272 | 	t->rintvl = 1;
 273 | 
 274 | 	if (count < 1) {
 275 | 		fprintf(stderr, "count must be at least 1\n");
 276 | 		exit(1);
 277 | 	}
 278 | 
 279 | 	for (i = 0; i < count; i++) {
 280 | 
 281 | 		uint16_t ssz, rsz;
 282 | 		uint32_t sdly, rdly;
 283 | 		sh = (void *)sbuf;
 284 | 		sptr = sbuf;
 285 | 
 286 | 		ssz = (uint16_t) range(t->ssz_min, t->ssz_max);
 287 | 		rsz = (uint16_t) range(t->rsz_min, t->rsz_max);
 288 | 		sdly = range(t->sdly_min, t->sdly_min);
 289 | 		rdly = range(t->rdly_min, t->rdly_min);
 290 | 
 291 | 		sh->ssz = ssz;
 292 | 		sh->rsz = (t->rintvl && ((i % t->rintvl) == 0)) ? rsz : 0;
 293 | 		sh->rdly = sh->rsz ? rdly : 0;
 294 | 		sh->seqno = t->sseqno++;
 295 | 
 296 | 		ndelay(sdly);
 297 | 
 298 | 		stime = gethrtime();
 299 | 		sh->ts3 = 0;
 300 | 		sh->ts2 = 0;
 301 | 		sh->ts1 = stime;
 302 | 
 303 | 		while (ssz > 0) {
 304 | 			rv = send(t->sock, sptr, ssz, 0);
 305 | 			if (rv < 0) {
 306 | 				perror("sender/send");
 307 | 				goto out;
 308 | 			}
 309 | 			ssz -= rv;
 310 | 			sptr += rv;
 311 | 		}
 312 | 		if (debug)
 313 | 			write(1, ">", 1);
 314 | 
 315 | 		rh = (void *)rbuf;
 316 | 		for (;;) {
 317 | 			size_t resid;
 318 | 			if (nbytes < sizeof (*rh)) {
 319 | 				/* suck in as much as we can */
 320 | 				resid = maxmsg - sizeof (*rh);
 321 | 			} else if (rh->rsz > maxmsg) {
 322 | 				fprintf(stderr, "h->rsz too big\n");
 323 | 				goto out;
 324 | 			} else if (nbytes < rh->rsz) {
 325 | 				resid = rh->rsz - nbytes;
 326 | 			} else {
 327 | 				break;
 328 | 			}
 329 | 			rv = recv(t->sock, rptr, resid, 0);
 330 | 			now = gethrtime();
 331 | 			if (rv < 0) {
 332 | 				perror("rcvr/recv");
 333 | 				goto out;
 334 | 			}
 335 | 			if (rv == 0) {
 336 | 				fprintf(stderr,
 337 | 				    "sender: recv closed too soon "
 338 | 				    "(%d rx, expected %d)\n", i, count);
 339 | 				goto out;
 340 | 			}
 341 | 			nbytes += rv;
 342 | 			rptr += rv;
 343 | 		}
 344 | 		assert(nbytes >= sizeof (*rh));
 345 | 		assert(nbytes >= rh->rsz);
 346 | 
 347 | 		if (rh->seqno != sh->seqno) {
 348 | 			fprintf(stderr,
 349 | 			    "reply seqno out of order (%"
 350 | 			    PRIu64 " != %" PRIu64 ")!!\n",
 351 | 			    rh->seqno, sh->seqno);
 352 | 			goto out;
 353 | 		}
 354 | 		if (rh->ts3 < rh->ts2) {
 355 | 			fprintf(stderr, "negative packet processing cost\n");
 356 | 			goto out;
 357 | 		}
 358 | 		if (rh->ts1 != sh->ts1) {
 359 | 			fprintf(stderr, "mismatched timestamps: %" PRIu64
 360 | 			    " != %" PRIu64 "\n", rh->ts1, sh->ts1);
 361 | 			goto out;
 362 | 		}
 363 | 		deltat = (now - rh->ts1) - (rh->ts3 - rh->ts2);
 364 | 		t->samples[t->rseqno].when = rh->ts1;
 365 | 		t->samples[t->rseqno].lat = deltat;
 366 | 		t->samples[t->rseqno].ssz = sh->ssz;
 367 | 		t->samples[t->rseqno].rsz = rh->rsz;
 368 | 		t->rseqno++;
 369 | 		/* if seqno dropped or duplicate, we expect many error msgs */
 370 | 
 371 | 		t->replies++;
 372 | 
 373 | 		if (debug)
 374 | 			write(1, "<", 1);
 375 | 
 376 | 		nbytes -= rh->rsz;
 377 | 		memmove(rbuf, rbuf + rh->rsz, nbytes);
 378 | 		rptr = rbuf + nbytes;
 379 | 	}
 380 | 	if (i < count) {
 381 | 		fprintf(stderr,
 382 | 			"only exchanged %d out of %d messages\n", i, count);
 383 | 		goto out;
 384 | 	}
 385 | 
 386 | 	good = 1;
 387 | 
 388 | out:
 389 | 	close(t->sock);
 390 | 	free(rbuf);
 391 | 	free(sbuf);
 392 | 
 393 | 	if (!good) {
 394 | 		exit(1);
 395 | 	}
 396 | 	return (NULL);
 397 | }
 398 | 
 399 | /*
 400 |  * sender is a pthread worker that sends the initial messages.
 401 |  */
 402 | void *
 403 | sender(void *arg)
 404 | {
 405 | 	test_t		*t = arg;
 406 | 	char		*buf, *ptr;
 407 | 	uint64_t	count = 0, stime;
 408 | 	int		rv;
 409 | 	test_header_t	*h;
 410 | 	int		i;
 411 | 
 412 | 	buf = malloc(maxmsg);
 413 | 
 414 | 	count = t->count;
 415 | 
 416 | 	if (count < 1) {
 417 | 		fprintf(stderr, "count must be at least 1\n");
 418 | 		exit(1);
 419 | 	}
 420 | 
 421 | 	for (i = 0; i < count; i++) {
 422 | 
 423 | 		uint16_t ssz, rsz;
 424 | 		uint32_t sdly, rdly;
 425 | 		h = (void *)buf;
 426 | 		ptr = buf;
 427 | 
 428 | 		ssz = (uint16_t) range(t->ssz_min, t->ssz_max);
 429 | 		rsz = (uint16_t) range(t->rsz_min, t->rsz_max);
 430 | 		sdly = range(t->sdly_min, t->sdly_min);
 431 | 		rdly = range(t->rdly_min, t->rdly_min);
 432 | 
 433 | 		h->ssz = ssz;
 434 | 		h->rsz = (t->rintvl && ((i % t->rintvl) == 0)) ? rsz : 0;
 435 | 		h->rdly = h->rsz ? rdly : 0;
 436 | 		h->seqno = t->sseqno++;
 437 | 
 438 | 		ndelay(sdly);
 439 | 
 440 | 		stime = gethrtime();
 441 | 		h->ts3 = 0;
 442 | 		h->ts2 = 0;
 443 | 		h->ts1 = stime;
 444 | 
 445 | 		while (ssz > 0) {
 446 | 			rv = send(t->sock, ptr, ssz, 0);
 447 | 			if (rv < 0) {
 448 | 				perror("sender/send");
 449 | 				return (NULL);
 450 | 			}
 451 | 			ssz -= rv;
 452 | 			ptr += rv;
 453 | 		}
 454 | 		if (debug)
 455 | 			write(1, ">", 1);
 456 | 	}
 457 | 	free(buf);
 458 | 	return (NULL);
 459 | }
 460 | 
 461 | /*
 462 |  * receiver is a pthread worker that receives any replies.  It runs in the
 463 |  * same process as sender.
 464 |  */
 465 | void *
 466 | receiver(void *arg)
 467 | {
 468 | 	test_t		*t = arg;
 469 | 	char		*buf, *ptr;
 470 | 	uint32_t	exp;
 471 | 	uint32_t	nbytes = 0;
 472 | 	uint64_t	ltime = 0, now = 0, deltat = 0;
 473 | 	test_header_t	*h;
 474 | 	int		rv;
 475 | 
 476 | 	buf = malloc(maxmsg);
 477 | 	ptr = buf;
 478 | 	exp = t->rintvl ? t->count / t->rintvl : 0;
 479 | 
 480 | 	while (t->count == 0 || (exp > 0)) {
 481 | 		h = (void *)buf;
 482 | 		for (;;) {
 483 | 			size_t resid;
 484 | 			if (nbytes < sizeof (*h)) {
 485 | 				/* suck in as much as we can */
 486 | 				resid = maxmsg - sizeof (*h);
 487 | 			} else if (h->rsz > maxmsg) {
 488 | 				fprintf(stderr, "h->rsz too big\n");
 489 | 				goto out;
 490 | 			} else if (nbytes < h->rsz) {
 491 | 				resid = h->rsz - nbytes;
 492 | 			} else {
 493 | 				break;
 494 | 			}
 495 | 			rv = recv(t->sock, ptr, resid, 0);
 496 | 			now = gethrtime();
 497 | 			if (rv < 0) {
 498 | 				perror("rcvr/recv");
 499 | 				goto out;
 500 | 			}
 501 | 			if (rv == 0) {
 502 | 				fprintf(stderr, "receiver: recv closed too soon\n");
 503 | 				goto out;
 504 | 			}
 505 | 			nbytes += rv;
 506 | 			ptr += rv;
 507 | 		}
 508 | 		assert(nbytes >= sizeof (*h));
 509 | 		assert(nbytes >= h->rsz);
 510 | 
 511 | 		if (h->ts1 < ltime) {
 512 | 			fprintf(stderr, "ts1 backwards %" PRIu64
 513 | 			    " < %" PRIu64 " !!\n",
 514 | 			    h->ts1, ltime);
 515 | 		}
 516 | 		if (now < ltime) {
 517 | 			fprintf(stderr, "time-travelling packet\n");
 518 | 		}
 519 | 		if (h->ts3 < h->ts2) {
 520 | 			fprintf(stderr, "negative packet processing cost\n");
 521 | 		}
 522 | 		deltat = (now - h->ts1) - (h->ts3 - h->ts2);
 523 | 		ltime = h->ts1;
 524 | 		if (h->seqno != t->rseqno) {
 525 | 			fprintf(stderr,
 526 | 			    "reply seqno out of order (%" PRIu64
 527 | 			    " != %" PRIu64 ")!!\n",
 528 | 			    h->seqno, t->rseqno);
 529 | 		}
 530 | 		t->samples[t->rseqno].when = h->ts1;
 531 | 		t->samples[t->rseqno].lat = deltat;
 532 | 		t->samples[t->rseqno].ssz = 0;	/* probably of no use here */
 533 | 		t->samples[t->rseqno].rsz = 0;
 534 | 
 535 | 		t->rseqno++;
 536 | 		/* if seqno dropped or duplicate, we expect many error msgs */
 537 | 
 538 | 		/* XXX: we could check timestamps, figure latency, etc. */
 539 | 		t->replies++;
 540 | 
 541 | 		if (debug)
 542 | 			write(1, "<", 1);
 543 | 
 544 | 		nbytes -= h->rsz;
 545 | 		memmove(buf, buf + h->rsz, nbytes);
 546 | 		ptr = buf + nbytes;
 547 | 		if (exp > 0)
 548 | 			exp--;
 549 | 	}
 550 | 
 551 | out:
 552 | 	free(buf);
 553 | 	return (NULL);
 554 | }
 555 | 
 556 | /*
 557 |  * replier is a pthread worker that services the initial sent messages,
 558 |  * checking them for correctness and optionally sending a reply.  Note that
 559 |  * the nature of the reply is driven by the message received, rather than
 560 |  * by the test.  This allows this to run mostly configuration free.
 561 |  */
 562 | void *
 563 | replier(void *arg)
 564 | {
 565 | 	test_t		*t = arg;
 566 | 	char		*sbuf, *sptr;
 567 | 	char		*rbuf, *rptr;
 568 | 	uint32_t	nbytes = 0;
 569 | 	uint64_t	ltime = 0, now = 0;
 570 | 	test_header_t	*h;
 571 | 	uint32_t	rdly;
 572 | 	uint16_t	rsz, ssz;
 573 | 	int		rv;
 574 | 
 575 | 	rbuf = malloc(maxmsg);
 576 | 	sbuf = malloc(maxmsg);
 577 | 	rptr = rbuf;
 578 | 	sptr = sbuf;
 579 | 
 580 | 	for (;;) {
 581 | 		h = (void *)rbuf;
 582 | 		for (;;) {
 583 | 			size_t resid;
 584 | 			if (nbytes < sizeof (*h)) {
 585 | 				/* suck in as much as we can */
 586 | 				resid = t->ssz_max - nbytes;
 587 | 			} else if (h->ssz > maxmsg) {
 588 | 				fprintf(stderr, "h->ssz too big\n");
 589 | 				goto out;
 590 | 			} else if (nbytes < h->ssz) {
 591 | 				resid = h->ssz - nbytes;
 592 | 			} else {
 593 | 				break;
 594 | 			}
 595 | 			rv = recv(t->sock, rptr, resid, 0);
 596 | 			now = gethrtime();
 597 | 			if (rv < 0) {
 598 | 				perror("replier/recv");
 599 | 				goto out;
 600 | 			}
 601 | 			if (rv == 0) {
 602 | 				goto out;
 603 | 			}
 604 | 			rptr += rv;
 605 | 			nbytes += rv;
 606 | 		}
 607 | 		if (debug)
 608 | 			write(1, "-", 1);
 609 | 		assert(nbytes >= sizeof (*h));
 610 | 		assert(nbytes >= h->ssz);
 611 | 
 612 | 		if (h->ts1 < ltime) {
 613 | 			fprintf(stderr, "replier: ts1 backwards!!\n");
 614 | 		}
 615 | 
 616 | 		ltime = h->ts1;
 617 | 		rdly = h->rdly;
 618 | 		rsz = h->rsz;
 619 | 		ssz = h->ssz;
 620 | 
 621 | 		if (h->seqno != t->sseqno++) {
 622 | 			fprintf(stderr, "reply seqno out of order!!\n");
 623 | 		}
 624 | 		/* if seqno dropped or duplicate, we expect many error msgs */
 625 | 
 626 | 		nbytes -= ssz;
 627 | 		memmove(rbuf, rptr, nbytes);
 628 | 		rptr = rbuf + nbytes;
 629 | 
 630 | 		if ((nbytes = rsz) == 0) {
 631 | 			continue;
 632 | 		}
 633 | 
 634 | 		ndelay(h->rdly);
 635 | 
 636 | 		h = (void *)sbuf;
 637 | 		sptr = (void *)sbuf;
 638 | 
 639 | 		h->seqno = t->rseqno++;
 640 | 		h->ssz = ssz;
 641 | 		h->rsz = rsz;
 642 | 		h->ts1 = ltime;
 643 | 		h->rdly = rdly;
 644 | 		h->ts2 = now;
 645 | 		h->ts3 = gethrtime();
 646 | 		while (nbytes) {
 647 | 			rv = send(t->sock, sptr, nbytes, 0);
 648 | 			if (rv < 0) {
 649 | 				perror("send");
 650 | 				goto out;
 651 | 			}
 652 | 			nbytes -= rv;
 653 | 			sptr += rv;
 654 | 		}
 655 | 		if (debug) {
 656 | 			write(1, "+", 1);
 657 | 		}
 658 | 	}
 659 | 
 660 | out:
 661 | 	close(t->sock);
 662 | 	free(sbuf);
 663 | 	free(rbuf);
 664 | 	free(arg);
 665 | 	return (NULL);
 666 | }
 667 | 
 668 | /*
 669 |  * acceptor runs in the replier's process, and is reponsible for firing
 670 |  * off a replier for each inbound connection.
 671 |  */
 672 | void *
 673 | acceptor(void *arg)
 674 | {
 675 | 	test_t		*t = arg;
 676 | 	test_t		*newt;
 677 | 	int s;
 678 | 	for (;;) {
 679 | 		socklen_t slen;
 680 | 		struct sockaddr_storage sa;
 681 | 		slen = sizeof (sa);
 682 | 		s  = accept(t->sock, (void *)&sa, &slen);
 683 | 		if (s < 0) {
 684 | 			perror("accept");
 685 | 			close(t->sock);
 686 | 			return (NULL);
 687 | 		}
 688 | 		newt = malloc(sizeof (*newt));
 689 | 		memcpy(newt, t, sizeof (*newt));
 690 | 		newt->sock = s;
 691 | 		newt->tid = 0;
 692 | 		pthread_create(&newt->tid, NULL, replier, newt);
 693 | 		pthread_detach(newt->tid);
 694 | 	}
 695 | }
 696 | 
 697 | test_t	*tests = NULL;
 698 | struct sockaddr **addrs = NULL;
 699 | int naddrs;
 700 | 
 701 | enum mode {
 702 | 	MODE_ASYNC_SEND = 0,
 703 | 	MODE_REPLIER,
 704 | 	MODE_SYNC_SEND
 705 | };
 706 | 
 707 | char *myopts[] = {
 708 | #define	SMIN		0
 709 | 	"ssize_min",
 710 | #define	SMAX		1
 711 | 	"ssize_max",
 712 | #define	SSIZE		2
 713 | 	"ssize",
 714 | #define	RMIN		3
 715 | 	"rsize_min",
 716 | #define	RMAX		4
 717 | 	"rsize_max",
 718 | #define	RSIZE		5
 719 | 	"rsize",
 720 | #define	THREADS		6
 721 | 	"threads",
 722 | #define	SDELAY		7
 723 | 	"sdelay",
 724 | #define	RDELAY		8
 725 | 	"rdelay",
 726 | #define	SDELAY_MIN	9
 727 | 	"sdelay_min",
 728 | #define	SDELAY_MAX	10
 729 | 	"sdelay_max",
 730 | #define	RDELAY_MIN	11
 731 | 	"rdelay_min",
 732 | #define	RDELAY_MAX	12
 733 | 	"rdelay_max",
 734 | #define	RINTERVAL	13
 735 | 	"rinterval",
 736 | #define COUNT		14
 737 | 	"count",
 738 | #define	DUMPFILE	15
 739 | 	"dump",
 740 | 	NULL
 741 | };
 742 | 
 743 | void
 744 | check_ndelay(void)
 745 | {
 746 | 	/* some timing tests to make sure our implementation doesn't suck */
 747 | 	uint64_t start, finish;
 748 | 	printf("randtime is %llu\n", (unsigned long long)randtime());
 749 | 	start = gethrtime();
 750 | 	ndelay(1000000);
 751 | 	finish = gethrtime();
 752 | 	printf("ndelay 1 msec took %llu ns\n", (unsigned long long)(finish - start));
 753 | 	start = gethrtime();
 754 | 	ndelay(1000000000);
 755 | 	finish = gethrtime();
 756 | 	printf("ndelay 1 sec took %llu ns\n", (unsigned long long)(finish - start));
 757 | 
 758 | 	start = gethrtime();
 759 | 	sleep(1);
 760 | 	finish = gethrtime();
 761 | 	printf("sleep 1 sec took %llu ns\n", (unsigned long long)(finish - start));
 762 | 
 763 | 	start = gethrtime();
 764 | 	usleep(10000);
 765 | 	finish = gethrtime();
 766 | 	printf("usleep(10ms) took %llu ns\n", (unsigned long long)(finish - start));
 767 | }
 768 | 
 769 | 
 770 | /*
 771 |  * Parse the local address from addrstr. If one exists, point
 772 |  * *local_addr to it and update *addrstr to point to the rest of the
 773 |  * address string. If no local address is found then set *local_addr
 774 |  * to NULL and leave *addrstr unchanged.
 775 |  *
 776 |  * E.g.
 777 |  *
 778 |  * *addrstr	= "192.168.1.115,192.168.1.119:6789"
 779 |  *
 780 |  * then:
 781 |  *
 782 |  * *local_addr	= "192.168.1.115"
 783 |  * *addstr	= "192.168.1.119:6789"
 784 |  */
 785 | static void
 786 | parse_local_addr(char **addrstr, char **local_addr)
 787 | {
 788 | 	char *delim = strchr(*addrstr, ',');
 789 | 
 790 | 	if (delim != NULL) {
 791 | 		/*
 792 | 		 * There is a local bind address specified. Split it
 793 | 		 * off from the rest of the address string.
 794 | 		 */
 795 | 		*delim = '\0';
 796 | 		*local_addr = *addrstr;
 797 | 		*addrstr = delim + 1;
 798 | 	} else {
 799 | 		*local_addr = NULL;
 800 | 	}
 801 | }
 802 | 
 803 | /*
 804 |  * Parse the host and port from addrstr. If the addrstr is well formed
 805 |  * then *host will point to the host string and *port to port.
 806 |  * Otherwise an error will print to stderr and the program will exit.
 807 |  *
 808 |  *
 809 |  * This function is destrcutive to *addrstr. It is equal to *host upon
 810 |  * return.
 811 |  *
 812 |  * E.g.
 813 |  *
 814 |  * *addrstr	= "192.168.1.119:6789"
 815 |  *
 816 |  * then:
 817 |  *
 818 |  * *host	= "192.168.1.119"
 819 |  * *port	= "6789"
 820 |  * *addrstr	= "192.168.1.119"
 821 |  */
 822 | static void
 823 | parse_addr(char **addrstr, char **host, char **port)
 824 | {
 825 | 	char *delim = strrchr(*addrstr, ':');
 826 | 
 827 | 	if (delim == NULL) {
 828 | 		fprintf(stderr, "no port found: %s\n", *addrstr);
 829 | 		exit(1);
 830 | 	}
 831 | 
 832 | 	/*
 833 | 	 * Separate host from port.
 834 | 	 */
 835 | 	*delim = '\0';
 836 | 	*host = *addrstr;
 837 | 	*port = delim + 1;
 838 | 
 839 | 	/*
 840 | 	 * Strip IPv6 brackets.
 841 | 	 */
 842 | 	if ((**host == '[') && ((*host)[strlen(*host) - 1] == ']')) {
 843 | 		(*host)[strlen(*host) - 1] = '\0';
 844 | 		(*host)++;
 845 | 	}
 846 | }
 847 | 
 848 | int
 849 | main(int argc, char **argv)
 850 | {
 851 | 	int c;
 852 | 	char *options, *optval;
 853 | 
 854 | 	uint16_t ssz_min, ssz_max, rsz_min, rsz_max;
 855 | 	uint32_t rdly_min, rdly_max;
 856 | 	uint32_t sdly_min, sdly_max;
 857 | 	uint32_t rintvl;
 858 | 	uint32_t nthreads;
 859 | 	uint32_t count;
 860 | 	enum mode mode;
 861 | 	int nais;
 862 | 	struct addrinfo **ais;
 863 | 	struct addrinfo **lais;
 864 | 	FILE *dumpfile = NULL;
 865 | 	uint64_t begin_time, finish_time;
 866 | 	int i;
 867 | 
 868 | 	ssz_min = ssz_max = rsz_min = rsz_max = sizeof (test_header_t);
 869 | 	rdly_min = rdly_max = 0;
 870 | 	sdly_min = sdly_max = 0;
 871 | 	rintvl = 1;
 872 | 	nthreads = 1;
 873 | 	count = 1;
 874 | 	mode = MODE_ASYNC_SEND;
 875 | 
 876 | 	/* initialize the timer */
 877 | 	(void) randtime();
 878 | 
 879 | 	while ((c = getopt(argc, argv, "o:srdS")) != EOF) {
 880 | 		switch (c) {
 881 | 		case 'd':
 882 | 			debug++;
 883 | 			break;
 884 | 		case 's':
 885 | 			mode = MODE_ASYNC_SEND;
 886 | 			break;
 887 | 		case 'S':
 888 | 			mode = MODE_SYNC_SEND;
 889 | 			break;
 890 | 		case 'r':
 891 | 			mode = MODE_REPLIER;
 892 | 			break;
 893 | 		case 'o':
 894 | 			options = optarg;
 895 | 			while (*options != '\0') {
 896 | 				switch (getsubopt(&options, myopts, &optval)) {
 897 | 				case SMIN:
 898 | 					if (optval == NULL) {
 899 | 						fprintf(stderr, "no value\n");
 900 | 						exit(1);
 901 | 					}
 902 | 					ssz_min = atoi(optval);
 903 | 					break;
 904 | 				case SMAX:
 905 | 					if (optval == NULL) {
 906 | 						fprintf(stderr, "no value\n");
 907 | 						exit(1);
 908 | 					}
 909 | 					ssz_max = atoi(optval);
 910 | 					break;
 911 | 				case SSIZE:
 912 | 					if (optval == NULL) {
 913 | 						fprintf(stderr, "no value\n");
 914 | 						exit(1);
 915 | 					}
 916 | 					ssz_max = ssz_min = atoi(optval);
 917 | 					break;
 918 | 				case RMIN:
 919 | 					if (optval == NULL) {
 920 | 						fprintf(stderr, "no value\n");
 921 | 						exit(1);
 922 | 					}
 923 | 					rsz_min = atoi(optval);
 924 | 					break;
 925 | 				case RMAX:
 926 | 					if (optval == NULL) {
 927 | 						fprintf(stderr, "no value\n");
 928 | 						exit(1);
 929 | 					}
 930 | 					rsz_max = atoi(optval);
 931 | 					break;
 932 | 				case RSIZE:
 933 | 					if (optval == NULL) {
 934 | 						fprintf(stderr, "no value\n");
 935 | 						exit(1);
 936 | 					}
 937 | 					rsz_max = rsz_min = atoi(optval);
 938 | 					break;
 939 | 
 940 | 				case THREADS:
 941 | 					if (optval == NULL) {
 942 | 						fprintf(stderr, "no value\n");
 943 | 						exit(1);
 944 | 					}
 945 | 					nthreads = atoi(optval);
 946 | 					break;
 947 | 				case RDELAY_MIN:
 948 | 					if (optval == NULL) {
 949 | 						fprintf(stderr, "no value\n");
 950 | 						exit(1);
 951 | 					}
 952 | 					rdly_min = atoi(optval);
 953 | 					break;
 954 | 				case RDELAY_MAX:
 955 | 					if (optval == NULL) {
 956 | 						fprintf(stderr, "no value\n");
 957 | 						exit(1);
 958 | 					}
 959 | 					rdly_max = atoi(optval);
 960 | 					break;
 961 | 				case RDELAY:
 962 | 					if (optval == NULL) {
 963 | 						fprintf(stderr, "no value\n");
 964 | 						exit(1);
 965 | 					}
 966 | 					rdly_min = rdly_max = atoi(optval);
 967 | 					break;
 968 | 				case SDELAY_MIN:
 969 | 					if (optval == NULL) {
 970 | 						fprintf(stderr, "no value\n");
 971 | 						exit(1);
 972 | 					}
 973 | 					sdly_min = atoi(optval);
 974 | 					break;
 975 | 				case SDELAY_MAX:
 976 | 					if (optval == NULL) {
 977 | 						fprintf(stderr, "no value\n");
 978 | 						exit(1);
 979 | 					}
 980 | 					sdly_max = atoi(optval);
 981 | 					break;
 982 | 				case SDELAY:
 983 | 					if (optval == NULL) {
 984 | 						fprintf(stderr, "no value\n");
 985 | 						exit(1);
 986 | 					}
 987 | 					sdly_min = sdly_max = atoi(optval);
 988 | 					break;
 989 | 				case RINTERVAL:
 990 | 					if (optval == NULL) {
 991 | 						fprintf(stderr, "no value\n");
 992 | 						exit(1);
 993 | 					}
 994 | 					rintvl = atoi(optval);
 995 | 					break;
 996 | 				case COUNT:
 997 | 					if (optval == NULL) {
 998 | 						fprintf(stderr, "no value\n");
 999 | 						exit(1);
1000 | 					}
1001 | 					count = atoi(optval);
1002 | 					break;
1003 | 				case DUMPFILE:
1004 | 					if (optval == NULL) {
1005 | 						fprintf(stderr, "no value\n");
1006 | 						exit(1);
1007 | 					}
1008 | 					dumpfile = fopen(optval, "w+");
1009 | 					if (dumpfile == NULL) {
1010 | 						fprintf(stderr, "open %s: %s\n", optval,
1011 | 						    strerror(errno));
1012 | 						exit(1);
1013 | 					}
1014 | 					break;
1015 | 				default:
1016 | 					fprintf(stderr, "bad option %s\n",
1017 | 						optval);
1018 | 					exit(1);
1019 | 				}
1020 | 			}
1021 | 		}
1022 | 	}
1023 | 
1024 | 	/* addresses */
1025 | 	if ((nais = (argc - optind)) == 0)  {
1026 | 		fprintf(stderr, "no address!\n");
1027 | 		exit(1);
1028 | 	}
1029 | 
1030 | 	ais = malloc(sizeof (struct addrinfo *) * (nais));
1031 | 	lais = malloc(sizeof (struct addrinfo *) * (nais));
1032 | 
1033 | 	for (i = 0; i < nais; i++) {
1034 | 		struct addrinfo hints;
1035 | 		int rv;
1036 | 		size_t arglen = strlen(argv[i + optind]);
1037 | 		char *hstr = malloc((arglen + 1) * sizeof(char));
1038 | 		char *host;
1039 | 		char *port;
1040 | 		char *lhost;
1041 | 
1042 | 		/*
1043 | 		 * Make a copy of the original argument to modify in
1044 | 		 * place for easier host and port extraction.
1045 | 		 */
1046 | 		strlcpy(hstr, argv[i + optind], arglen + 1);
1047 | 		lais[i] = NULL;
1048 | 		memset(&hints, 0, sizeof (hints));
1049 | 		hints.ai_socktype = SOCK_STREAM;
1050 | 
1051 | 		if (mode == MODE_REPLIER) {
1052 | 			hints.ai_flags = AI_PASSIVE;
1053 | 		}
1054 | 
1055 | 		if (mode == MODE_ASYNC_SEND || mode == MODE_SYNC_SEND) {
1056 | 			parse_local_addr(&hstr, &lhost);
1057 | 			if (lhost != NULL) {
1058 | 				/*
1059 | 				 * Sender has local address; store in
1060 | 				 * lais. Use ephemeral port because
1061 | 				 * otherwise each run will have to
1062 | 				 * wait for TIME-WAIT to expire.
1063 | 				 */
1064 | 				if ((rv = getaddrinfo(lhost, 0,
1065 | 					    &hints, &lais[i])) != 0) {
1066 | 					printf("failed to resolve %s: %s\n",
1067 | 					    lhost, gai_strerror(rv));
1068 | 					exit(1);
1069 | 				}
1070 | 			}
1071 | 		}
1072 | 
1073 | 		parse_addr(&hstr, &host, &port);
1074 | 
1075 | 		if ((rv = getaddrinfo(host, port, &hints, &ais[i])) != 0) {
1076 | 			printf("failed to resolve %s:%s: %s\n",
1077 | 			    host, port, gai_strerror(rv));
1078 | 			exit(1);
1079 | 		}
1080 | 
1081 | 		free(hstr);
1082 | 	}
1083 | 
1084 | 	naddrs = 0;
1085 | 
1086 | 	for (i = 0; i < nais; i++) {
1087 | 		struct addrinfo *ai;
1088 | 		for (ai = ais[i]; ai; ai = ai->ai_next) {
1089 | 			naddrs++;
1090 | 		}
1091 | 	}
1092 | 	if (mode == MODE_REPLIER) {
1093 | 		nthreads = naddrs;
1094 | 	}
1095 | 	addrs = malloc(naddrs * sizeof (struct sockaddr *));
1096 | 	naddrs = 0;
1097 | 	for (i = 0; i < nais; i++) {
1098 | 		char hbuf[64];
1099 | 		char pbuf[64];
1100 | 		struct addrinfo *ai;
1101 | 
1102 | 		for (ai = ais[i]; ai; ai = ai->ai_next) {
1103 | 			if (getnameinfo(ai->ai_addr, ai->ai_addrlen, hbuf,
1104 | 			    sizeof (hbuf), pbuf, sizeof (pbuf),
1105 | 			    NI_NUMERICHOST | NI_NUMERICSERV)) {
1106 | 				fprintf(stderr, "numeric host/port fail\n");
1107 | 				exit(1);
1108 | 			}
1109 | 			printf("Address %d: Host %s Port %s\n", naddrs,
1110 | 			    hbuf, pbuf);
1111 | 			addrs[naddrs++] = ai->ai_addr;
1112 | 		}
1113 | 	}
1114 | 
1115 | 	if (nthreads == 0) {
1116 | 		nthreads = naddrs;
1117 | 	}
1118 | 	if (mode == MODE_ASYNC_SEND) {
1119 | 		/* one for sender, and one for receiver */
1120 | 		nthreads *= 2;
1121 | 	}
1122 | 
1123 | 	begin_time = gethrtime();
1124 | 	tests = calloc(sizeof (test_t), nthreads);
1125 | 
1126 | 	for (i = 0; i < nthreads; i++) {
1127 | 		test_t *t = &tests[i];
1128 | 
1129 | 		t->ssz_min = (uint16_t) min(ssz_min, maxmsg);
1130 | 		t->ssz_min = (uint16_t) max(sizeof (test_header_t), t->ssz_min);
1131 | 
1132 | 		t->ssz_max = (uint16_t) min(ssz_max, maxmsg);
1133 | 		t->ssz_max = (uint16_t) max(t->ssz_min, t->ssz_max);
1134 | 
1135 | 		t->rsz_min = (uint16_t) min(rsz_min, maxmsg);
1136 | 		t->rsz_min = (uint16_t) max(sizeof (test_header_t), t->rsz_min);
1137 | 
1138 | 		t->rsz_max = (uint16_t) min(rsz_max, maxmsg);
1139 | 		t->rsz_max = (uint16_t) max(t->rsz_min, t->rsz_max);
1140 | 
1141 | 		t->count = count;
1142 | 		t->rdly_min = rdly_min;
1143 | 		t->rdly_max = rdly_max;
1144 | 		t->sdly_min = sdly_min;
1145 | 		t->sdly_max = sdly_max;
1146 | 		t->rintvl = rintvl;
1147 | 		t->sock = -1;
1148 | 		t->rseqno = 0;
1149 | 		t->sseqno = 0;
1150 | 		t->samples = calloc(count, sizeof (sample_t));
1151 | 
1152 | 		if (mode == MODE_ASYNC_SEND) {
1153 | 			t->addr = addrs[(i / 2) % naddrs];
1154 | 			if ((i % 2) != 0) {
1155 | 				t->sock = tests[i-1].sock;
1156 | 			}
1157 | 
1158 | 		} else {
1159 | 			t->addr = addrs[i % naddrs];
1160 | 			t->lai = lais[i % naddrs];
1161 | 		}
1162 | 		switch (t->addr->sa_family) {
1163 | 		case AF_INET:
1164 | 			t->addrlen = sizeof (struct sockaddr_in);
1165 | 			break;
1166 | 		case AF_INET6:
1167 | 			t->addrlen = sizeof (struct sockaddr_in6);
1168 | 			break;
1169 | 		default:
1170 | 			t->addrlen = 0;
1171 | 			break;
1172 | 		}
1173 | 
1174 | 		if (t->sock < 0) {
1175 | 			int on = 1;
1176 | 			t->sock = socket(t->addr->sa_family, SOCK_STREAM, 0);
1177 | 
1178 | 			if (setsockopt(t->sock, IPPROTO_TCP, TCP_NODELAY,
1179 | 			    &on, sizeof (on)) != 0)
1180 | 				perror("setting TCP_NODELAY");
1181 | 
1182 | 			if (t->lai != NULL) {
1183 | 				if (bind(t->sock,
1184 | 				    (struct sockaddr *) t->lai->ai_addr,
1185 | 				    t->lai->ai_addrlen) == -1) {
1186 | 					perror("binding sender");
1187 | 					exit(1);
1188 | 				}
1189 | 			}
1190 | 		}
1191 | 		if (t->sock == -1) {
1192 | 			perror("socket");
1193 | 			exit(1);
1194 | 		}
1195 | 		if ((mode == MODE_ASYNC_SEND) && ((i % 2) == 0)) {
1196 | 			if (connect(t->sock, t->addr, t->addrlen) != 0) {
1197 | 				perror("connect");
1198 | 				exit(1);
1199 | 			}
1200 | 			pthread_create(&t->tid, NULL, sender, t);
1201 | 
1202 | 		} else if (mode == MODE_ASYNC_SEND) {
1203 | 			pthread_create(&t->tid, NULL, receiver, t);
1204 | 
1205 | 		} else if (mode == MODE_SYNC_SEND) {
1206 | 			if (connect(t->sock, t->addr, t->addrlen) != 0) {
1207 | 				perror("connect");
1208 | 				exit(1);
1209 | 			}
1210 | 			pthread_create(&t->tid, NULL, senderreceiver, t);
1211 | 
1212 | 		} else if (mode == MODE_REPLIER) {
1213 | 			if (bind(t->sock, t->addr, t->addrlen) < 0) {
1214 | 				perror("bind");
1215 | 				exit(1);
1216 | 			}
1217 | 			if (listen(t->sock, 128) < 0) {
1218 | 				perror("listen");
1219 | 				exit(1);
1220 | 			}
1221 | 			pthread_create(&t->tid, NULL, acceptor, t);
1222 | 		}
1223 | 	}
1224 | 
1225 | #ifdef TIMETEST
1226 | 	check_ndelay();
1227 | #endif
1228 | 	/* start all threads together */
1229 | 	if (mode == MODE_SYNC_SEND) {
1230 | 		pthread_mutex_lock(&startmx);
1231 | 		while (start_wait < nthreads) {
1232 | 			pthread_cond_wait(&waitcv, &startmx);
1233 | 		}
1234 | 		start_ready = 1;
1235 | 		pthread_cond_broadcast(&startcv);
1236 | 		pthread_mutex_unlock(&startmx);
1237 | 		begin_time = gethrtime();
1238 | 	}
1239 | 
1240 | 	for (i = 0; i < nthreads; i++) {
1241 | 		test_t *t = &tests[i];
1242 | 		pthread_join(t->tid, NULL);
1243 | 	}
1244 | 
1245 | 	finish_time = gethrtime();
1246 | 
1247 | 	if (mode == MODE_ASYNC_SEND || mode == MODE_SYNC_SEND) {
1248 | 		uint64_t totmsgs = 0;
1249 | 		uint64_t latency = 0;
1250 | 		uint64_t mean = 0;
1251 | 		uint64_t variance = 0;
1252 | 		uint64_t *samples;
1253 | 		uint64_t sampno = 0;
1254 | 		int i, ii;
1255 | 
1256 | 		for (i = 0; i < nthreads; i++) {
1257 | 			test_t *t = &tests[i];
1258 | 			totmsgs += t->replies;
1259 | 		}
1260 | 
1261 | 		samples = calloc(totmsgs, sizeof (uint64_t));
1262 | 
1263 | 		for (i = 0; i < nthreads; i++) {
1264 | 			test_t *t = &tests[i];
1265 | 			for (ii = 0; ii < t->replies; ii++) {
1266 | 				samples[sampno++] = t->samples[ii].lat;
1267 | 			}
1268 | 		}
1269 | 
1270 | 		qsort(samples, totmsgs, sizeof (uint64_t), cmpu64);
1271 | 
1272 | 		for (i = 0; i < totmsgs; i++) {
1273 | 			latency += samples[i];
1274 | 		}
1275 | 
1276 | 		mean = totmsgs ? latency / totmsgs : 0;
1277 | 		for (i = 0; i < totmsgs; i++) {
1278 | 			uint64_t diff = samples[i] - mean;
1279 | 			variance += diff * diff;
1280 | 		}
1281 | 		if (totmsgs > 0) {
1282 | 			variance /= totmsgs;
1283 | 		}
1284 | 
1285 | 		printf("Received %" PRIu64 " replies\n", totmsgs);
1286 | 		printf("Time: %.1f us\n", (finish_time - begin_time) / 1000.0);
1287 | 		printf("ROUND TRIP LATENCY:\n");
1288 | 		printf("Average:  %.1f us\n", mean / 1000.0);
1289 | 		printf("Stddev:   %.1f us\n", sqrt((double)variance)/1000.0);
1290 | 		printf("Median:   %.1f us\n", pctile(samples, totmsgs, 50.0)/1000.0);
1291 | 		printf("90.0%%ile: %.1f us\n", pctile(samples, totmsgs, 90.0)/1000.0);
1292 | 		printf("99.0%%ile: %.1f us\n", pctile(samples, totmsgs, 99.0)/1000.0);
1293 | 		printf("99.9%%ile: %.1f us\n", pctile(samples, totmsgs, 99.9)/1000.0);
1294 | 		printf("Minimum:  %.1f us\n", samples[0]/1000.0);
1295 | 		printf("Maximum:  %.1f us\n", samples[totmsgs-1]/1000.0);
1296 | 
1297 | 		if (dumpfile != NULL) {
1298 | 			int i, ii;
1299 | 			fprintf(dumpfile, "# thread time latency rsz ssz\n");
1300 | 			for (i = 0; i < nthreads; i++) {
1301 | 				test_t *t = &tests[i];
1302 | 				for (ii = 0; ii < t->replies; ii++) {
1303 | 					fprintf(dumpfile,
1304 | 					    "%d %" PRIu64 " %" PRIu64
1305 | 					    " %u %u\n",
1306 | 					    i,
1307 | 					    t->samples[ii].when - begin_time,
1308 | 					    t->samples[ii].lat,
1309 | 					    t->samples[ii].rsz,
1310 | 					    t->samples[ii].ssz);
1311 | 				}
1312 | 			}
1313 | 			fclose(dumpfile);
1314 | 		}
1315 | 	}
1316 | 	return (0);
1317 | }
1318 | 


--------------------------------------------------------------------------------