├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.rst ├── jd_plugin.c ├── jd_samples_csv.c ├── jd_samples_hdf5.c ├── jd_sysinfo.c ├── jd_utils.c ├── jd_work.c ├── jitterdebugger.c ├── jitterdebugger.h ├── jitterplot ├── jittersamples.c ├── man ├── jitterdebugger.1 ├── jitterplot.1 └── jittersamples.1 └── scripts ├── genbuiltin └── release.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | *.o 3 | *_builtin.c 4 | jitterdebugger 5 | jittersamples 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.3 2 | - release 0.2 was a brown bag release, thus a new release 3 | 4 | 0.2 5 | - man pages added 6 | - license fixes 7 | - version identifier added to JSON output 8 | - various typos documentation fixed 9 | 10 | 0.1 11 | - initial release 12 | 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Siemens AG 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | CFLAGS += -pthread -Wall -Wstrict-aliasing=1 -Wno-unused-result \ 4 | -Wsign-compare -Wtype-limits -Wmissing-prototypes \ 5 | -Wstrict-prototypes 6 | LDFLAGS += -pthread 7 | 8 | ifdef DEBUG 9 | CFLAGS += -O0 -g 10 | LDFLAGS += -g 11 | else 12 | CFLAGS += -O2 13 | endif 14 | 15 | TARGETS=jitterdebugger jittersamples 16 | 17 | all: $(TARGETS) 18 | 19 | jitterdebugger: jd_utils.o jd_work.o jd_sysinfo.o jitterdebugger.o 20 | 21 | 22 | jittersamples_builtin_modules = jd_samples_csv 23 | 24 | # Determine if HDF5 support will be built into jittersamples 25 | JSCC=${CC} 26 | ifneq ("$(wildcard /usr/include/hdf5.h)", "") 27 | CFLAGS += -DCONFIG_HDF5 28 | JSCC = h5cc -shlib 29 | jittersamples_builtin_modules += jd_samples_hdf5 30 | endif 31 | 32 | jittersamples_builtin_sources = $(addsuffix .c,$(jittersamples_builtin_modules)) 33 | jittersamples_builtin_objs = $(addsuffix .o,$(jittersamples_builtin_modules)) 34 | 35 | jittersamples_objs = jd_utils.o $(jittersamples_builtin_objs) \ 36 | jd_samples_builtin.o jd_plugin.o jittersamples.o 37 | 38 | jd_samples_builtin.c: scripts/genbuiltin $(jittersamples_builtin_sources) 39 | scripts/genbuiltin $(jittersamples_builtin_modules) > $@ 40 | $(jittersamples_objs): %.o: %.c 41 | export HDF5_CC=${CC} 42 | $(JSCC) ${CFLAGS} -D_FILENAME=$(basename $<) -c $< -o $@ 43 | jittersamples: $(jittersamples_objs) 44 | $(JSCC) $^ -o $@ 45 | 46 | 47 | PHONY: .clean 48 | clean: 49 | rm -f *.o 50 | rm -f $(TARGETS) 51 | rm -f jd_samples_builtin.c 52 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. SPDX-License-Identifier: MIT 2 | 3 | ============== 4 | jitterdebugger 5 | ============== 6 | 7 | jitterdebugger measures wake up latencies. jitterdebugger starts a 8 | thread on each CPU which programs a timer and measures the time it 9 | takes from the timer expiring until the thread which set the timer 10 | runs again. 11 | 12 | This tool is a re-implementation of cyclictest. It doesn't have all the 13 | command line options as cyclictest which results are easy to get wrong 14 | and therefore an invalid latency report. 15 | 16 | The default settings of jitterdebugger will produce a correct 17 | measurement out of the box. 18 | 19 | Furthermore, the tool supports storing all samples for post 20 | processing. 21 | 22 | ################# 23 | Runtime Depenency 24 | ################# 25 | 26 | jitterdebugger has only dependency to glibc (incl pthread). 27 | 28 | - glibc >= 2.24 29 | 30 | jitterplot and jittersamples have additional dependency to 31 | 32 | - Python3 33 | - Matlibplot 34 | - Pandas 35 | - HDF5 >= 1.8.17 36 | 37 | ##### 38 | Usage 39 | ##### 40 | 41 | When running jitterdebugger without any command line options, there 42 | wont be any output until terminated by sending CTRL-C. It will print 43 | out the wake up latency for each CPU including a histogram of all 44 | latencies as JSON. The output can direclty be saved into a file using 45 | the -f command line option. 46 | 47 | :: 48 | 49 | # jitterdebugger 50 | ^C 51 | { 52 | "cpu": { 53 | "0": { 54 | "histogram": { 55 | "3": 3678, 56 | "4": 220, 57 | "5": 1, 58 | "8": 1, 59 | "11": 1 60 | }, 61 | "count": 3901, 62 | "min": 3, 63 | "max": 11, 64 | "avg": 3.06 65 | }, 66 | "1": { 67 | "histogram": { 68 | "3": 3690, 69 | "4": 188, 70 | "5": 2, 71 | "8": 3, 72 | "9": 2, 73 | "18": 1 74 | }, 75 | "count": 3886, 76 | "min": 3, 77 | "max": 18, 78 | "avg": 3.06 79 | } 80 | } 81 | } 82 | 83 | When providing '-v', jitterdebugger will live update all counters: 84 | 85 | :: 86 | 87 | # jitterdebugger -v 88 | affinity: 0,1 = 2 [0x3] 89 | T: 0 ( 614) A: 0 C: 13476 Min: 3 Avg: 3.08 Max: 10 90 | T: 1 ( 615) A: 1 C: 13513 Min: 3 Avg: 3.10 Max: 20 91 | ^C 92 | { 93 | "cpu": { 94 | "0": { 95 | "histogram": { 96 | "3": 4070, 97 | "4": 269, 98 | "5": 26, 99 | "6": 5, 100 | "7": 1, 101 | "8": 1, 102 | "9": 2, 103 | "10": 1 104 | }, 105 | "count": 4375, 106 | "min": 3, 107 | "max": 10, 108 | "avg": 3.08 109 | }, 110 | "1": { 111 | "histogram": { 112 | "3": 4002, 113 | "4": 320, 114 | "5": 22, 115 | "6": 4, 116 | "7": 2, 117 | "8": 1, 118 | "10": 2, 119 | "11": 1, 120 | "16": 2, 121 | "20": 1 122 | }, 123 | "count": 4357, 124 | "min": 3, 125 | "max": 20, 126 | "avg": 3.10 127 | } 128 | } 129 | } 130 | 131 | 132 | Field explanation: 133 | 134 | - T: Thread id (PID) 135 | - A: CPU affinity 136 | - C: Number of measurement cycles 137 | - Min: Smallest wake up latency observed 138 | - Max: Biggest wake up latency observed 139 | - Avg: Arithmetic average of all observed wake up latencies. 140 | 141 | 142 | ################ 143 | Measurement loop 144 | ################ 145 | 146 | The tool will start a measurement thread on each available CPU. 147 | 148 | The measurement loop does following: 149 | 150 | :: 151 | 152 | next = clock_gettime(CLOCK_MONOTONIC) + 1000us 153 | while not terminated: 154 | next = next + 1000us 155 | 156 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, next) 157 | 158 | now = clock_gettime() 159 | diff = now - next 160 | 161 | store(diff) 162 | 163 | 164 | ############## 165 | Histogram plot 166 | ############## 167 | 168 | This project provides a very simple analisys tool to a 169 | histogram. First let jitterdebugger collect some data and store the 170 | output into a file. 171 | 172 | :: 173 | 174 | # jitterdebugger -f results.json 175 | ^C 176 | # jitterplot hist results.json 177 | 178 | 179 | ################# 180 | Exporting samples 181 | ################# 182 | 183 | jitterdebugger is able to store all samples to a binary file. For post 184 | processing use jittersamples to print data as normal ASCII output: 185 | 186 | :: 187 | 188 | # jitterdebugger -o samples.raw 189 | ^C 190 | # jittersamples samples.raw | head 191 | 0;1114.936950838;9 192 | 0;1114.937204763;3 193 | 0;1114.937458457;3 194 | 0;1114.937711970;3 195 | 0;1114.937965595;3 196 | 0;1114.938218986;3 197 | 0;1114.938472416;3 198 | 0;1114.938725788;3 199 | 0;1114.938979191;3 200 | 0;1114.939232594;3 201 | 202 | The fields are: 203 | 204 | 1. CPUID 205 | 2. Timestamp in seconds 206 | 3. Wake up latency in micro seconds 207 | -------------------------------------------------------------------------------- /jd_plugin.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | 5 | #include "jitterdebugger.h" 6 | 7 | extern struct jd_plugin_desc *__jd_builtin[]; 8 | 9 | void jd_slist_append(struct jd_slist *jd_slist, void *data) 10 | { 11 | struct jd_slist *l; 12 | 13 | for (l = jd_slist; l && l->next; l = l->next) 14 | /* do nothing */ ; 15 | 16 | if (!l) 17 | err_abort("linked list is inconsistent"); 18 | 19 | l->next = malloc(sizeof(struct jd_slist)); 20 | if (!l->next) 21 | err_abort("allocation for link list failed"); 22 | bzero(l->next, sizeof(*l->next)); 23 | 24 | l->next->data = data; 25 | } 26 | 27 | void jd_slist_remove(struct jd_slist *jd_slist, void *data) 28 | { 29 | struct jd_slist *last, *l; 30 | 31 | last = jd_slist; 32 | l = last->next; 33 | while (l && l->data != data) { 34 | last = l; 35 | l = l->next; 36 | } 37 | 38 | if (!l) { 39 | warn_handler("Element not found to remove"); 40 | return; 41 | } 42 | } 43 | 44 | void __jd_plugin_init(void) 45 | { 46 | int i; 47 | 48 | for (i = 0; __jd_builtin[i]; i++) { 49 | struct jd_plugin_desc *desc = __jd_builtin[i]; 50 | 51 | if (desc->init()) { 52 | err_abort("plugin initialization failed: %s", 53 | desc->name); 54 | } 55 | } 56 | } 57 | 58 | void __jd_plugin_cleanup(void) 59 | { 60 | int i; 61 | 62 | for (i = 0; __jd_builtin[i]; i++) { 63 | struct jd_plugin_desc *desc = __jd_builtin[i]; 64 | desc->cleanup(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /jd_samples_csv.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | 6 | #include "jitterdebugger.h" 7 | 8 | static int output_csv(struct jd_samples_info *info, FILE *input) 9 | { 10 | struct latency_sample sample; 11 | FILE *output; 12 | 13 | output = jd_fopen(info->dir, "samples.csv", "w"); 14 | if (!output) 15 | err_handler(errno, "Could not open '%s/samples.csv' for writing", 16 | info->dir); 17 | 18 | while(fread(&sample, sizeof(struct latency_sample), 1, input)) { 19 | fprintf(output, "%d;%lld.%.9ld;%" PRIu64 "\n", 20 | sample.cpuid, 21 | (long long)sample.ts.tv_sec, 22 | sample.ts.tv_nsec, 23 | sample.val); 24 | } 25 | 26 | fclose(output); 27 | 28 | return 0; 29 | } 30 | 31 | struct jd_samples_ops csv_ops = { 32 | .name = "comma separate values", 33 | .format = "csv", 34 | .output = output_csv, 35 | }; 36 | 37 | static int csv_plugin_init(void) 38 | { 39 | return jd_samples_register(&csv_ops); 40 | } 41 | 42 | static void csv_plugin_cleanup(void) 43 | { 44 | jd_samples_unregister(&csv_ops); 45 | } 46 | 47 | JD_PLUGIN_DEFINE(csv_plugin_init, csv_plugin_cleanup); 48 | -------------------------------------------------------------------------------- /jd_samples_hdf5.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #define _GNU_SOURCE 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include "jitterdebugger.h" 12 | 13 | #define min(x,y) \ 14 | ({ \ 15 | typeof(x) __x = (x); \ 16 | typeof(x) __y = (y); \ 17 | __x < __y ? __x : __y; \ 18 | }) 19 | 20 | #define BLOCK_SIZE 10000 21 | 22 | struct cpu_data { 23 | hid_t set; 24 | uint64_t count; 25 | struct latency_sample *data; 26 | }; 27 | 28 | static int output_hdf5(struct jd_samples_info *info, FILE *input) 29 | { 30 | struct cpu_data *cpudata; 31 | struct latency_sample *data, *s, *d; 32 | hid_t file, type; 33 | herr_t err; 34 | uint64_t nrs, bs; 35 | size_t nr; 36 | off_t sz; 37 | unsigned int i, cnt; 38 | char *sid; 39 | char *ofile; 40 | 41 | if (asprintf(&ofile, "%s/samples.hdf5", info->dir) < 0) 42 | err_handler(errno, "asprintf()"); 43 | 44 | if (fseeko(input, 0L, SEEK_END) < 0) 45 | err_handler(errno, "fseek()"); 46 | sz = ftello(input); 47 | if (fseeko(input, 0L, SEEK_SET) < 0) 48 | err_handler(errno, "fseek()"); 49 | 50 | nrs = sz / sizeof(struct latency_sample); 51 | bs = min(nrs, BLOCK_SIZE); 52 | 53 | data = malloc(sizeof(struct latency_sample) * bs); 54 | if (!data) 55 | err_handler(ENOMEM, "malloc()"); 56 | 57 | file = H5Fcreate(ofile, H5F_ACC_TRUNC, 58 | H5P_DEFAULT, H5P_DEFAULT); 59 | if (file == H5I_INVALID_HID) 60 | err_handler(EIO, "failed to open file %s\n", ofile); 61 | 62 | type = H5Tcreate(H5T_COMPOUND, sizeof(struct latency_sample)); 63 | if (type == H5I_INVALID_HID) 64 | err_handler(EIO, "failed to create compound HDF5 type"); 65 | 66 | err = H5Tinsert(type, "CPUID", 0, H5T_NATIVE_UINT32); 67 | if (err < 0) 68 | err_handler(EIO, 69 | "failed to add type info to HDF5 compound type"); 70 | err = H5Tinsert(type, "Seconds", 4, H5T_NATIVE_UINT64); 71 | if (err < 0) 72 | err_handler(EIO, 73 | "failed to add type info to HDF5 compound type"); 74 | err = H5Tinsert(type, "Nanoseconds", 12, H5T_NATIVE_UINT64); 75 | if (err < 0) 76 | err_handler(EIO, 77 | "failed to add type info to HDF5 compound type"); 78 | err = H5Tinsert(type, "Value", 20, H5T_NATIVE_UINT64); 79 | if (err < 0) 80 | err_handler(EIO, 81 | "failed to add type info to HDF5 compound type"); 82 | 83 | cpudata = malloc(info->cpus_online * sizeof(struct cpu_data)); 84 | if (!cpudata) 85 | err_handler(errno, "failed to allocated memory for cpu sets\n"); 86 | 87 | for (i = 0; i < info->cpus_online; i++) { 88 | cpudata[i].count = 0; 89 | cpudata[i].data = malloc(BLOCK_SIZE * sizeof(struct latency_sample)); 90 | 91 | if (asprintf(&sid, "cpu%d\n", i) < 0) 92 | err_handler(errno, "failed to create label\n"); 93 | cpudata[i].set = H5PTcreate(file, sid, type, (hsize_t)bs, H5P_DEFAULT); 94 | free(sid); 95 | if (cpudata[i].set == H5I_INVALID_HID) 96 | err_handler(EIO, "failed to create HDF5 packet table"); 97 | } 98 | 99 | for (;;) { 100 | nr = fread(data, sizeof(struct latency_sample), bs, input); 101 | if (nr != bs) { 102 | if (feof(input)) 103 | break; 104 | if (ferror(input)) 105 | err_handler(errno, "fread()"); 106 | } 107 | 108 | for (i = 0; i < nr; i++) { 109 | s = &data[i]; 110 | if (s->cpuid >= info->cpus_online) { 111 | fprintf(stderr, "invalid sample found (cpuid %d)\n", 112 | s->cpuid); 113 | continue; 114 | } 115 | 116 | cnt = cpudata[s->cpuid].count; 117 | d = &(cpudata[s->cpuid].data[cnt]); 118 | memcpy(d, s, sizeof(struct latency_sample)); 119 | cpudata[s->cpuid].count++; 120 | } 121 | 122 | for (i = 0; i < info->cpus_online; i++) { 123 | H5PTappend(cpudata[i].set, cpudata[i].count, cpudata[i].data); 124 | cpudata[i].count = 0; 125 | } 126 | } 127 | 128 | free(data); 129 | 130 | for (i = 0; i < info->cpus_online; i++) { 131 | H5PTclose(cpudata[i].set); 132 | free(cpudata[i].data); 133 | } 134 | free(cpudata); 135 | 136 | H5Tclose(type); 137 | H5Fclose(file); 138 | free(ofile); 139 | 140 | return 0; 141 | } 142 | 143 | static struct jd_samples_ops hdf5_ops = { 144 | .name = "Hierarchical Data Format", 145 | .format = "hdf5", 146 | .output = output_hdf5, 147 | }; 148 | 149 | static int hdf5_plugin_init(void) 150 | { 151 | return jd_samples_register(&hdf5_ops); 152 | } 153 | 154 | static void hdf5_plugin_cleanup(void) 155 | { 156 | jd_samples_unregister(&hdf5_ops); 157 | } 158 | 159 | JD_PLUGIN_DEFINE(hdf5_plugin_init, hdf5_plugin_cleanup); 160 | -------------------------------------------------------------------------------- /jd_sysinfo.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #define _GNU_SOURCE 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "jitterdebugger.h" 14 | 15 | #define SYSLOG_ACTION_READ_ALL 3 16 | #define SYSLOG_ACTION_SIZE_BUFFER 10 17 | 18 | struct system_info *collect_system_info(void) 19 | { 20 | struct system_info *info; 21 | struct utsname buf; 22 | int err; 23 | 24 | info = malloc(sizeof(*info)); 25 | if (!info) 26 | err_handler(errno, "malloc()"); 27 | 28 | err = uname(&buf); 29 | if (err) 30 | err_handler(errno, "Could not retrieve name and information about current kernel"); 31 | 32 | info->sysname = jd_strdup(buf.sysname); 33 | info->nodename = jd_strdup(buf.nodename); 34 | info->release = jd_strdup(buf.release); 35 | info->version = jd_strdup(buf.version); 36 | info->machine = jd_strdup(buf.machine); 37 | 38 | info->cpus_online = get_nprocs(); 39 | 40 | return info; 41 | } 42 | 43 | void store_system_info(const char *path, struct system_info *sysinfo) 44 | { 45 | char *buf; 46 | FILE *fd; 47 | unsigned int len; 48 | 49 | jd_cp("/proc/cmdline", path); 50 | jd_cp("/proc/config.gz", path); 51 | jd_cp("/proc/cpuinfo", path); 52 | jd_cp("/proc/interrupts", path); 53 | jd_cp("/proc/sched_debug", path); 54 | 55 | // cpus_online 56 | fd = jd_fopen(path, "cpus_online", "w"); 57 | if (!fd) 58 | return; 59 | fprintf(fd, "%d\n", sysinfo->cpus_online); 60 | fclose(fd); 61 | 62 | // uname 63 | fd = jd_fopen(path, "uname", "w"); 64 | if (!fd) 65 | return; 66 | fprintf(fd, "%s %s %s %s %s\n", 67 | sysinfo->sysname, 68 | sysinfo->nodename, 69 | sysinfo->release, 70 | sysinfo->version, 71 | sysinfo->machine); 72 | fclose(fd); 73 | 74 | // dmesg 75 | len = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0); 76 | buf = malloc(len * sizeof(char)); 77 | if (!buf) 78 | err_handler(errno, "malloc()"); 79 | 80 | if (klogctl(SYSLOG_ACTION_READ_ALL, buf, len) < 0) 81 | return; 82 | 83 | fd = jd_fopen(path, "dmesg", "w"); 84 | if (!fd) { 85 | free(buf); 86 | return; 87 | } 88 | 89 | if (fwrite(buf, sizeof(char), len, fd) != len) 90 | warn_handler("writing dmesg failed\n"); 91 | 92 | fclose(fd); 93 | free(buf); 94 | } 95 | 96 | void free_system_info(struct system_info *sysinfo) 97 | { 98 | if (sysinfo->sysname) 99 | free(sysinfo->sysname); 100 | if (sysinfo->nodename) 101 | free(sysinfo->nodename); 102 | if (sysinfo->release) 103 | free(sysinfo->release); 104 | if (sysinfo->version) 105 | free(sysinfo->version); 106 | if (sysinfo->machine) 107 | free(sysinfo->machine); 108 | free(sysinfo); 109 | } 110 | -------------------------------------------------------------------------------- /jd_utils.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #define _GNU_SOURCE 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "jitterdebugger.h" 17 | 18 | #define BUFSIZE 4096 19 | 20 | struct ringbuffer_sample { 21 | struct timespec ts; 22 | uint64_t val; 23 | }; 24 | 25 | struct ringbuffer { 26 | uint32_t size; 27 | uint32_t overflow; 28 | uint32_t read; 29 | uint32_t write; 30 | struct ringbuffer_sample *data; 31 | }; 32 | 33 | static int ringbuffer_full(uint32_t size, uint32_t read, uint32_t write) 34 | { 35 | return write - read == size; 36 | } 37 | 38 | static int ringbuffer_empty(uint32_t read, uint32_t write) 39 | { 40 | return write == read; 41 | } 42 | 43 | static uint32_t ringbuffer_mask(uint32_t size, uint32_t idx) 44 | { 45 | return idx & (size - 1); 46 | } 47 | 48 | struct ringbuffer *ringbuffer_create(unsigned int size) 49 | { 50 | struct ringbuffer *rb; 51 | 52 | /* Size needs to be of power of 2 */ 53 | if ((size & (size - 1)) != 0) 54 | return NULL; 55 | 56 | rb = calloc(1, sizeof(*rb)); 57 | if (!rb) 58 | return NULL; 59 | 60 | rb->size = size; 61 | rb->data = calloc(rb->size, sizeof(struct ringbuffer_sample)); 62 | 63 | return rb; 64 | } 65 | 66 | void ringbuffer_free(struct ringbuffer *rb) 67 | { 68 | free(rb->data); 69 | free(rb); 70 | } 71 | 72 | int ringbuffer_write(struct ringbuffer *rb, struct timespec ts, uint64_t val) 73 | { 74 | uint32_t read, idx; 75 | 76 | read = READ_ONCE(rb->read); 77 | if (ringbuffer_full(rb->size, read, rb->write)) { 78 | rb->overflow++; 79 | return 1; 80 | } 81 | 82 | idx = ringbuffer_mask(rb->size, rb->write + 1); 83 | rb->data[idx].ts = ts; 84 | rb->data[idx].val = val; 85 | 86 | WRITE_ONCE(rb->write, rb->write + 1); 87 | return 0; 88 | } 89 | 90 | int ringbuffer_read(struct ringbuffer *rb, struct timespec *ts, uint64_t *val) 91 | { 92 | uint32_t write, idx; 93 | 94 | write = READ_ONCE(rb->write); 95 | if (ringbuffer_empty(rb->read, write)) 96 | return 1; 97 | 98 | idx = ringbuffer_mask(rb->size, rb->read + 1); 99 | *ts = rb->data[idx].ts; 100 | *val = rb->data[idx].val; 101 | 102 | WRITE_ONCE(rb->read, rb->read + 1); 103 | return 0; 104 | } 105 | 106 | void _err_handler(int error, char *fmt, ...) 107 | { 108 | va_list ap; 109 | char *msg; 110 | 111 | va_start(ap, fmt); 112 | vasprintf(&msg, fmt, ap); 113 | va_end(ap); 114 | 115 | fprintf(stderr, "%s: %s\n", msg, strerror(error)); 116 | 117 | free(msg); 118 | exit(1); 119 | } 120 | 121 | void _warn_handler(char *fmt, ...) 122 | { 123 | va_list ap; 124 | 125 | va_start(ap, fmt); 126 | vfprintf(stdout, fmt, ap); 127 | va_end(ap); 128 | } 129 | 130 | /* 131 | * Returns parsed zero or positive number, or a negative error value. 132 | * len optionally stores the length of the parsed string, may be NULL. 133 | */ 134 | long int parse_num(const char *str, int base, size_t *len) 135 | { 136 | long int ret; 137 | char *endptr; 138 | 139 | errno = 0; 140 | ret = strtol(str, &endptr, base); 141 | if (errno) 142 | return -errno; 143 | if (ret < 0) 144 | return -ERANGE; 145 | 146 | if (len) 147 | *len = endptr - str; 148 | 149 | return ret; 150 | } 151 | 152 | long int parse_time(const char *str) 153 | { 154 | long int t; 155 | char *end; 156 | 157 | t = strtol(str, &end, 10); 158 | 159 | if (!(strlen(end) == 0 || strlen(end) == 1)) 160 | return -EINVAL; 161 | 162 | if (end) { 163 | switch (*end) { 164 | case 's': 165 | case 'S': 166 | break; 167 | case 'm': 168 | case 'M': 169 | t *= 60; 170 | break; 171 | 172 | case 'h': 173 | case 'H': 174 | t *= 60 * 60; 175 | break; 176 | 177 | case 'd': 178 | case 'D': 179 | t *= 24 * 60 * 60; 180 | break; 181 | default: 182 | return -EINVAL; 183 | } 184 | } 185 | 186 | return t; 187 | } 188 | 189 | static void cpuset_from_bits(cpu_set_t *set, unsigned long bits) 190 | { 191 | unsigned i; 192 | 193 | for (i = 0; i < sizeof(bits) * 8; i++) 194 | if ((1UL << i) & bits) 195 | CPU_SET(i, set); 196 | } 197 | 198 | static unsigned long cpuset_to_bits(cpu_set_t *set) 199 | { 200 | unsigned long bits = 0; 201 | unsigned int i, t, bit; 202 | 203 | for (i = 0, t = 0; t < (unsigned) CPU_COUNT(set); i++) { 204 | bit = CPU_ISSET(i, set); 205 | bits |= bit << i; 206 | t += bit; 207 | } 208 | 209 | return bits; 210 | } 211 | 212 | static inline void _cpuset_fprint_end(FILE *f, unsigned long i, unsigned long r) 213 | { 214 | if (r > 1) 215 | fprintf(f, "%c%lu", r > 2 ? '-' : ',', i - 1); 216 | } 217 | 218 | /* Prints cpu_set_t as an affinity specification. */ 219 | void cpuset_fprint(FILE *f, cpu_set_t *set) 220 | { 221 | unsigned long bit = 0, range = 0; 222 | unsigned long i, t, comma = 0; 223 | 224 | for (i = 0, t = 0; t < (unsigned) CPU_COUNT(set); t += bit, i++) { 225 | bit = CPU_ISSET(i, set); 226 | if (!range && bit) { 227 | if (comma) 228 | fputc(',', f); 229 | 230 | comma = 1; 231 | fprintf(f, "%lu", i); 232 | } else if (!bit) { 233 | _cpuset_fprint_end(f, i, range); 234 | range = 0; 235 | } 236 | range += bit; 237 | } 238 | 239 | _cpuset_fprint_end(f, i, range); 240 | fprintf(f, " = %u [0x%lX]", CPU_COUNT(set), cpuset_to_bits(set)); 241 | } 242 | 243 | static long int _cpuset_parse_num(const char *str, int base, size_t *len) 244 | { 245 | long int ret; 246 | 247 | ret = parse_num(str, base, len); 248 | if (ret < 0) 249 | err_abort("cpuset: unable to parse string %s", str); 250 | 251 | return ret; 252 | } 253 | 254 | /* 255 | * Parses affinity specification (i.e. 0,2-3,7) into a cpu_set_t. 256 | * Returns parsed string length or -errno. 257 | */ 258 | ssize_t cpuset_parse(cpu_set_t *set, const char *str) 259 | { 260 | unsigned int i, first, last; 261 | ssize_t len_next; 262 | long int num = 0; 263 | size_t len = 0; 264 | 265 | if (!strncmp(str, "0x", 2)) { 266 | num = _cpuset_parse_num(str, 16, &len); 267 | cpuset_from_bits(set, (unsigned long) num); 268 | return len; 269 | } 270 | 271 | num = _cpuset_parse_num(str, 10, &len); 272 | 273 | str += len; 274 | first = num; 275 | 276 | if (str[0] == '-') { 277 | num = _cpuset_parse_num(str + 1, 10, &len); 278 | str += 1 + len; 279 | last = len ? num + 1 : CPU_SETSIZE; /* "x-" means x-CPU_SIZE */ 280 | } else 281 | last = first + 1; 282 | 283 | if (CPU_SETSIZE < last) { 284 | warn_handler("cpu num %d bigger than CPU_SETSIZE(%u), reducing", 285 | last, CPU_SETSIZE); 286 | last = CPU_SETSIZE; 287 | } 288 | 289 | for (i = first; i < last; i++) 290 | CPU_SET(i, set); 291 | 292 | if (str[0] == ',') { 293 | len_next = cpuset_parse(set, str + 1); 294 | if (len_next < 0) 295 | return len_next; 296 | 297 | len += len_next; 298 | } 299 | 300 | return len; 301 | } 302 | 303 | int sysfs_load_str(const char *path, char **buf) 304 | { 305 | int fd, ret; 306 | size_t len; 307 | 308 | fd = TEMP_FAILURE_RETRY(open(path, O_RDONLY)); 309 | if (fd < 0) 310 | return -errno; 311 | 312 | len = sysconf(_SC_PAGESIZE); 313 | 314 | *buf = malloc(len); 315 | if (!*buf) { 316 | ret = -ENOMEM; 317 | goto out_fd; 318 | } 319 | 320 | ret = read(fd, *buf, len - 1); 321 | if (ret < 0) { 322 | ret = -errno; 323 | goto out_buf; 324 | } 325 | 326 | (*buf)[ret] = 0; 327 | out_buf: 328 | if (ret < 0) 329 | free(*buf); 330 | out_fd: 331 | close(fd); 332 | return ret; 333 | } 334 | 335 | char *jd_strdup(const char *src) 336 | { 337 | char *dst; 338 | 339 | dst = strdup(src); 340 | if (!dst) 341 | err_handler(errno, "strdup()"); 342 | 343 | return dst; 344 | } 345 | 346 | FILE *jd_fopen(const char *path, const char *filename, const char *mode) 347 | { 348 | FILE *fd; 349 | char *fn, *tmp; 350 | 351 | tmp = jd_strdup(filename); 352 | if (asprintf(&fn, "%s/%s", path, basename(tmp)) < 0) 353 | err_handler(errno, "asprintf()"); 354 | 355 | fd = fopen(fn, mode); 356 | 357 | free(tmp); 358 | free(fn); 359 | 360 | return fd; 361 | } 362 | 363 | void jd_cp(const char *src, const char *path) 364 | { 365 | char buf[BUFSIZ]; 366 | FILE *fds, *fdd; 367 | size_t n; 368 | 369 | fds = fopen(src, "r"); 370 | if (!fds) { 371 | warn_handler("Could not open '%s' for reading", src); 372 | return; 373 | } 374 | 375 | fdd = jd_fopen(path, src, "w"); 376 | if (!fdd) { 377 | fclose(fdd); 378 | warn_handler("Could not copy '%s'", src); 379 | return; 380 | } 381 | 382 | while ((n = fread(buf, sizeof(char), sizeof(buf), fds)) > 0) { 383 | if (fwrite(buf, sizeof(char), n, fdd) != n) { 384 | warn_handler("Could not copy '%s'", src); 385 | goto out; 386 | } 387 | } 388 | out: 389 | fclose(fdd); 390 | fclose(fds); 391 | } 392 | -------------------------------------------------------------------------------- /jd_work.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "jitterdebugger.h" 11 | 12 | static pid_t bpid; 13 | 14 | int start_workload(const char *cmd) 15 | { 16 | int err; 17 | 18 | if (!cmd) 19 | return 0; 20 | 21 | bpid = fork(); 22 | if (bpid > 0) 23 | return 0; 24 | if (bpid < 0) 25 | return errno; 26 | 27 | err = setpgid(0,0); 28 | if (err) 29 | err_handler(errno, "setpgid()"); 30 | 31 | printf("start background workload: %s\n", cmd); 32 | err = execl("/bin/sh", "sh", "-c", cmd, (char *)0); 33 | if (err) { 34 | err_handler(err, "execl()"); 35 | exit(EXIT_FAILURE); 36 | } 37 | 38 | exit(0); 39 | } 40 | 41 | void stop_workload(void) 42 | { 43 | int status; 44 | int err; 45 | 46 | if (bpid == 0) 47 | return; 48 | 49 | err = killpg(bpid, SIGTERM); 50 | if (err) 51 | err_handler(errno, "kill()"); 52 | 53 | if (!waitpid(bpid, &status, 0)) 54 | err_handler(errno, "waitpid()"); 55 | 56 | err = WEXITSTATUS(status); 57 | if (WIFEXITED(status) && err) 58 | warn_handler("workload exited with %d", err); 59 | } 60 | -------------------------------------------------------------------------------- /jitterdebugger.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #define _GNU_SOURCE 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "jitterdebugger.h" 30 | 31 | #define VT100_ERASE_EOL "\033[K" 32 | #define VT100_CURSOR_UP "\033[%uA" 33 | 34 | #define NSEC_PER_SEC 1000000000 35 | #define NSEC_PER_US 1000UL 36 | 37 | /* Default test interval in us */ 38 | #define DEFAULT_INTERVAL 1000 39 | 40 | struct stats { 41 | pthread_t pid; 42 | pid_t tid; 43 | unsigned int affinity; 44 | uint64_t max; 45 | uint64_t min; 46 | uint64_t avg; 47 | unsigned int hist_size; 48 | uint64_t *hist; /* each slot is one us */ 49 | uint64_t total; 50 | uint64_t count; 51 | struct ringbuffer *rb; 52 | }; 53 | 54 | struct record_data { 55 | struct stats *stats; 56 | char *server; 57 | char *port; 58 | FILE *fd; 59 | }; 60 | 61 | static int jd_shutdown; 62 | static cpu_set_t affinity; 63 | static unsigned int num_threads; 64 | static unsigned int priority = 80; 65 | static uint64_t break_val = UINT64_MAX; 66 | static unsigned int sleep_interval_us = DEFAULT_INTERVAL; 67 | static unsigned int interval_resolution = NSEC_PER_US; 68 | static unsigned int max_loops = 0; 69 | static int trace_fd = -1; 70 | static int tracemark_fd = -1; 71 | 72 | static void sig_handler(int sig) 73 | { 74 | WRITE_ONCE(jd_shutdown, 1); 75 | } 76 | 77 | static inline int64_t ts_sub(struct timespec t1, struct timespec t2) 78 | { 79 | int64_t diff; 80 | 81 | diff = NSEC_PER_SEC * (int64_t)((int) t1.tv_sec - (int) t2.tv_sec); 82 | diff += ((int) t1.tv_nsec - (int) t2.tv_nsec); 83 | 84 | return diff / interval_resolution; 85 | } 86 | 87 | static inline struct timespec ts_add(struct timespec t1, struct timespec t2) 88 | { 89 | t1.tv_sec = t1.tv_sec + t2.tv_sec; 90 | t1.tv_nsec = t1.tv_nsec + t2.tv_nsec; 91 | 92 | while (t1.tv_nsec >= NSEC_PER_SEC) { 93 | t1.tv_nsec -= NSEC_PER_SEC; 94 | t1.tv_sec++; 95 | } 96 | 97 | return t1; 98 | } 99 | 100 | static int c_states_disable(void) 101 | { 102 | uint32_t latency = 0; 103 | int fd; 104 | 105 | /* Disable on all CPUs all C states. */ 106 | const char *cpu_dma_latency = "/dev/cpu_dma_latency"; 107 | 108 | fd = TEMP_FAILURE_RETRY(open(cpu_dma_latency, O_RDWR | O_CLOEXEC)); 109 | if (fd < 0) { 110 | if (errno == EACCES) { 111 | fprintf(stderr, "No permission to open %s\n", cpu_dma_latency); 112 | } else if (errno == ENOENT) { 113 | return fd; 114 | } 115 | err_handler(errno, "open()"); 116 | } 117 | 118 | write(fd, &latency, sizeof(latency)); 119 | 120 | return fd; 121 | } 122 | 123 | static void c_states_enable(int fd) 124 | { 125 | /* By closing the fd, the PM settings are restored. */ 126 | if (fd >= 0) 127 | close(fd); 128 | } 129 | 130 | static void open_trace_fds(void) 131 | { 132 | const char *tracing_on = "/sys/kernel/debug/tracing/tracing_on"; 133 | const char *trace_marker = "/sys/kernel/debug/tracing/trace_marker"; 134 | 135 | trace_fd = TEMP_FAILURE_RETRY(open(tracing_on, O_WRONLY)); 136 | if (trace_fd < 0) 137 | err_handler(errno, "open()"); 138 | 139 | tracemark_fd = TEMP_FAILURE_RETRY(open(trace_marker, O_WRONLY)); 140 | if (tracemark_fd < 0) 141 | err_handler(errno, "open()"); 142 | } 143 | 144 | static void stop_tracer(uint64_t diff) 145 | { 146 | char buf[128]; 147 | int len; 148 | 149 | len = snprintf(buf, 128, "Hit latency %" PRIu64, diff); 150 | write(tracemark_fd, buf, len); 151 | write(trace_fd, "0\n", 2); 152 | } 153 | 154 | static inline pid_t __gettid(void) 155 | { 156 | /* 157 | * glibc wrapper available since glibc-2.30, but use our own 158 | * implementation for compatibility with older versions. 159 | */ 160 | return syscall(SYS_gettid); 161 | } 162 | 163 | static void dump_stats(FILE *f, struct system_info *sysinfo, struct stats *s) 164 | { 165 | unsigned int i, j, comma; 166 | 167 | fprintf(f, "{\n"); 168 | fprintf(f, " \"version\": 3,\n"); 169 | fprintf(f, " \"sysinfo\": {\n"); 170 | fprintf(f, " \"sysname\": \"%s\",\n", sysinfo->sysname); 171 | fprintf(f, " \"nodename\": \"%s\",\n", sysinfo->nodename); 172 | fprintf(f, " \"release\": \"%s\",\n", sysinfo->release); 173 | fprintf(f, " \"version\": \"%s\",\n", sysinfo->version); 174 | fprintf(f, " \"machine\": \"%s\",\n", sysinfo->machine); 175 | fprintf(f, " \"cpus_online\": %d,\n", sysinfo->cpus_online); 176 | fprintf(f, " \"resolution_in_ns\": %u\n", interval_resolution); 177 | fprintf(f, " },\n"); 178 | fprintf(f, " \"cpu\": {\n"); 179 | for (i = 0; i < num_threads; i++) { 180 | fprintf(f, " \"%u\": {\n", i); 181 | 182 | fprintf(f, " \"histogram\": {"); 183 | for (j = 0, comma = 0; j < s[i].hist_size; j++) { 184 | if (!s[i].hist[j]) 185 | continue; 186 | fprintf(f, "%s", comma ? ",\n" : "\n"); 187 | fprintf(f, " \"%u\": %" PRIu64,j, s[i].hist[j]); 188 | comma = 1; 189 | } 190 | if (comma) 191 | fprintf(f, "\n"); 192 | fprintf(f, " },\n"); 193 | fprintf(f, " \"count\": %" PRIu64 ",\n", s[i].count); 194 | fprintf(f, " \"min\": %" PRIu64 ",\n", s[i].min); 195 | fprintf(f, " \"max\": %" PRIu64 ",\n", s[i].max); 196 | fprintf(f, " \"avg\": %.2f\n", 197 | (double)s[i].total / (double)s[i].count); 198 | fprintf(f, " }%s\n", i == num_threads - 1 ? "" : ","); 199 | } 200 | fprintf(f, " }\n"); 201 | fprintf(f, "}\n"); 202 | } 203 | 204 | static void __display_stats(struct stats *s) 205 | { 206 | unsigned int i; 207 | 208 | for (i = 0; i < num_threads; i++) { 209 | printf("T:%2u (%5lu) A:%2u C:%10" PRIu64 210 | " Min:%10" PRIu64 " Avg:%8.2f Max:%10" PRIu64 " " 211 | VT100_ERASE_EOL "\n", 212 | i, (long)s[i].tid, s[i].affinity, 213 | s[i].count, 214 | s[i].min, 215 | (double) s[i].total / (double) s[i].count, 216 | s[i].max); 217 | } 218 | } 219 | 220 | static void *display_stats(void *arg) 221 | { 222 | struct stats *s = arg; 223 | unsigned int i; 224 | 225 | for (i = 0; i < num_threads; i++) 226 | printf("\n"); 227 | 228 | while (!READ_ONCE(jd_shutdown)) { 229 | printf(VT100_CURSOR_UP, num_threads); 230 | 231 | __display_stats(s); 232 | 233 | fflush(stdout); 234 | usleep(100 * 1000); /* 100 ms interval */ 235 | } 236 | 237 | return NULL; 238 | } 239 | 240 | static void store_file(struct record_data *rec) 241 | { 242 | struct stats *s = rec->stats; 243 | struct latency_sample sample; 244 | struct timespec ts; 245 | uint64_t val; 246 | unsigned int i; 247 | 248 | while (!READ_ONCE(jd_shutdown)) { 249 | for (i = 0; i < num_threads; i++) { 250 | sample.cpuid = i; 251 | while (!ringbuffer_read(s[i].rb, &ts, &val)) { 252 | memcpy(&sample.ts, &ts, sizeof(sample.ts)); 253 | memcpy(&sample.val, &val, sizeof(sample.val)); 254 | fwrite(&sample, sizeof(struct latency_sample), 1, rec->fd); 255 | } 256 | } 257 | 258 | usleep(DEFAULT_INTERVAL); 259 | } 260 | } 261 | 262 | static void store_network(struct record_data *rec) 263 | { 264 | struct latency_sample sp[SAMPLES_PER_PACKET]; 265 | struct addrinfo hints, *res, *tmp; 266 | struct sockaddr *sa; 267 | socklen_t salen; 268 | int err, sk, len; 269 | unsigned int i, c; 270 | struct stats *s = rec->stats; 271 | struct timespec ts; 272 | uint64_t val; 273 | 274 | bzero(&hints, sizeof(struct addrinfo)); 275 | hints.ai_family = AF_UNSPEC; 276 | hints.ai_socktype = SOCK_DGRAM; 277 | hints.ai_protocol = IPPROTO_UDP; 278 | 279 | err = getaddrinfo(rec->server, rec->port, &hints, &res); 280 | if (err < 0) 281 | err_handler(err, "getaddrinfo()"); 282 | 283 | tmp = res; 284 | do { 285 | sk = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 286 | if (sk >= 0) 287 | break; 288 | res = res->ai_next; 289 | } while (res); 290 | if (sk < 0) 291 | err_handler(ENOENT, "no server"); 292 | 293 | sa = malloc(res->ai_addrlen); 294 | memcpy(sa, res->ai_addr, res->ai_addrlen); 295 | salen = res->ai_addrlen; 296 | 297 | freeaddrinfo(tmp); 298 | 299 | err = fcntl(sk, F_SETFL, O_NONBLOCK, 1); 300 | if (err < 0) 301 | err_handler(errno, "fcntl"); 302 | 303 | c = 0; 304 | while (!READ_ONCE(jd_shutdown)) { 305 | for (i = 0; i < num_threads; i++) { 306 | while (!ringbuffer_read(s[i].rb, &ts, &val)) { 307 | sp[c].cpuid = i; 308 | memcpy(&sp[c].ts, &ts, sizeof(sp[c].ts)); 309 | memcpy(&sp[c].val, &val, sizeof(sp[c].val)); 310 | if (c == SAMPLES_PER_PACKET - 1) { 311 | len = sendto(sk, (const void *)sp, sizeof(sp), 0, 312 | sa, salen); 313 | if (len < 0) 314 | perror("sendto"); 315 | c = 0; 316 | } else 317 | c++; 318 | } 319 | } 320 | 321 | usleep(DEFAULT_INTERVAL); 322 | } 323 | 324 | close(sk); 325 | } 326 | 327 | static void *store_samples(void *arg) 328 | { 329 | struct record_data *rec = arg; 330 | if (rec->fd) 331 | store_file(rec); 332 | else 333 | store_network(rec); 334 | 335 | return NULL; 336 | } 337 | 338 | static void *worker(void *arg) 339 | { 340 | struct stats *s = arg; 341 | struct timespec now, next, interval; 342 | sigset_t mask; 343 | uint64_t diff; 344 | int err; 345 | 346 | /* Don't handle any signals */ 347 | sigfillset(&mask); 348 | if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) 349 | err_handler(errno, "sigprocmask()"); 350 | 351 | s->tid = __gettid(); 352 | 353 | interval.tv_sec = 0; 354 | interval.tv_nsec = sleep_interval_us * NSEC_PER_US; 355 | 356 | err = clock_gettime(CLOCK_MONOTONIC, &now); 357 | if (err) 358 | err_handler(err, "clock_gettime()"); 359 | 360 | next = now; 361 | 362 | while (!READ_ONCE(jd_shutdown)) { 363 | next = ts_add(next, interval); 364 | 365 | err = clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, 366 | &next, NULL); 367 | if (err) 368 | err_handler(err, "clock_nanosleep()"); 369 | 370 | err = clock_gettime(CLOCK_MONOTONIC, &now); 371 | if (err) 372 | err_handler(err, "clock_gettime()"); 373 | 374 | /* Update the statistics */ 375 | diff = ts_sub(now, next); 376 | if (diff > s->max) 377 | s->max = diff; 378 | 379 | if (diff < s->min) 380 | s->min = diff; 381 | 382 | s->count++; 383 | s->total += diff; 384 | 385 | if (diff < s->hist_size) 386 | s->hist[diff]++; 387 | 388 | if (s->rb) 389 | ringbuffer_write(s->rb, now, diff); 390 | 391 | if (diff > break_val) { 392 | stop_tracer(diff); 393 | WRITE_ONCE(jd_shutdown, 1); 394 | } 395 | 396 | if (max_loops > 0 && s->count >= max_loops) 397 | break; 398 | } 399 | 400 | return NULL; 401 | } 402 | 403 | static void start_measuring(struct stats *s, struct record_data *rec) 404 | { 405 | struct sched_param sched; 406 | pthread_attr_t attr; 407 | cpu_set_t mask; 408 | unsigned int i, t; 409 | int err; 410 | 411 | pthread_attr_init(&attr); 412 | 413 | for (i = 0, t = 0; i < num_threads; i++) { 414 | CPU_ZERO(&mask); 415 | 416 | /* 417 | * Skip unset cores. will not crash as long as 418 | * num_threads <= CPU_COUNT(&affinity) 419 | */ 420 | while (!CPU_ISSET(t, &affinity)) 421 | t++; 422 | 423 | CPU_SET(t, &mask); 424 | 425 | /* Don't stay on the same core in next loop */ 426 | s[i].affinity = t++; 427 | s[i].min = UINT64_MAX; 428 | s[i].hist_size = NSEC_PER_SEC / interval_resolution / 1000; 429 | s[i].hist = calloc(s[i].hist_size, sizeof(uint64_t)); 430 | if (!s[i].hist) 431 | err_handler(errno, "calloc()"); 432 | 433 | if (rec) { 434 | s[i].rb = ringbuffer_create(1024 * 1024); 435 | if (!s[i].rb) 436 | err_handler(ENOMEM, "ringbuffer_create()"); 437 | } 438 | 439 | err = pthread_attr_setaffinity_np(&attr, sizeof(mask), &mask); 440 | if (err) 441 | err_handler(err, "pthread_attr_setaffinity_np()"); 442 | 443 | err = pthread_attr_setschedpolicy(&attr, SCHED_FIFO); 444 | if (err) 445 | err_handler(err, "pthread_attr_setschedpolicy()"); 446 | 447 | sched.sched_priority = priority; 448 | err = pthread_attr_setschedparam(&attr, &sched); 449 | if (err) 450 | err_handler(err, "pthread_attr_setschedparam()"); 451 | 452 | err = pthread_attr_setinheritsched(&attr, 453 | PTHREAD_EXPLICIT_SCHED); 454 | if (err) 455 | err_handler(err, "pthread_attr_setinheritsched()"); 456 | 457 | err = pthread_create(&s[i].pid, &attr, &worker, &s[i]); 458 | if (err) { 459 | if (err == EPERM) 460 | fprintf(stderr, "No permission to set the " 461 | "scheduling policy and/or priority\n"); 462 | else if (err == EINVAL) 463 | fprintf(stderr, "Invalid settings in thread attributes. " 464 | "Check your affinity mask\n"); 465 | err_handler(err, "pthread_create()"); 466 | } 467 | } 468 | 469 | pthread_attr_destroy(&attr); 470 | } 471 | 472 | static struct option long_options[] = { 473 | { "help", no_argument, 0, 'h' }, 474 | { "verbose", no_argument, 0, 'v' }, 475 | { "version", no_argument, 0, 0 }, 476 | { "command", required_argument, 0, 'c' }, 477 | 478 | { "loops", required_argument, 0, 'l' }, 479 | { "duration", required_argument, 0, 'D' }, 480 | { "break", required_argument, 0, 'b' }, 481 | { "nsec", no_argument, 0, 'N' }, 482 | { "interval", required_argument, 0, 'i' }, 483 | { "output", required_argument, 0, 'o' }, 484 | 485 | { "affinity", required_argument, 0, 'a' }, 486 | { "priority", required_argument, 0, 'p' }, 487 | 488 | { 0, }, 489 | }; 490 | 491 | static void __attribute__((noreturn)) usage(int status) 492 | { 493 | printf("jitterdebugger [options]\n"); 494 | printf("\n"); 495 | printf("General usage:\n"); 496 | printf(" -h, --help Print this help\n"); 497 | printf(" -v, --verbose Print live statistics\n"); 498 | printf(" --version Print version of jitterdebugger\n"); 499 | printf(" -o, --output DIR Store collected data into DIR\n"); 500 | printf(" -c, --command CMD Execute CMD (workload) in background\n"); 501 | printf("\n"); 502 | printf("Sampling:\n"); 503 | printf(" -l, --loops VALUE Max number of measurements\n"); 504 | printf(" -D, --duration TIME Specify a length for the test run.\n"); 505 | printf(" Append 'm', 'h', or 'd' to specify minutes, hours or days.\n"); 506 | printf(" -b, --break VALUE Stop if max latency exceeds VALUE.\n"); 507 | printf(" Also the tracers\n"); 508 | printf(" -N, --nsec Meassurement in nano seconds\n"); 509 | printf(" -i, --interval TIME Sleep interval for sampling threads in microseconds\n"); 510 | printf(" or nano seconds (see -N/--nsec)\n"); 511 | printf(" -n Send samples to host:port\n"); 512 | printf(" -s Store samples into --output DIR\n"); 513 | printf("\n"); 514 | printf("Threads: \n"); 515 | printf(" -a, --affinity CPUSET Core affinity specification\n"); 516 | printf(" e.g. 0,2,5-7 starts a thread on first, third and last two\n"); 517 | printf(" cores on a 8-core system.\n"); 518 | printf(" May also be set in hexadecimal with '0x' prefix\n"); 519 | printf(" -p, --priority PRI Worker thread priority. [1..98]\n"); 520 | 521 | exit(status); 522 | } 523 | 524 | int main(int argc, char *argv[]) 525 | { 526 | struct sigaction sa; 527 | unsigned int i; 528 | int c, fd, err; 529 | struct stats *s; 530 | pthread_t pid, iopid; 531 | cpu_set_t affinity_available, affinity_set; 532 | int long_idx; 533 | long val; 534 | struct record_data *rec = NULL; 535 | FILE *rfd = NULL; 536 | struct system_info *sysinfo; 537 | 538 | /* Command line options */ 539 | unsigned int opt_duration = 0; 540 | char *opt_dir = NULL; 541 | char *opt_cmd = NULL; 542 | char *opt_net = NULL; 543 | int opt_samples = 0; 544 | int opt_verbose = 0; 545 | 546 | CPU_ZERO(&affinity_set); 547 | 548 | while (1) { 549 | c = getopt_long(argc, argv, "c:n:sp:vD:l:b:Ni:o:a:h", long_options, 550 | &long_idx); 551 | if (c < 0) 552 | break; 553 | 554 | switch (c) { 555 | case 0: 556 | if (!strcmp(long_options[long_idx].name, "version")) { 557 | printf("jitterdebugger %s\n", 558 | JD_VERSION); 559 | exit(0); 560 | } 561 | break; 562 | case 'o': 563 | opt_dir = optarg; 564 | break; 565 | case 'c': 566 | opt_cmd = optarg; 567 | break; 568 | case 'n': 569 | opt_net = optarg; 570 | break; 571 | case 's': 572 | opt_samples = 1; 573 | break; 574 | case 'p': 575 | val = parse_dec(optarg); 576 | if (val < 1 || val > 98) 577 | err_abort("Invalid value for priority. " 578 | "Valid range is [1..98]\n"); 579 | priority = val; 580 | break; 581 | case 'v': 582 | opt_verbose = 1; 583 | break; 584 | case 'D': 585 | val = parse_time(optarg); 586 | if (val < 0) 587 | err_abort("Invalid value for duration. " 588 | "Valid postfixes are 'd', 'h', 'm', 's'\n"); 589 | opt_duration = val; 590 | break; 591 | case 'l': 592 | val = parse_dec(optarg); 593 | if (val <= 0) 594 | err_abort("Invalid value for loops. " 595 | "Valid range is [1..]\n"); 596 | max_loops = val; 597 | break; 598 | case 'b': 599 | val = parse_dec(optarg); 600 | if (val <= 0) 601 | err_abort("Invalid value for break. " 602 | "Valid range is [1..]\n"); 603 | break_val = val; 604 | break; 605 | case 'N': 606 | interval_resolution = 1; 607 | break; 608 | case 'i': 609 | val = parse_dec(optarg); 610 | if (val < 1) 611 | err_abort("Invalid value for interval. " 612 | "Valid range is [1..]. " 613 | "Default: %u us.\n", sleep_interval_us); 614 | sleep_interval_us = val; 615 | break; 616 | case 'h': 617 | usage(0); 618 | case 'a': 619 | val = cpuset_parse(&affinity_set, optarg); 620 | if (val < 0) { 621 | fprintf(stderr, "Invalid value for affinity. Valid range is [0..]\n"); 622 | exit(1); 623 | } 624 | break; 625 | default: 626 | usage(1); 627 | } 628 | } 629 | 630 | if (geteuid() != 0) 631 | printf("jitterdebugger is not running with root rights.\n"); 632 | 633 | sysinfo = collect_system_info(); 634 | if (opt_dir) { 635 | err = mkdir(opt_dir, 0777); 636 | if (err) { 637 | if (errno != EEXIST) { 638 | err_handler(errno, 639 | "Creating directory '%s' failed\n", 640 | opt_dir); 641 | } 642 | warn_handler("Directory '%s' already exist: overwriting contents", opt_dir); 643 | } else { 644 | store_system_info(opt_dir, sysinfo); 645 | } 646 | } 647 | 648 | if (opt_net || opt_samples) { 649 | if (opt_net && opt_samples) { 650 | fprintf(stdout, "Can't use both options -s or -n together\n"); 651 | exit(1); 652 | } 653 | 654 | rec = malloc(sizeof(*rec)); 655 | if (!rec) 656 | err_handler(ENOMEM, "malloc()"); 657 | 658 | if (opt_net) { 659 | rec->server = strtok(opt_net, " :"); 660 | rec->port = strtok(NULL, " :"); 661 | 662 | if (!rec->server || !rec->port) { 663 | fprintf(stdout, "Invalid server name and/or port string\n"); 664 | exit(1); 665 | } 666 | } 667 | 668 | if (opt_samples) { 669 | if (!opt_dir) { 670 | fprintf(stdout, "-o/--output is needed with -s option\n"); 671 | exit(1); 672 | } 673 | rec->fd = jd_fopen(opt_dir, "samples.raw", "w"); 674 | if (!rec->fd) 675 | err_handler(errno, "Couldn't create samples.raw file"); 676 | } 677 | } 678 | 679 | sa.sa_flags = 0; 680 | sa.sa_handler = sig_handler; 681 | sigemptyset(&sa.sa_mask); 682 | 683 | if (sigaction(SIGINT, &sa, NULL) < 0) 684 | err_handler(errno, "sigaction()"); 685 | 686 | if (sigaction(SIGTERM, &sa, NULL) < 0) 687 | err_handler(errno, "sigaction()"); 688 | 689 | if (sigaction(SIGALRM, &sa, NULL) < 0) 690 | err_handler(errno, "sigaction()"); 691 | 692 | if (opt_duration > 0) 693 | alarm(opt_duration); 694 | 695 | if (mlockall(MCL_CURRENT|MCL_FUTURE) < 0) { 696 | if (errno == ENOMEM || errno == EPERM) 697 | fprintf(stderr, "Nonzero RTLIMIT_MEMLOCK soft resource " 698 | "limit or missing process privileges " 699 | "(CAP_IPC_LOCK)\n"); 700 | err_handler(errno, "mlockall()"); 701 | } 702 | 703 | fd = c_states_disable(); 704 | 705 | if (break_val != UINT64_MAX) 706 | open_trace_fds(); 707 | 708 | if (CPU_COUNT(&affinity_set)) { 709 | /* 710 | * The user is able to override the affinity mask with 711 | * a set which contains more CPUs than 712 | * sched_getaffinity() returns. 713 | */ 714 | affinity = affinity_set; 715 | } else { 716 | if (sched_getaffinity(0, sizeof(cpu_set_t), &affinity_available)) 717 | err_handler(errno, "sched_getaffinity()"); 718 | affinity = affinity_available; 719 | } 720 | 721 | if (opt_verbose) { 722 | printf("affinity: "); 723 | cpuset_fprint(stdout, &affinity); 724 | printf("\n"); 725 | } 726 | 727 | num_threads = CPU_COUNT(&affinity); 728 | s = calloc(num_threads, sizeof(struct stats)); 729 | if (!s) 730 | err_handler(errno, "calloc()"); 731 | 732 | err = start_workload(opt_cmd); 733 | if (err < 0) 734 | err_handler(errno, "starting workload failed"); 735 | 736 | start_measuring(s, rec); 737 | 738 | if (opt_net || opt_samples) { 739 | rec->stats = s; 740 | err = pthread_create(&iopid, NULL, store_samples, rec); 741 | if (err) 742 | err_handler(err, "pthread_create()"); 743 | } 744 | 745 | if (opt_verbose) { 746 | err = pthread_create(&pid, NULL, display_stats, s); 747 | if (err) 748 | err_handler(err, "pthread_create()"); 749 | } 750 | 751 | for (i = 0; i < num_threads; i++) { 752 | err = pthread_join(s[i].pid, NULL); 753 | if (err) 754 | err_handler(err, "pthread_join()"); 755 | } 756 | 757 | WRITE_ONCE(jd_shutdown, 1); 758 | stop_workload(); 759 | 760 | if (rec) { 761 | err = pthread_join(iopid, NULL); 762 | if (err) 763 | err_handler(err, "pthread_join()"); 764 | 765 | if (rec->fd) 766 | fclose(rec->fd); 767 | free(rec); 768 | } 769 | 770 | if (opt_verbose) { 771 | err = pthread_join(pid, NULL); 772 | if (err) 773 | err_handler(err, "pthread_join()"); 774 | } else { 775 | printf("\n"); 776 | __display_stats(s); 777 | } 778 | 779 | printf("\n"); 780 | 781 | if (opt_dir) { 782 | rfd = jd_fopen(opt_dir, "results.json", "w"); 783 | if (rfd) { 784 | dump_stats(rfd, sysinfo, s); 785 | fclose(rfd); 786 | } else { 787 | warn_handler("Couldn't create results.json"); 788 | } 789 | free_system_info(sysinfo); 790 | } 791 | 792 | if (opt_verbose && break_val != UINT_MAX) { 793 | for (i = 0; i < num_threads; i++) { 794 | if (s[i].max > break_val) { 795 | const char *unit = "us"; 796 | if (interval_resolution == 1) 797 | unit = "ns"; 798 | printf("Thread %lu on CPU %u hit %" PRIu64 " %s latency\n", 799 | (long)s[i].tid, i, s[i].max, unit); 800 | } 801 | } 802 | } 803 | 804 | for (i = 0; i < num_threads; i++) { 805 | free(s[i].hist); 806 | if (s[i].rb) 807 | ringbuffer_free(s[i].rb); 808 | } 809 | free(s); 810 | 811 | if (tracemark_fd > 0) 812 | close(tracemark_fd); 813 | 814 | if (trace_fd > 0) 815 | close(trace_fd); 816 | 817 | c_states_enable(fd); 818 | 819 | return 0; 820 | } 821 | -------------------------------------------------------------------------------- /jitterdebugger.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #ifndef __JITTERDEBUGGER_H 4 | #define __JITTERDEBUGGER_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #define JD_VERSION "0.3" 13 | 14 | // Results in a 1400 bytes payload per UDP packet 15 | #define SAMPLES_PER_PACKET 50 16 | 17 | #define READ_ONCE(x) \ 18 | ({ \ 19 | union { typeof(x) __v; char __t[1]; } __u = { .__t = { 0 } }; \ 20 | *(typeof(x) *) __u.__t = *(volatile typeof(x) *) &x; \ 21 | __u.__v; \ 22 | }) 23 | 24 | #define WRITE_ONCE(x, v) \ 25 | ({ \ 26 | union { typeof(x) __v; char __t[1]; } __u = { .__v = (v) } ; \ 27 | *(volatile typeof(x) *) &x = *(typeof(x) *) __u.__t; \ 28 | __u.__v; \ 29 | }) 30 | 31 | struct latency_sample { 32 | uint32_t cpuid; 33 | struct timespec ts; 34 | uint64_t val; 35 | } __attribute__((packed)); 36 | 37 | struct ringbuffer; 38 | 39 | struct ringbuffer *ringbuffer_create(unsigned int size); 40 | void ringbuffer_free(struct ringbuffer *rb); 41 | int ringbuffer_read(struct ringbuffer *rb, struct timespec *ts, uint64_t *val); 42 | int ringbuffer_write(struct ringbuffer *rb, struct timespec ts, uint64_t val); 43 | 44 | void _err_handler(int error, char *format, ...) 45 | __attribute__((format(printf, 2, 3))); 46 | void _warn_handler(char *format, ...) 47 | __attribute__((format(printf, 1, 2))); 48 | 49 | #define err_handler(error, fmt, arg...) do { \ 50 | _err_handler(error, "%s:%s(): " fmt, \ 51 | __FILE__, __func__, ## arg); \ 52 | } while (0) 53 | #define warn_handler(fmt, arg...) do { \ 54 | _warn_handler("%s:%s(): " fmt "\n", \ 55 | __FILE__, __func__, ## arg); \ 56 | } while (0) 57 | #define err_abort(fmt, arg...) do { \ 58 | fprintf(stderr, fmt "\n", ## arg); \ 59 | exit(1); \ 60 | } while (0) 61 | 62 | long int parse_num(const char *str, int base, size_t *len); 63 | long int parse_time(const char *str); 64 | 65 | static inline long int parse_dec(const char *str) 66 | { 67 | return parse_num(str, 10, NULL); 68 | } 69 | 70 | int sysfs_load_str(const char *path, char **buf); 71 | 72 | /* cpu_set_t helpers */ 73 | void cpuset_fprint(FILE *f, cpu_set_t *set); 74 | ssize_t cpuset_parse(cpu_set_t *set, const char *str); 75 | 76 | int start_workload(const char *cmd); 77 | void stop_workload(void); 78 | 79 | struct system_info { 80 | char *sysname; 81 | char *nodename; 82 | char *release; 83 | char *version; 84 | char *machine; 85 | int cpus_online; 86 | }; 87 | 88 | struct system_info *collect_system_info(void); 89 | void store_system_info(const char *path, struct system_info *sysinfo); 90 | void free_system_info(struct system_info *sysinfo); 91 | 92 | char *jd_strdup(const char *src); 93 | FILE *jd_fopen(const char *path, const char *filename, const char *mode); 94 | void jd_cp(const char *src, const char *path); 95 | 96 | struct jd_samples_info { 97 | const char *dir; 98 | unsigned int cpus_online; 99 | }; 100 | 101 | struct jd_samples_ops { 102 | const char *name; 103 | const char *format; 104 | int (*output)(struct jd_samples_info *info, FILE *input); 105 | }; 106 | 107 | int jd_samples_register(struct jd_samples_ops *ops); 108 | void jd_samples_unregister(struct jd_samples_ops *ops); 109 | 110 | struct jd_plugin_desc { 111 | const char *name; 112 | int (*init)(void); 113 | void (*cleanup)(void); 114 | }; 115 | 116 | #define JD_PLUGIN_ID(x) jd_plugin_desc __jd_builtin_ ## x 117 | #define JD_PLUGIN_DEFINE_1(name, init, cleanup) \ 118 | struct JD_PLUGIN_ID(name) = { \ 119 | #name, init, cleanup \ 120 | }; 121 | #define JD_PLUGIN_DEFINE(init, cleanup) \ 122 | JD_PLUGIN_DEFINE_1(_FILENAME, init, cleanup) 123 | 124 | void __jd_plugin_init(void); 125 | void __jd_plugin_cleanup(void); 126 | 127 | // XXX replace single list with some library implementation 128 | struct jd_slist { 129 | void *data; 130 | struct jd_slist *next; 131 | }; 132 | 133 | void jd_slist_append(struct jd_slist *jd_slist, void *data); 134 | void jd_slist_remove(struct jd_slist *jd_slist, void *data); 135 | 136 | #endif /* __JITTERDEBUGGER_H */ 137 | -------------------------------------------------------------------------------- /jitterplot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: MIT 3 | 4 | import os 5 | import sys 6 | import json 7 | import argparse 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import pandas as pd 11 | 12 | __version__ = '0.3' 13 | 14 | # silence SettingWithCopyWarning globally 15 | # https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#evaluation-order-matters 16 | pd.options.mode.chained_assignment = None 17 | 18 | 19 | def plot_cdf(filename, outfilename): 20 | with open(filename) as file: 21 | rawdata = json.load(file) 22 | 23 | cpu_id = 0 24 | fig, ax = plt.subplots() 25 | while True: 26 | if str(cpu_id) not in rawdata['cpu']: 27 | break 28 | 29 | cid = str(cpu_id) 30 | data = rawdata['cpu'][cid] 31 | bins = [int(i) for i in data['histogram'].keys()] 32 | 33 | values = [int(i) for i in data['histogram'].values()] 34 | cumulative = np.cumsum(values) / np.sum(values) 35 | # show 99.9 and 99.99 percentiles in legend 36 | p3_idx = list(map(lambda i: i > 0.999, cumulative)).index(True) 37 | p4_idx = list(map(lambda i: i > 0.9999, cumulative)).index(True) 38 | p3 = bins[p3_idx] 39 | p4 = bins[p4_idx] 40 | pmax = bins[-1] 41 | ax.step(bins, cumulative, 42 | label='cpu{}: p99.9={}, p99.99={}, max={}' 43 | .format(cid, p3, p4, pmax)) 44 | 45 | cpu_id = cpu_id + 1 46 | 47 | L = ax.legend() 48 | plt.grid(color='lightgrey', linestyle='-', linewidth=1, which='both') 49 | plt.yticks(ticks=np.arange(0, 1.1, 0.1)) 50 | plt.xlabel('jitter [us]') 51 | plt.ylabel('probability') 52 | 53 | plt.setp(L.texts, family='monospace') 54 | if outfilename is not None: 55 | plt.savefig(outfilename) 56 | plt.show() 57 | 58 | 59 | def plot_histogram(filename, outfilename): 60 | with open(filename) as file: 61 | rawdata = json.load(file) 62 | 63 | cpu_id = 0 64 | fig, ax = plt.subplots() 65 | while True: 66 | if str(cpu_id) not in rawdata['cpu']: 67 | break 68 | 69 | cid = str(cpu_id) 70 | data = rawdata['cpu'][cid] 71 | d = {int(k): int(v) for k, v in data['histogram'].items()} 72 | lbl = 'cpu{} min{:>3} avg{:>7} max{:>3}'.format( 73 | cid, 74 | rawdata['cpu'][cid]['min'], 75 | rawdata['cpu'][cid]['avg'], 76 | rawdata['cpu'][cid]['max']) 77 | ax.bar(list(d.keys()), list(d.values()), 78 | log=True, alpha=0.5, label=lbl) 79 | 80 | cpu_id = cpu_id + 1 81 | 82 | L = ax.legend() 83 | plt.setp(L.texts, family='monospace') 84 | plt.xlabel('jitter [us]') 85 | plt.ylabel('frequency') 86 | if outfilename is not None: 87 | plt.savefig(outfilename) 88 | plt.show() 89 | 90 | 91 | def load_samples(filename): 92 | dt = np.dtype([('CPUID', 'u4'), 93 | ('Seconds', 'u8'), 94 | ('Nanoseconds', 'u8'), 95 | ('Value', 'u8')]) 96 | data = np.fromfile(filename, dtype=dt) 97 | df = pd.DataFrame(data) 98 | return df 99 | 100 | 101 | def plot_all_cpus(df, outfilename): 102 | ids = df["CPUID"].unique() 103 | max_jitter = max(df["Value"]) 104 | 105 | fig = plt.figure() 106 | axes = fig.subplots(len(ids)) 107 | for ax, data in zip( 108 | iter(axes), 109 | (df[df["CPUID"] == id] for id in ids), 110 | ): 111 | data["Time"] = data["Seconds"] + data["Nanoseconds"] * 10**-9 112 | ax.plot("Time", "Value", data=data) 113 | ax.set_xlabel("Time [s]") 114 | ax.set_ylabel("Latency [us]") 115 | ax.set_ylim(bottom=0, top=max_jitter) 116 | if outfilename is not None: 117 | plt.savefig(outfilename) 118 | plt.show() 119 | 120 | 121 | def main(): 122 | ap = argparse.ArgumentParser( 123 | description='Plot statistics collected with jitterdebugger') 124 | ap.add_argument('--version', action='version', 125 | version='%(prog)s ' + __version__) 126 | ap.add_argument('--output', help='output file name to save figure', 127 | default=None, action='store', type=str) 128 | sap = ap.add_subparsers(dest='cmd') 129 | 130 | hrs = sap.add_parser('hist', help='Print historgram') 131 | hrs.add_argument('HIST_FILE') 132 | 133 | crs = sap.add_parser('cdf', help='Plot cumulative densitiy function') 134 | crs.add_argument('CDF_FILE') 135 | 136 | srs = sap.add_parser('samples', help='Plot samples graph') 137 | srs.add_argument('SAMPLE_FILE') 138 | 139 | args = ap.parse_args(sys.argv[1:]) 140 | if args.cmd == 'hist': 141 | fname = args.HIST_FILE 142 | if os.path.isdir(fname): 143 | fname = fname + '/results.json' 144 | plot_histogram(fname, args.output) 145 | elif args.cmd == 'cdf': 146 | fname = args.CDF_FILE 147 | if os.path.isdir(fname): 148 | fname = fname + '/results.json' 149 | plot_cdf(fname, args.output) 150 | elif args.cmd == 'samples': 151 | fname = args.SAMPLE_FILE 152 | if os.path.isdir(fname): 153 | fname = fname + '/samples.raw' 154 | 155 | df = load_samples(fname) 156 | plot_all_cpus(df, args.output) 157 | 158 | 159 | if __name__ == '__main__': 160 | main() 161 | -------------------------------------------------------------------------------- /jittersamples.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | #define _GNU_SOURCE 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "jitterdebugger.h" 19 | 20 | static void read_online_cpus(struct jd_samples_info *info) 21 | { 22 | FILE *fd; 23 | int n; 24 | 25 | fd = jd_fopen(info->dir, "cpus_online", "r"); 26 | if (!fd) 27 | err_handler(errno, "Could not read %s/cpus_online\n", info->dir); 28 | n = fscanf(fd, "%d\n", &info->cpus_online); 29 | if (n == EOF) { 30 | if (ferror(fd)) { 31 | err_handler(errno, "fscanf()"); 32 | perror("fscanf"); 33 | } else { 34 | fprintf(stderr, "cpus_online: No matching characters, no matching failure\n"); 35 | exit(1); 36 | } 37 | } else if (n != 1) { 38 | fprintf(stderr, "fscan() read more then one element\n"); 39 | exit(1); 40 | } 41 | fclose(fd); 42 | 43 | if (info->cpus_online < 1) { 44 | fprintf(stderr, "invalid input from cpus_online\n"); 45 | exit(1); 46 | } 47 | } 48 | 49 | static void dump_samples(const char *port) 50 | { 51 | struct addrinfo hints, *res, *tmp; 52 | struct latency_sample sp[SAMPLES_PER_PACKET]; 53 | ssize_t len; 54 | size_t wlen; 55 | int err, sk; 56 | 57 | bzero(&hints, sizeof(struct addrinfo)); 58 | hints.ai_flags = AI_PASSIVE; 59 | hints.ai_family = AF_UNSPEC; 60 | hints.ai_socktype = SOCK_DGRAM; 61 | hints.ai_protocol = IPPROTO_UDP; 62 | 63 | err = getaddrinfo(NULL, port, &hints, &res); 64 | if (err < 0) 65 | err_handler(errno, "getaddrinfo()"); 66 | 67 | tmp = res; 68 | do { 69 | sk = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 70 | if (sk < 0) 71 | continue; 72 | err = bind(sk, res->ai_addr, res->ai_addrlen); 73 | if (err == 0) 74 | break; 75 | 76 | close(sk); 77 | res = res->ai_next; 78 | } while(res); 79 | 80 | freeaddrinfo(tmp); 81 | 82 | while (1) { 83 | len = recvfrom(sk, sp, sizeof(sp), 0, NULL, NULL); 84 | if (len != sizeof(sp)) { 85 | warn_handler("UDP packet has wrong size\n"); 86 | continue; 87 | } 88 | wlen = fwrite(sp, sizeof(sp), 1, stdout); 89 | if (wlen != sizeof(sp)) 90 | err_handler(errno, "fwrite()"); 91 | } 92 | 93 | close(sk); 94 | } 95 | 96 | struct jd_slist jd_samples_plugins = { 97 | NULL, 98 | }; 99 | 100 | int jd_samples_register(struct jd_samples_ops *ops) 101 | { 102 | jd_slist_append(&jd_samples_plugins, ops); 103 | 104 | return 0; 105 | } 106 | 107 | void jd_samples_unregister(struct jd_samples_ops *ops) 108 | { 109 | jd_slist_remove(&jd_samples_plugins, ops); 110 | } 111 | 112 | static struct option long_options[] = { 113 | { "help", no_argument, 0, 'h' }, 114 | { "version", no_argument, 0, 0 }, 115 | { "format", required_argument, 0, 'f' }, 116 | { "listen", required_argument, 0, 'l' }, 117 | { 0, }, 118 | }; 119 | 120 | static void __attribute__((noreturn)) usage(int status) 121 | { 122 | printf("jittersamples [options] [DIR]\n"); 123 | printf(" DIR Directory generated by jitterdebugger --output\n"); 124 | printf("\n"); 125 | printf("Usage:\n"); 126 | printf(" -h, --help Print this help\n"); 127 | printf(" --version Print version of jittersamples\n"); 128 | printf(" -f, --format FMT Exporting samples in format [csv, hdf5]\n"); 129 | printf(" -l, --listen PORT Listen on PORT, dump samples to stdout\n"); 130 | 131 | exit(status); 132 | } 133 | 134 | int main(int argc, char *argv[]) 135 | { 136 | FILE *input; 137 | int c, long_idx; 138 | char *format = "csv"; 139 | char *port = NULL; 140 | struct jd_samples_info info; 141 | struct jd_slist *list; 142 | 143 | while (1) { 144 | c = getopt_long(argc, argv, "hf:l:", long_options, &long_idx); 145 | if (c < 0) 146 | break; 147 | 148 | switch (c) { 149 | case 0: 150 | if (!strcmp(long_options[long_idx].name, "version")) { 151 | printf("jittersamples %s\n", 152 | JD_VERSION); 153 | exit(0); 154 | } 155 | break; 156 | case 'h': 157 | usage(0); 158 | case 'f': 159 | format = optarg; 160 | break; 161 | case 'l': 162 | port = optarg; 163 | break; 164 | default: 165 | printf("unknown option\n"); 166 | usage(1); 167 | } 168 | } 169 | 170 | if (port) { 171 | dump_samples(port); 172 | exit(0); 173 | } 174 | 175 | if (optind == argc) { 176 | fprintf(stderr, "Missing input DIR\n"); 177 | usage(1); 178 | } 179 | info.dir = argv[optind]; 180 | 181 | read_online_cpus(&info); 182 | 183 | __jd_plugin_init(); 184 | 185 | input = jd_fopen(info.dir, "samples.raw", "r"); 186 | if (!input) 187 | err_handler(errno, "Could not open '%s/samples.raw' for reading", info.dir); 188 | 189 | for (list = jd_samples_plugins.next; list; list = list->next) { 190 | struct jd_samples_ops *plugin = list->data; 191 | 192 | if (strcmp(plugin->format, format)) 193 | continue; 194 | 195 | plugin->output(&info, input); 196 | break; 197 | } 198 | 199 | fclose(input); 200 | 201 | __jd_plugin_cleanup(); 202 | 203 | if (!list) { 204 | fprintf(stderr, "Unsupported file format \"%s\"\n", format); 205 | exit(1); 206 | } 207 | 208 | return 0; 209 | } 210 | -------------------------------------------------------------------------------- /man/jitterdebugger.1: -------------------------------------------------------------------------------- 1 | .\" SPDX-License-Identifier: MIT 2 | .TH JITTERDEBUGGER 1 3 | .SH NAME 4 | jitterdebugger \- measures wake up latencies 5 | .SH SYNOPSIS 6 | .B jitterdebugger [OPTIONS] 7 | .SH DESCRIPTION 8 | .B jitterdebugger 9 | measures wake up latencies. jitterdebugger starts a 10 | thread on each CPU which programs a timer and measures the time it 11 | takes from the timer expiring until the thread which set the timer 12 | runs again. 13 | 14 | jitterdebugger default settings will produce a correct meassurement. 15 | 16 | The program runs until CTRL-C or SIGTERM is received. The results are 17 | printed to STDOUT as JSON encoded string. 18 | .SH OPTIONS 19 | .TP 20 | .BI "-h, --help" 21 | Show help text and exit. 22 | .TP 23 | .BI "-v, --verbose" 24 | Show live updates of the measurments 25 | .TP 26 | .BI "-f, --file=" FILE 27 | Write the results into the FILE 28 | .TP 29 | .BI "-c, --command=" CMD 30 | Start workload CMD in background, e.g. -c "while true; do; hackbench; 31 | done". The CMD is executed in a full shell, that means VARIABLE 32 | expansion works as well. When jitterdebugger terminates itself 33 | (because of break value or loops), the background workload is also 34 | terminated. 35 | .TP 36 | .BI "-N " SERVER:PORT 37 | Send samples to SERVER:PORT as UDP packets. See also jittersamples --listen. 38 | .TP 39 | .BI "-t, --timeout=" N 40 | Run meassurments for N seconds. N is allowed to be postfix with 'd' 41 | (days), 'h' (hours), 'm' (minutes) or 's' seconds. 42 | .TP 43 | .BI "-l, --loops=" N 44 | Run meassurments N times and the terminate jitterdebugger. 45 | .TP 46 | .BI "-b, --break=" N 47 | Run jitterdebugger until the N or greater latency has been 48 | observed. jitterdebugger will also stop a running tracer by writing 49 | 0 to tracing/tracing_on. Furthermore, the value observed will be 50 | written to the trace buffers tracing/trace_marker, e.g "Hit latency 51 | 249". 52 | .TP 53 | .BI "-i, --interval=" N 54 | Set the sleep time between each measuring. The default value is 1000us 55 | .TP 56 | .BI "-o, --output=" DIR 57 | Write all samples measured into directory DIR. The file is called 58 | samples.raw and it is binary encoded and can be decoded using 59 | jittersamples. Additional meta data is stored into DIR. 60 | .TP 61 | .BI "-a, --affinity=" CPUSET 62 | Set the CPU affinity mask. jitterdebugger starts only meassuring 63 | threads on CPUSET,. e.g. 0,2,5-7 starts a thread on first, third and 64 | 5, 6 and 7 CPU. 65 | May also be set in hexadecimal with '0x' prefix 66 | .TP 67 | .BI "-p, --priority=" PRI 68 | Set the priority of the meassuring threads. The default value is 69 | 98. Note priority 99 is not available because 99 should only be used 70 | for kernel housekeeping tasks. 71 | .SH EXAMPLES 72 | .EX 73 | # jitterdebugger -v 74 | affinity: 0,1 = 2 [0x3] 75 | T: 0 ( 614) A: 0 C: 13476 Min: 3 Avg: 3.08 Max: 10 76 | T: 1 ( 615) A: 1 C: 13513 Min: 3 Avg: 3.10 Max: 20 77 | ^C 78 | { 79 | "cpu": { 80 | "0": { 81 | "histogram": { 82 | "3": 4070, 83 | "4": 269, 84 | "5": 26, 85 | "6": 5, 86 | "7": 1, 87 | "8": 1, 88 | "9": 2, 89 | "10": 1 90 | }, 91 | "count": 4375, 92 | "min": 3, 93 | "max": 10, 94 | "avg": 3.08 95 | }, 96 | "1": { 97 | "histogram": { 98 | "3": 4002, 99 | "4": 320, 100 | "5": 22, 101 | "6": 4, 102 | "7": 2, 103 | "8": 1, 104 | "10": 2, 105 | "11": 1, 106 | "16": 2, 107 | "20": 1 108 | }, 109 | "count": 4357, 110 | "min": 3, 111 | "max": 20, 112 | "avg": 3.10 113 | } 114 | } 115 | } 116 | .EE 117 | .SH SEE ALSO 118 | .ad l 119 | .nh 120 | .BR jittersamples (1) 121 | .BR jitterplot (1) 122 | -------------------------------------------------------------------------------- /man/jitterplot.1: -------------------------------------------------------------------------------- 1 | .\" SPDX-License-Identifier: MIT 2 | .TH JITTERPLOT 1 3 | .SH NAME 4 | jitterplot \- plot collected samples by jitterdebugger 5 | .SH SYNOPSIS 6 | .B jitterplot [OPTIONS] {hist,samples} 7 | .SH DESCRIPTION 8 | .B jittersamples 9 | procudes plots from the collected samples by jitterdebugger. 10 | 11 | hist produces a histogram plot using the default output (JSON encoded) 12 | from jitterdebugger. 13 | 14 | samples procudes a plot using the all the collected samples by 15 | jitterdebugger in CSV format. 16 | .SH OPTIONS 17 | .TP 18 | .BI "-h, --help" 19 | Show help text and exit. 20 | .TP 21 | .BI "--version" 22 | Show jitterplot version. 23 | .TP 24 | .BI "--output " 25 | Filename to save the figure to, for non-interactive plotting. The format 26 | can be controlled via the file extension (e.g. "png", "pdf", "svg") 27 | 28 | .SH EXAMPLES 29 | .EX 30 | # jitterdebugger -f results.json 31 | ^C 32 | # jitterplot hist results.json 33 | # jitterplot --output /tmp/hist.pdf hist results.json 34 | 35 | # jitterdebugger -o samples.raw 36 | ^C 37 | # jittersamples samples.raw > samples.txt 38 | # jitterplot samples samples.txt 39 | # jitterplot --output /tmp/samples.png samples samples.txt 40 | .EE 41 | .SH SEE ALSO 42 | .ad l 43 | .nh 44 | .BR jitterdebugger (1) 45 | .BR jittersamples (1) 46 | -------------------------------------------------------------------------------- /man/jittersamples.1: -------------------------------------------------------------------------------- 1 | .\" SPDX-License-Identifier: MIT 2 | .TH JITTERSAMPLES 1 3 | .SH NAME 4 | jittersamples \- decodes collected samples by jitterdebugger 5 | .SH SYNOPSIS 6 | .B jittersamples hist [OPTIONS] 7 | .SH DESCRIPTION 8 | .B jittersamples 9 | prints all samples stored by jitterdebugger to standard output. 10 | 11 | The input file is expected to be in raw format. The default output is 12 | in CSV (Comma Separated Values) format. 13 | 14 | The fields are: 15 | .IP \[bu] 16 | CPUID 17 | .IP \[bu] 18 | Timestamp in seconds 19 | .IP \[bu] 20 | Wake up latency in micro seconds 21 | .SH OPTIONS 22 | .TP 23 | .BI "-h, --help" 24 | Show help text and exit. 25 | .TP 26 | .BI "-f, --format" FMT 27 | Set output format. Supported formats are CSV and HDF5. For HDF5 an 28 | output file has to be provided via --output command line. 29 | .TP 30 | .BI "-o, --output" FILE 31 | Write data to FILE instead to STDOUT. 32 | .TP 33 | .BI "-l, --listen" PORT 34 | Listen on PORT for incoming samples and store the data into FILE in raw format. 35 | .SH EXAMPLES 36 | .EX 37 | # jitterdebugger -o samples.raw 38 | ^C 39 | # jittersamples samples.raw | head 40 | 0;1114.936950838;9 41 | 0;1114.937204763;3 42 | 0;1114.937458457;3 43 | 0;1114.937711970;3 44 | 0;1114.937965595;3 45 | 0;1114.938218986;3 46 | 0;1114.938472416;3 47 | 0;1114.938725788;3 48 | 0;1114.938979191;3 49 | 0;1114.939232594;3 50 | .EE 51 | .SH SEE ALSO 52 | .ad l 53 | .nh 54 | .BR jitterdebugger (1) 55 | .BR jitterplot (1) 56 | -------------------------------------------------------------------------------- /scripts/genbuiltin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # SPDX-License-Identifier: MIT 3 | 4 | echo "#include \"jitterdebugger.h\"" 5 | echo 6 | 7 | for i in $* 8 | do 9 | echo "extern struct jd_plugin_desc __jd_builtin_$i;" 10 | done 11 | 12 | echo 13 | echo "struct jd_plugin_desc *__jd_builtin[] = {" 14 | 15 | for i in $* 16 | do 17 | echo " &__jd_builtin_$i," 18 | done 19 | 20 | echo " NULL" 21 | echo "};" 22 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # SPDX-License-Identifier: MIT 3 | 4 | OLD_VERSION=${1:-} 5 | NEW_VERSION=${2:-} 6 | 7 | usage() { 8 | echo "$0: OLD_VERSION NEW_VERSION" 9 | echo "" 10 | echo "example:" 11 | echo " $0 0.1 0.2" 12 | } 13 | 14 | if [ -z "$OLD_VERSION" ] || [ -z "$NEW_VERSION" ] ; then 15 | usage 16 | exit 1 17 | fi 18 | 19 | echo "$NEW_VERSION" > newchangelog 20 | git shortlog "$OLD_VERSION".. >> newchangelog 21 | cat CHANGELOG.md >> newchangelog 22 | 23 | emacs newchangelog --eval "(text-mode)" 24 | 25 | echo -n "All fine, ready to release? [y/N]" 26 | read a 27 | a=$(echo "$a" | tr '[:upper:]' '[:lower:]') 28 | if [ "$a" != "y" ]; then 29 | echo "no not happy, let's stop doing the release" 30 | exit 1 31 | fi 32 | 33 | mv newchangelog CHANGELOG.md 34 | sed -i "s,\(__version__ =\).*,\1 \'$NEW_VERSION\'," jitterplot 35 | sed -i "s,\(#define JD_VERSION\).*,\1 \"$NEW_VERSION\"," jitterdebugger.h 36 | 37 | git add CHANGELOG.md jitterplot jitterdebugger.h 38 | 39 | git commit -m "Release $NEW_VERSION" 40 | git tag -s -m "Release $NEW_VERSION" "$NEW_VERSION" 41 | git push --follow-tags 42 | --------------------------------------------------------------------------------