├── README.md ├── afl_fork.txt ├── afl_persist.txt ├── afl_shared_memory_persist.txt ├── benchmark.py ├── plot.plt └── test.c /README.md: -------------------------------------------------------------------------------- 1 | # Readme 2 | 3 | This is just a tool to benchmark AFL performance. Simply run benchmark.py with python 3 and you can see the scaling properties of AFL! Change `test.c` to be the test application you want it to be. 4 | 5 | Make sure that `../AFLplusplus` contains an AFL++ checkout, with a valid `make source-only` build 6 | 7 | -------------------------------------------------------------------------------- /afl_fork.txt: -------------------------------------------------------------------------------- 1 | 1 10408.460000 2 | 2 17472.700000 3 | 3 24851.300000 4 | 4 31763.770000 5 | 5 39348.530000 6 | 6 45828.810000 7 | 7 53279.020000 8 | 8 60056.750000 9 | 9 65347.730000 10 | 10 72989.620000 11 | 11 80577.610000 12 | 12 86465.200000 13 | 13 93086.390000 14 | 14 98667.580000 15 | 15 107162.820000 16 | 16 109929.440000 17 | 17 115888.630000 18 | 18 120922.340000 19 | 19 128752.210000 20 | 20 131877.170000 21 | 21 133486.620000 22 | 22 141108.650000 23 | 23 142365.430000 24 | 24 148272.800000 25 | 25 135037.140000 26 | 26 136756.770000 27 | 27 140586.160000 28 | 28 141350.010000 29 | 29 140508.950000 30 | 30 143619.340000 31 | 31 138691.770000 32 | 32 143143.890000 33 | 33 143501.690000 34 | 34 143497.950000 35 | 35 148075.130000 36 | 36 141687.880000 37 | 37 147544.100000 38 | 38 145717.750000 39 | 39 146750.900000 40 | 40 158780.370000 41 | 41 139833.910000 42 | 42 146558.320000 43 | 43 149683.820000 44 | 44 144859.790000 45 | 45 154675.360000 46 | 46 152866.250000 47 | 47 150967.950000 48 | 48 145170.950000 49 | 49 161742.290000 50 | 50 143109.970000 51 | 51 163975.360000 52 | 52 142874.520000 53 | 53 142175.920000 54 | 54 150348.920000 55 | 55 142814.500000 56 | 56 158333.920000 57 | 57 163860.650000 58 | 58 156998.770000 59 | 59 152311.610000 60 | 60 160941.810000 61 | 61 168975.710000 62 | 62 166860.520000 63 | 63 172681.210000 64 | 64 176756.530000 65 | 65 152207.690000 66 | 66 164560.890000 67 | 67 133390.090000 68 | 68 124388.040000 69 | 69 119222.950000 70 | 70 117403.700000 71 | 71 117383.500000 72 | 72 116566.900000 73 | 73 116566.700000 74 | 74 116014.870000 75 | 75 116199.420000 76 | 76 116021.970000 77 | 77 114668.360000 78 | 78 114204.520000 79 | 79 114445.630000 80 | 80 114212.680000 81 | 81 113631.140000 82 | 82 115049.920000 83 | 83 113710.160000 84 | 84 114477.400000 85 | 85 113944.230000 86 | 86 113553.950000 87 | 87 113068.340000 88 | 88 112614.590000 89 | 89 113046.420000 90 | 90 113488.870000 91 | 91 112905.910000 92 | 92 114584.710000 93 | 93 112999.670000 94 | 94 113508.440000 95 | 95 112315.440000 96 | 96 112577.030000 97 | 97 112770.270000 98 | 98 114258.940000 99 | 99 113656.800000 100 | 100 113433.180000 101 | 101 114884.740000 102 | 102 113673.820000 103 | 103 114237.510000 104 | 104 116382.110000 105 | 105 117138.840000 106 | 106 116808.470000 107 | 107 114353.430000 108 | 108 117083.450000 109 | 109 118283.380000 110 | 110 118318.420000 111 | 111 116155.600000 112 | 112 116015.380000 113 | 113 117067.610000 114 | 114 118635.440000 115 | 115 116548.800000 116 | 116 117078.490000 117 | 117 116189.090000 118 | 118 117187.690000 119 | 119 116661.850000 120 | 120 117332.850000 121 | 121 117700.510000 122 | 122 118302.660000 123 | 123 117469.550000 124 | 124 118928.540000 125 | 125 116798.110000 126 | 126 116341.570000 127 | 127 115365.530000 128 | 128 114983.700000 129 | 129 115097.270000 130 | 130 114484.010000 131 | 131 115318.750000 132 | 132 114627.720000 133 | 133 114837.410000 134 | 134 114430.830000 135 | 135 114113.070000 136 | 136 113992.810000 137 | 137 114042.370000 138 | 138 114163.700000 139 | 139 114617.100000 140 | 140 114474.320000 141 | 141 114477.820000 142 | 142 114868.500000 143 | 143 115290.510000 144 | 144 115231.150000 145 | 145 115147.250000 146 | 146 114702.160000 147 | 147 114215.330000 148 | 148 114276.200000 149 | 149 114108.060000 150 | 150 113849.650000 151 | 151 113775.230000 152 | 152 114023.340000 153 | 153 113959.710000 154 | 154 114314.960000 155 | 155 114219.880000 156 | 156 113623.960000 157 | 157 113578.400000 158 | 158 114657.780000 159 | 159 114449.760000 160 | 160 114392.450000 161 | 161 114551.130000 162 | 162 114406.900000 163 | 163 114422.930000 164 | 164 113764.930000 165 | 165 114211.020000 166 | 166 115125.950000 167 | 167 114743.490000 168 | 168 114857.710000 169 | 169 114557.750000 170 | 170 113589.730000 171 | 171 114234.510000 172 | 172 114606.660000 173 | 173 114578.590000 174 | 174 114403.240000 175 | 175 113638.920000 176 | 176 113990.700000 177 | 177 114875.420000 178 | 178 115157.000000 179 | 179 114391.280000 180 | 180 114721.600000 181 | 181 114637.080000 182 | 182 114069.860000 183 | 183 114947.350000 184 | 184 113983.110000 185 | 185 114529.340000 186 | 186 114460.100000 187 | 187 114386.460000 188 | 188 115027.850000 189 | 189 114769.160000 190 | 190 114532.800000 191 | 191 115341.300000 192 | 192 115046.290000 193 | -------------------------------------------------------------------------------- /afl_persist.txt: -------------------------------------------------------------------------------- 1 | 1 25331.720000 2 | 2 46357.680000 3 | 3 66640.010000 4 | 4 86694.310000 5 | 5 102600.310000 6 | 6 125472.090000 7 | 7 144609.920000 8 | 8 163928.530000 9 | 9 180677.850000 10 | 10 198863.320000 11 | 11 220742.730000 12 | 12 225737.440000 13 | 13 246273.050000 14 | 14 268127.560000 15 | 15 282806.660000 16 | 16 298069.920000 17 | 17 305924.590000 18 | 18 314313.900000 19 | 19 325963.340000 20 | 20 335628.640000 21 | 21 337565.580000 22 | 22 353541.970000 23 | 23 362127.760000 24 | 24 369830.330000 25 | 25 315990.870000 26 | 26 287688.650000 27 | 27 264351.270000 28 | 28 256537.290000 29 | 29 245415.930000 30 | 30 238045.650000 31 | 31 234520.750000 32 | 32 228915.890000 33 | 33 225614.650000 34 | 34 224489.840000 35 | 35 224303.540000 36 | 36 224169.290000 37 | 37 220218.620000 38 | 38 217747.750000 39 | 39 218574.690000 40 | 40 220701.720000 41 | 41 218722.760000 42 | 42 217390.670000 43 | 43 219455.460000 44 | 44 218254.920000 45 | 45 216093.590000 46 | 46 214937.710000 47 | 47 213855.210000 48 | 48 217540.470000 49 | 49 214632.020000 50 | 50 207279.150000 51 | 51 203396.670000 52 | 52 203979.070000 53 | 53 209062.610000 54 | 54 214012.440000 55 | 55 200857.060000 56 | 56 201039.750000 57 | 57 206329.770000 58 | 58 199842.480000 59 | 59 201743.890000 60 | 60 192347.660000 61 | 61 198176.610000 62 | 62 211152.430000 63 | 63 200633.880000 64 | 64 201993.070000 65 | 65 203600.610000 66 | 66 206673.230000 67 | 67 206238.410000 68 | 68 208941.210000 69 | 69 203891.790000 70 | 70 203576.230000 71 | 71 205205.060000 72 | 72 214368.700000 73 | 73 207383.450000 74 | 74 208886.990000 75 | 75 210033.360000 76 | 76 218546.610000 77 | 77 207582.870000 78 | 78 203541.710000 79 | 79 208376.430000 80 | 80 219046.220000 81 | 81 223649.260000 82 | 82 225065.970000 83 | 83 217496.780000 84 | 84 211610.960000 85 | 85 200536.000000 86 | 86 204321.720000 87 | 87 188402.130000 88 | 88 185748.040000 89 | 89 190910.600000 90 | 90 191110.510000 91 | 91 181492.530000 92 | 92 172821.880000 93 | 93 168023.240000 94 | 94 166304.520000 95 | 95 166787.860000 96 | 96 163181.960000 97 | 97 163636.500000 98 | 98 164984.600000 99 | 99 165069.510000 100 | 100 165436.490000 101 | 101 165131.620000 102 | 102 166639.910000 103 | 103 166522.250000 104 | 104 166070.890000 105 | 105 166349.720000 106 | 106 167784.940000 107 | 107 166680.620000 108 | 108 167306.610000 109 | 109 167388.590000 110 | 110 168000.620000 111 | 111 167886.660000 112 | 112 169246.770000 113 | 113 168519.720000 114 | 114 169643.900000 115 | 115 169810.490000 116 | 116 170123.920000 117 | 117 170419.900000 118 | 118 171261.140000 119 | 119 169988.910000 120 | 120 170455.330000 121 | 121 171294.090000 122 | 122 171297.880000 123 | 123 171522.190000 124 | 124 171371.580000 125 | 125 170144.580000 126 | 126 170070.190000 127 | 127 171788.670000 128 | 128 170887.350000 129 | 129 169325.220000 130 | 130 171918.420000 131 | 131 172179.430000 132 | 132 170057.430000 133 | 133 170418.030000 134 | 134 172432.050000 135 | 135 174393.730000 136 | 136 171720.780000 137 | 137 173380.180000 138 | 138 172339.190000 139 | 139 175273.210000 140 | 140 174402.620000 141 | 141 171829.010000 142 | 142 173040.300000 143 | 143 171680.530000 144 | 144 175290.350000 145 | 145 172874.470000 146 | 146 173313.310000 147 | 147 173086.840000 148 | 148 173028.200000 149 | 149 174165.190000 150 | 150 173749.340000 151 | 151 172940.500000 152 | 152 172997.330000 153 | 153 176370.060000 154 | 154 175954.550000 155 | 155 175881.520000 156 | 156 178980.150000 157 | 157 173743.650000 158 | 158 181818.240000 159 | 159 180106.040000 160 | 160 176600.330000 161 | 161 175794.080000 162 | 162 179417.180000 163 | 163 178938.270000 164 | 164 176432.190000 165 | 165 175898.620000 166 | 166 176906.100000 167 | 167 178965.750000 168 | 168 173877.900000 169 | 169 175680.250000 170 | 170 174334.510000 171 | 171 173513.010000 172 | 172 171723.990000 173 | 173 172689.620000 174 | 174 172797.240000 175 | 175 173036.390000 176 | 176 171712.230000 177 | 177 173399.130000 178 | 178 172906.460000 179 | 179 172445.840000 180 | 180 172367.890000 181 | 181 173557.900000 182 | 182 172343.030000 183 | 183 171838.870000 184 | 184 172140.620000 185 | 185 172286.260000 186 | 186 172798.550000 187 | 187 171340.460000 188 | 188 172411.770000 189 | 189 172441.840000 190 | 190 170438.740000 191 | 191 172828.110000 192 | 192 168763.130000 193 | -------------------------------------------------------------------------------- /afl_shared_memory_persist.txt: -------------------------------------------------------------------------------- 1 | 1 29432.060000 2 | 2 71751.690000 3 | 3 103979.280000 4 | 4 136438.030000 5 | 5 168184.870000 6 | 6 201185.410000 7 | 7 234454.450000 8 | 8 254447.280000 9 | 9 294196.500000 10 | 10 327294.620000 11 | 11 366875.080000 12 | 12 380474.850000 13 | 13 432752.540000 14 | 14 465820.150000 15 | 15 491746.940000 16 | 16 509643.720000 17 | 17 535300.140000 18 | 18 554458.720000 19 | 19 604279.650000 20 | 20 607399.210000 21 | 21 627842.910000 22 | 22 646076.140000 23 | 23 687476.470000 24 | 24 705622.290000 25 | 25 729475.360000 26 | 26 768306.830000 27 | 27 789188.340000 28 | 28 815242.990000 29 | 29 842514.250000 30 | 30 853190.580000 31 | 31 895459.640000 32 | 32 920927.580000 33 | 33 940342.760000 34 | 34 967863.190000 35 | 35 983727.270000 36 | 36 1005553.870000 37 | 37 1037818.070000 38 | 38 1034575.640000 39 | 39 1085850.110000 40 | 40 1082015.040000 41 | 41 1107557.970000 42 | 42 1122818.170000 43 | 43 1139141.240000 44 | 44 1150478.840000 45 | 45 1149936.360000 46 | 46 1165467.620000 47 | 47 1180832.110000 48 | 48 1189344.280000 49 | 49 1193406.730000 50 | 50 1167518.110000 51 | 51 1172598.850000 52 | 52 1162635.850000 53 | 53 1168572.090000 54 | 54 1164361.620000 55 | 55 1146216.240000 56 | 56 1156306.580000 57 | 57 1165168.420000 58 | 58 1170895.040000 59 | 59 1170774.340000 60 | 60 1195431.900000 61 | 61 1190418.640000 62 | 62 1200341.810000 63 | 63 1205730.140000 64 | 64 1207282.250000 65 | 65 1217745.700000 66 | 66 1214247.970000 67 | 67 1224166.420000 68 | 68 1222254.230000 69 | 69 1222600.340000 70 | 70 1230397.150000 71 | 71 1222239.090000 72 | 72 1230876.180000 73 | 73 1206478.160000 74 | 74 1190942.700000 75 | 75 1184883.580000 76 | 76 1183141.090000 77 | 77 1197669.530000 78 | 78 1178086.360000 79 | 79 1172599.930000 80 | 80 1194803.020000 81 | 81 1187374.360000 82 | 82 1198215.830000 83 | 83 1201184.010000 84 | 84 1212699.270000 85 | 85 1211519.290000 86 | 86 1218557.760000 87 | 87 1231245.930000 88 | 88 1232424.650000 89 | 89 1234766.390000 90 | 90 1239037.930000 91 | 91 1247666.670000 92 | 92 1238184.240000 93 | 93 1242816.640000 94 | 94 1240292.620000 95 | 95 1241197.480000 96 | 96 1247165.910000 97 | 97 1249131.550000 98 | 98 1253682.530000 99 | 99 1256372.160000 100 | 100 1253103.470000 101 | 101 1255273.520000 102 | 102 1254202.590000 103 | 103 1250325.750000 104 | 104 1254481.390000 105 | 105 1256781.760000 106 | 106 1254361.830000 107 | 107 1259211.690000 108 | 108 1255141.230000 109 | 109 1258602.360000 110 | 110 1255344.030000 111 | 111 1251468.320000 112 | 112 1258813.330000 113 | 113 1254920.340000 114 | 114 1258432.600000 115 | 115 1267672.150000 116 | 116 1259672.770000 117 | 117 1262594.680000 118 | 118 1257110.600000 119 | 119 1246065.110000 120 | 120 1244731.690000 121 | 121 1261013.380000 122 | 122 1256622.000000 123 | 123 1250300.480000 124 | 124 1253325.680000 125 | 125 1252184.230000 126 | 126 1261195.990000 127 | 127 1253257.900000 128 | 128 1250183.790000 129 | 129 1262258.850000 130 | 130 1257276.790000 131 | 131 1242113.950000 132 | 132 1254096.380000 133 | 133 1251636.550000 134 | 134 1259803.910000 135 | 135 1257138.230000 136 | 136 1253319.690000 137 | 137 1253568.920000 138 | 138 1244505.300000 139 | 139 1254981.370000 140 | 140 1254391.470000 141 | 141 1256986.470000 142 | 142 1254943.250000 143 | 143 1263730.000000 144 | 144 1249110.760000 145 | 145 1256435.560000 146 | 146 1251542.600000 147 | 147 1251623.520000 148 | 148 1254580.110000 149 | 149 1249737.120000 150 | 150 1249978.140000 151 | 151 1249189.270000 152 | 152 1244742.570000 153 | 153 1251117.700000 154 | 154 1251148.350000 155 | 155 1256114.080000 156 | 156 1250656.030000 157 | 157 1255560.450000 158 | 158 1250687.210000 159 | 159 1259361.400000 160 | 160 1247634.510000 161 | 161 1253774.930000 162 | 162 1254570.160000 163 | 163 1250798.700000 164 | 164 1248174.320000 165 | 165 1253381.110000 166 | 166 1254469.240000 167 | 167 1254153.190000 168 | 168 1253497.810000 169 | 169 1250120.460000 170 | 170 1250264.020000 171 | 171 1255206.830000 172 | 172 1254258.230000 173 | 173 1249192.300000 174 | 174 1246557.620000 175 | 175 1243309.480000 176 | 176 1252894.930000 177 | 177 1254553.030000 178 | 178 1254233.760000 179 | 179 1258352.100000 180 | 180 1259386.860000 181 | 181 1256245.070000 182 | 182 1261240.840000 183 | 183 1261584.200000 184 | 184 1260829.220000 185 | 185 1257199.930000 186 | 186 1256247.630000 187 | 187 1257883.160000 188 | 188 1256866.150000 189 | 189 1257244.090000 190 | 190 1253752.620000 191 | 191 1263931.800000 192 | 192 1254247.630000 193 | -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os, shutil, multiprocessing, time, subprocess 4 | 5 | # Maximum number of threads to fuzz with AFL in parallel during benchmark 6 | MAX_THREADS = 192 7 | 8 | def fuzz_worker(thr_id): 9 | os.sched_setaffinity(0, [thr_id]) 10 | sp=subprocess.Popen([f"../AFLplusplus/afl-fuzz -i inputs/ -o outputs/ -d -S {thr_id} -- ./aflfuzzbencher @@"], 11 | stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, shell=True) 12 | while sp.poll() == None: 13 | print(sp.stdout.read()) 14 | print(sp.stderr.read()) 15 | time.sleep(0.1) 16 | 17 | def benchmark_afl(num_threads): 18 | assert num_threads > 0 19 | 20 | # Kill anything that may already have been running 21 | os.system("killall -9 aflfuzzbencher 2> /dev/null") 22 | 23 | # Kill active shared memory, AFL will run out and crash eventually 24 | # due to shmget() failures 25 | assert os.system("ipcrm -a") == 0 26 | 27 | # Set the environment variables we want so AFL behaves 28 | os.environ["AFL_NO_AFFINITY"] = "123" 29 | 30 | # Remove our input and output directories 31 | if os.path.exists("inputs"): 32 | shutil.rmtree("inputs") 33 | if os.path.exists("outputs"): 34 | shutil.rmtree("outputs") 35 | 36 | # Make the input and output directories 37 | os.mkdir("inputs") 38 | os.mkdir("outputs") 39 | 40 | # Create a test input 41 | with open("inputs/test_input", "wb") as fd: 42 | fd.write(b"A"*128) 43 | 44 | # Spawn threads 45 | threads = [] 46 | for thr_id in range(num_threads): 47 | thread = multiprocessing.Process(target=fuzz_worker, args=[thr_id]) 48 | thread.start() 49 | threads.append(thread) 50 | 51 | # Wait a bit for the benchmark 52 | time.sleep(5.0) 53 | 54 | # Kill all workers 55 | for thread in threads: 56 | thread.terminate() 57 | thread.join() 58 | 59 | # Kill the process, as `terminate` orphans processes, I'm just lazy this 60 | # could be done better 61 | os.system("killall -9 aflfuzzbencher 2> /dev/null") 62 | 63 | # Go through AFL status messages, pick the most recently logged line and 64 | # sum up the per-thread cases per second to get a total cases per second 65 | total_per_second = 0 66 | for thr_id in range(num_threads): 67 | with open(f"outputs/{thr_id}/plot_data", "r") as fd: 68 | # Get the last status line for this thread 69 | contents = fd.read().splitlines() 70 | last_status = contents[-1] 71 | iters_per_sec = float(last_status.split(",")[-1].strip()) 72 | total_per_second += iters_per_sec 73 | 74 | print(f"{num_threads:6} {total_per_second:16.6f}") 75 | 76 | # Build our test program 77 | assert os.system("../AFLplusplus/afl-clang-fast -O2 test.c -o aflfuzzbencher > /dev/null") == 0 78 | 79 | # Benchmark AFL performance for each number of threads 80 | for num_threads in range(1, MAX_THREADS + 1): 81 | while True: 82 | try: 83 | benchmark_afl(num_threads) 84 | break 85 | except: 86 | continue 87 | 88 | -------------------------------------------------------------------------------- /plot.plt: -------------------------------------------------------------------------------- 1 | set term wxt persist size 1440,900 2 | set multiplot layout 1,2 3 | set title "AFL++ scaling in fork and persist modes" 4 | set xlabel "Number of cores" 5 | set ylabel "Fuzz cases per second (total)" 6 | set grid xtics ytics mxtics mytics 7 | set logscale y 8 | plot "afl_fork.txt" u 1:2 w l t "Fork Mode", "afl_persist.txt" u 1:2 w l t "Persist Mode", "afl_shared_memory_persist.txt" u 1:2 w l t "AFL Shared Memory Persist Mode" 9 | 10 | set ylabel "Fuzz cases per second per core" 11 | plot "afl_fork.txt" u 1:($2 / $1) w l t "Fork Mode", "afl_persist.txt" u 1:($2 / $1) w l t "Persist Mode", "afl_shared_memory_persist.txt" u 1:($2 / $1) w l t "AFL Shared Memory Persist Mode" 12 | 13 | -------------------------------------------------------------------------------- /test.c: -------------------------------------------------------------------------------- 1 | #if 1 2 | // WIP: shared memory test case handover, requires current afl++ dev branch 3 | __AFL_FUZZ_INIT(); 4 | #endif 5 | 6 | int 7 | main(void) { 8 | #if 1 9 | #ifdef __AFL_HAVE_MANUAL_CONTROL 10 | __AFL_INIT(); 11 | #endif 12 | while (__AFL_LOOP(100000)) { 13 | } 14 | #endif 15 | 16 | return 0; 17 | } 18 | --------------------------------------------------------------------------------