├── README.md ├── github_detail_stat.txt ├── github_stat.py ├── github_stat.txt ├── github_top_stars.py ├── github_top_stars.txt └── github_visual_stat.numbers /README.md: -------------------------------------------------------------------------------- 1 | # github_stat 2 | 3 | ## Plan 4 | + 统计数据 5 | + 定时更新 6 | + 存储 7 | + 可视化报表 8 | + 网站托管 9 | -------------------------------------------------------------------------------- /github_detail_stat.txt: -------------------------------------------------------------------------------- 1 | 2000-2010 15 2 | 2011-2020 30 3 | 2021-2030 27 4 | 2031-2040 29 5 | 2041-2050 32 6 | 2051-2060 29 7 | 2061-2070 30 8 | 2071-2080 17 9 | 2081-2090 23 10 | 2091-2100 31 11 | 2101-2110 21 12 | 2111-2120 12 13 | 2121-2130 34 14 | 2131-2140 28 15 | 2141-2150 26 16 | 2151-2160 21 17 | 2161-2170 31 18 | 2171-2180 21 19 | 2181-2190 28 20 | 2191-2200 23 21 | 2201-2210 17 22 | 2211-2220 20 23 | 2221-2230 19 24 | 2231-2240 22 25 | 2241-2250 17 26 | 2251-2260 18 27 | 2261-2270 24 28 | 2271-2280 17 29 | 2281-2290 22 30 | 2291-2300 25 31 | 2301-2310 29 32 | 2311-2320 22 33 | 2321-2330 18 34 | 2331-2340 17 35 | 2341-2350 21 36 | 2351-2360 18 37 | 2361-2370 23 38 | 2371-2380 27 39 | 2381-2390 22 40 | 2391-2400 29 41 | 2401-2410 18 42 | 2411-2420 20 43 | 2421-2430 15 44 | 2431-2440 20 45 | 2441-2450 22 46 | 2451-2460 10 47 | 2461-2470 15 48 | 2471-2480 13 49 | 2481-2490 20 50 | 2491-2500 19 51 | 2501-2510 13 52 | 2511-2520 13 53 | 2521-2530 19 54 | 2531-2540 17 55 | 2541-2550 10 56 | 2551-2560 18 57 | 2561-2570 15 58 | 2571-2580 21 59 | 2581-2590 20 60 | 2591-2600 19 61 | 2601-2610 17 62 | 2611-2620 15 63 | 2621-2630 11 64 | 2631-2640 20 65 | 2641-2650 14 66 | 2651-2660 16 67 | 2661-2670 18 68 | 2671-2680 14 69 | 2681-2690 19 70 | 2691-2700 14 71 | 2701-2710 10 72 | 2711-2720 17 73 | 2721-2730 13 74 | 2731-2740 17 75 | 2741-2750 18 76 | 2751-2760 7 77 | 2761-2770 10 78 | 2771-2780 17 79 | 2781-2790 15 80 | 2791-2800 17 81 | 2801-2810 16 82 | 2811-2820 5 83 | 2821-2830 16 84 | 2831-2840 14 85 | 2841-2850 4 86 | 2851-2860 12 87 | 2861-2870 14 88 | 2871-2880 12 89 | 2881-2890 16 90 | 2891-2900 14 91 | 2901-2910 15 92 | 2911-2920 19 93 | 2921-2930 15 94 | 2931-2940 9 95 | 2941-2950 12 96 | 2951-2960 16 97 | 2961-2970 18 98 | 2971-2980 12 99 | 2981-2990 14 100 | 2991-3000 6 101 | 3001-3010 13 102 | 3011-3020 10 103 | 3021-3030 13 104 | 3031-3040 9 105 | 3041-3050 7 106 | 3051-3060 16 107 | 3061-3070 9 108 | 3071-3080 14 109 | 3081-3090 9 110 | 3091-3100 7 111 | 3101-3110 9 112 | 3111-3120 13 113 | 3121-3130 15 114 | 3131-3140 8 115 | 3141-3150 16 116 | 3151-3160 14 117 | 3161-3170 11 118 | 3171-3180 13 119 | 3181-3190 5 120 | 3191-3200 9 121 | 3201-3300 97 122 | 3301-3400 100 123 | 3401-3500 93 124 | 3501-3600 68 125 | 3601-3700 89 126 | 3701-3800 90 127 | 3801-3900 64 128 | 3901-4000 70 129 | 4001-4100 74 130 | 4101-4200 65 131 | 4201-4300 62 132 | 4301-4400 55 133 | 4401-4500 59 134 | 4501-4600 52 135 | 4601-4700 49 136 | 4701-4800 40 137 | 4801-4900 41 138 | 4901-5000 43 139 | 5001-5100 43 140 | 5101-5200 31 141 | 5201-5300 31 142 | 5301-5400 26 143 | 5401-5500 24 144 | 5501-5600 31 145 | 5601-5700 28 146 | 5701-5800 36 147 | 5801-5900 24 148 | 5901-6000 25 149 | 6001-6100 38 150 | 6101-6200 32 151 | 6201-6300 18 152 | 6301-6400 23 153 | 6401-6500 22 154 | 6501-6600 18 155 | 6601-6700 16 156 | 6701-6800 17 157 | 6801-6900 16 158 | 6901-7000 17 159 | 7001-7100 17 160 | 7101-7200 17 161 | 7201-7300 19 162 | 7301-7400 21 163 | 7401-7500 17 164 | 7501-7600 21 165 | 7601-7700 10 166 | 7701-7800 18 167 | 7801-7900 8 168 | 7901-8000 18 169 | 8001-8100 16 170 | 8101-8200 13 171 | 8201-8300 16 172 | 8301-8400 10 173 | 8401-8500 14 174 | 8501-8600 12 175 | 8601-8700 18 176 | 8701-8800 6 177 | 8801-8900 10 178 | 8901-9000 12 179 | 9001-10000 97 180 | 10001-11000 48 181 | 11001-12000 51 182 | 12001-13000 40 183 | 13001-14000 32 184 | 14001-15000 33 185 | 15001-16000 22 186 | 16001-17000 20 187 | 17001-18000 15 188 | 18001-19000 12 189 | 19001-20000 10 190 | 20001-21000 12 191 | 21001-22000 9 192 | 22001-23000 10 193 | 23001-24000 3 194 | 24001-25000 5 195 | 25001-26000 7 196 | 26001-27000 5 197 | 27001-28000 2 198 | 28001-29000 5 199 | 29001-30000 3 200 | 30001-31000 5 201 | 31001-32000 4 202 | 32001-33000 2 203 | 33001-34000 1 204 | 34001-35000 1 205 | 35001-36000 1 206 | 36001-37000 3 207 | 37001-38000 2 208 | 38001-39000 0 209 | 39001-40000 0 210 | 40001-41000 2 211 | 41001-42000 0 212 | 42001-43000 0 213 | 43001-44000 3 214 | 44001-45000 0 215 | 45001-46000 1 216 | 46001-47000 0 217 | 47001-48000 2 218 | 48001-49000 0 219 | 49001-50000 2 220 | 50001-51000 1 221 | 51001-52000 1 222 | 52001-53000 0 223 | 53001-54000 1 224 | 54001-55000 1 225 | 55001-56000 1 226 | 56001-57000 0 227 | 57001-58000 0 228 | 58001-59000 0 229 | 59001-60000 0 230 | 60001-61000 0 231 | 61001-62000 1 232 | 62001-63000 1 233 | 63001-64000 0 234 | 64001-65000 0 235 | 65001-66000 0 236 | 66001-67000 0 237 | 67001-68000 0 238 | 68001-69000 0 239 | 69001-70000 0 240 | 70001-71000 0 241 | 71001-72000 0 242 | 72001-73000 0 243 | 73001-74000 0 244 | 74001-75000 0 245 | 75001-76000 0 246 | 76001-77000 0 247 | 77001-78000 0 248 | 78001-79000 0 249 | 79001-80000 0 250 | 80001-81000 1 251 | 81001-82000 0 252 | 82001-83000 0 253 | 83001-84000 0 254 | 84001-85000 0 255 | 85001-86000 0 256 | 86001-87000 0 257 | 87001-88000 0 258 | 88001-89000 0 259 | 89001-90000 0 260 | 90001-91000 0 261 | 91001-92000 0 262 | 92001-93000 0 263 | 93001-94000 0 264 | 94001-95000 0 265 | 95001-96000 0 266 | 96001-97000 0 267 | 97001-98000 0 268 | 98001-99000 0 269 | 99001-100000 0 270 | 100001-101000 0 271 | 101001-102000 0 272 | 102001-103000 0 273 | 103001-104000 0 274 | 104001-105000 0 275 | 105001-106000 0 276 | 106001-107000 0 277 | 107001-108000 1 278 | 108001-109000 0 279 | 109001-110000 0 280 | 110001-111000 0 281 | 111001-112000 0 282 | 112001-113000 0 283 | 113001-114000 0 284 | 114001-115000 0 285 | 115001-116000 0 286 | 116001-117000 0 287 | 117001-118000 0 288 | 118001-119000 0 289 | 119001-120000 0 290 | 120001-121000 0 291 | 121001-122000 0 292 | 122001-123000 0 293 | 123001-124000 0 294 | 124001-125000 0 295 | 125001-126000 0 296 | 126001-127000 0 297 | 127001-128000 0 298 | 128001-129000 0 299 | 129001-130000 0 300 | 130001-131000 0 301 | 131001-132000 0 302 | 132001-133000 0 303 | 133001-134000 0 304 | 134001-135000 0 305 | 135001-136000 0 306 | 136001-137000 0 307 | 137001-138000 0 308 | 138001-139000 0 309 | 139001-140000 0 310 | 140001-141000 0 311 | 141001-142000 0 312 | 142001-143000 0 313 | 143001-144000 0 314 | 144001-145000 0 315 | 145001-146000 0 316 | 146001-147000 0 317 | 147001-148000 0 318 | 148001-149000 0 319 | 149001-150000 0 320 | 150001-151000 0 321 | 151001-152000 0 322 | 152001-153000 0 323 | 153001-154000 0 324 | 154001-155000 0 325 | 155001-156000 0 326 | 156001-157000 0 327 | 157001-158000 0 328 | 158001-159000 0 329 | 159001-160000 0 330 | 160001-161000 0 331 | 161001-162000 0 332 | 162001-163000 0 333 | 163001-164000 0 334 | 164001-165000 0 335 | 165001-166000 0 336 | 166001-167000 0 337 | 167001-168000 0 338 | 168001-169000 0 339 | 169001-170000 0 340 | 170001-171000 0 341 | 171001-172000 0 342 | 172001-173000 0 343 | 173001-174000 0 344 | 174001-175000 0 345 | 175001-176000 0 346 | 176001-177000 0 347 | 177001-178000 0 348 | 178001-179000 0 349 | 179001-180000 0 350 | 180001-181000 0 351 | 181001-182000 0 352 | 182001-183000 0 353 | 183001-184000 0 354 | 184001-185000 0 355 | 185001-186000 0 356 | 186001-187000 0 357 | 187001-188000 0 358 | 188001-189000 0 359 | 189001-190000 0 360 | 190001-191000 0 361 | 191001-192000 0 362 | 192001-193000 0 363 | 193001-194000 0 364 | 194001-195000 0 365 | 195001-196000 0 366 | 196001-197000 0 367 | 197001-198000 0 368 | 198001-199000 0 369 | 199001-200000 0 370 | 200001-201000 0 371 | 201001-202000 0 372 | 202001-203000 0 373 | 203001-204000 0 374 | 204001-205000 0 375 | 205001-206000 0 376 | 206001-207000 0 377 | 207001-208000 0 378 | 208001-209000 0 379 | 209001-210000 0 380 | 210001-211000 0 381 | 211001-212000 0 382 | 212001-213000 0 383 | 213001-214000 0 384 | 214001-215000 0 385 | 215001-216000 0 386 | 216001-217000 0 387 | 217001-218000 0 388 | 218001-219000 0 389 | 219001-220000 0 390 | 220001-221000 0 391 | 221001-222000 0 392 | 222001-223000 0 393 | 223001-224000 0 394 | 224001-225000 0 395 | 225001-226000 0 396 | 226001-227000 0 397 | 227001-228000 0 398 | 228001-229000 0 399 | 229001-230000 0 400 | 230001-231000 0 401 | 231001-232000 0 402 | 232001-233000 0 403 | 233001-234000 0 404 | 234001-235000 0 405 | 235001-236000 0 406 | 236001-237000 0 407 | 237001-238000 0 408 | 238001-239000 0 409 | 239001-240000 0 410 | 240001-241000 0 411 | 241001-242000 0 412 | 242001-243000 0 413 | 243001-244000 0 414 | 244001-245000 0 415 | 245001-246000 1 416 | 246001-247000 0 417 | 247001-248000 0 418 | -------------------------------------------------------------------------------- /github_stat.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import sys 4 | 5 | def getHelp(): 6 | print "read the fucking source code" 7 | 8 | def __exit__(): 9 | switch['h']() 10 | exit() 11 | 12 | switch = { 13 | 'h':getHelp 14 | } 15 | 16 | # you must provide your github username and password to access api 17 | username = sys.argv[1] 18 | password = sys.argv[2] 19 | 20 | #the search api response has max 100 items per_page, max 10 pages, that mean max 1000 result for each query condition, see document 21 | start_time = time.time() 22 | 23 | target = open('github_stat.txt', 'w') 24 | start = 100 25 | limit = 250000 26 | step = 1000 27 | 28 | page = 1 29 | count = 1 30 | while start <= limit: 31 | end = start + step 32 | mod = end % 1000 33 | if mod != 0: 34 | end = end - mod 35 | url = 'https://api.github.com/search/repositories?q=stars:%d..%d&sort=stars&per_page=1&page=%d' % (start, end, page) 36 | print url 37 | public_repos = requests.get(url, auth=(username, password)).json() 38 | total_count = public_repos['total_count'] 39 | print "round %d [%d, %d] %d" % (count, start, end, total_count) 40 | if total_count > 0: 41 | target.write("%d-%d\t%d" % (start, end, total_count)) 42 | target.write("\n") 43 | start = end + 1 44 | if count % 30 == 0: 45 | # api call has limit 30 times/m' 46 | print "sleeping for 61 seconds" 47 | time.sleep(61) 48 | count = count + 1 49 | target.close() 50 | 51 | finish_time = time.time() 52 | cost = (finish_time - start_time) 53 | print "task finished cost %d seconds" % cost -------------------------------------------------------------------------------- /github_stat.txt: -------------------------------------------------------------------------------- 1 | 100-1000 70544 2 | 1001-2000 5402 3 | 2001-3000 1829 4 | 3001-4000 891 5 | 4001-5000 540 6 | 5001-6000 299 7 | 6001-7000 217 8 | 7001-8000 166 9 | 8001-9000 127 10 | 9001-10000 97 11 | 10001-11000 48 12 | 11001-12000 51 13 | 12001-13000 40 14 | 13001-14000 32 15 | 14001-15000 33 16 | 15001-16000 22 17 | 16001-17000 20 18 | 17001-18000 15 19 | 18001-19000 12 20 | 19001-20000 10 21 | 20001-21000 12 22 | 21001-22000 9 23 | 22001-23000 10 24 | 23001-24000 3 25 | 24001-25000 5 26 | 25001-26000 7 27 | 26001-27000 5 28 | 27001-28000 2 29 | 28001-29000 5 30 | 29001-30000 3 31 | 30001-31000 5 32 | 31001-32000 4 33 | 32001-33000 2 34 | 33001-34000 1 35 | 34001-35000 1 36 | 35001-36000 1 37 | 36001-37000 3 38 | 37001-38000 2 39 | 40001-41000 2 40 | 43001-44000 3 41 | 45001-46000 1 42 | 47001-48000 2 43 | 49001-50000 2 44 | 50001-51000 1 45 | 51001-52000 1 46 | 53001-54000 1 47 | 54001-55000 1 48 | 55001-56000 1 49 | 61001-62000 1 50 | 62001-63000 1 51 | 80001-81000 1 52 | 107001-108000 1 53 | 245001-246000 1 54 | -------------------------------------------------------------------------------- /github_top_stars.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import sys 4 | 5 | def getHelp(): 6 | print "read the fucking source code" 7 | 8 | def __exit__(): 9 | switch['h']() 10 | exit() 11 | 12 | switch = { 13 | 'h':getHelp 14 | } 15 | 16 | # you must provide your github username and password to access api 17 | username = sys.argv[1] 18 | password = sys.argv[2] 19 | 20 | start_time = time.time() 21 | 22 | target = open('github_top_stars.txt', 'w') 23 | detail_stat = open('github_detail_stat.txt', 'w') 24 | 25 | start = 2000 26 | limit = 248000 27 | step = 10 28 | 29 | page = 1 30 | count = 1 31 | while start <= limit: 32 | end = start + step 33 | mod = end % step 34 | if mod != 0: 35 | end = end - mod 36 | url = 'https://api.github.com/search/repositories?q=stars:%d..%d&sort=stars&per_page=100&page=%d&order=asc' % (start, end, page) 37 | print url 38 | public_repos = requests.get(url, auth=(username, password)).json() 39 | total_count = public_repos['total_count'] 40 | print "round %d [%d, %d] %d" % (count, start, end, total_count) 41 | if total_count > 0: 42 | detail_stat.write("%d-%d\t%d" % (start, end, total_count)) 43 | detail_stat.write("\n") 44 | if total_count > 0: 45 | repos = public_repos['items'] 46 | for repo in repos: 47 | name = repo['name'] 48 | owner = repo['owner']['login'] 49 | stars = repo['stargazers_count'] 50 | watchers = repo['watchers_count'] 51 | language = repo['language'] 52 | forks_count = repo['forks_count'] 53 | issues_count = repo['open_issues_count'] 54 | target.write("%d-%d\t%s\t%s\t%d\t%d\t%s\t%d\t%d" % (start, end, name, owner, stars, watchers, language, forks_count, issues_count)) 55 | target.write("\n") 56 | start = end + 1 57 | if start >= 3200: 58 | step = 100 59 | if start >= 9000: 60 | step = 1000 61 | if count % 30 == 0: 62 | # api call has limit 30 times/m' 63 | print "sleeping for 61 seconds" 64 | time.sleep(61) 65 | count = count + 1 66 | detail_stat.close() 67 | target.close() 68 | 69 | finish_time = time.time() 70 | cost = (finish_time - start_time) 71 | print "task finished cost %d seconds" % cost -------------------------------------------------------------------------------- /github_visual_stat.numbers: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/longerian/github_stat/13a0066fe26ab2e1e0d4a1b4d6b6face7307ad59/github_visual_stat.numbers --------------------------------------------------------------------------------