├── README.md ├── c++ ├── README.md ├── hello ├── hello.cpp ├── log4cplus-2.0.8.tar.bz2 └── test.log ├── generate_log.py ├── golang ├── README.md ├── go.mod ├── go.sum ├── golang ├── hello └── hello.go ├── java └── helloLog │ ├── README.md │ ├── pom.xml │ ├── src │ └── main │ │ ├── java │ │ └── com │ │ │ └── yuxiaoba │ │ │ └── hello │ │ │ └── Hello.java │ │ └── resources │ │ └── log4j2.xml │ └── target │ ├── classes │ ├── com │ │ └── yuxiaoba │ │ │ └── hello │ │ │ ├── Hello.class │ │ │ └── Task.class │ └── log4j2.xml │ ├── helloLog-1.0.jar │ ├── helloLog-jar-with-dependencies.jar │ ├── maven-archiver │ └── pom.properties │ └── maven-status │ └── maven-compiler-plugin │ ├── compile │ └── default-compile │ │ ├── createdFiles.lst │ │ └── inputFiles.lst │ └── testCompile │ └── default-testCompile │ └── inputFiles.lst ├── logReducer.c ├── logReducer.py ├── python ├── README.md ├── __pycache__ │ └── nb_log_config.cpython-36.pyc ├── hello.py └── nb_log_config.py ├── run_loglength.sh ├── run_lognumber.sh └── run_userspace.sh /README.md: -------------------------------------------------------------------------------- 1 | # LogReducer 2 | 3 | eBPF-based Log Filter and log benmark of paper `LogReducer: Identify and Reduce Log Hotspots in Kernel on the Fly` submit to ICSE 2023 4 | 5 | ## Require 6 | ``` 7 | python 3.6 8 | 9 | bcc 0.24 10 | 11 | linux kerenl 5.4 12 | ``` 13 | 14 | ## Running 15 | 16 | ### Log lenth evaluation 17 | ``` 18 | bash run_loglength.sh 19 | ``` 20 | 21 | ### Log number evaluation 22 | ``` 23 | bash run_lognumber.sh 24 | ``` 25 | 26 | ### Userspace evaluation 27 | ``` 28 | bash run_userspace.sh 29 | ``` 30 | 31 | 32 | -------------------------------------------------------------------------------- /c++/README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | tar xjvf log4cplus-2.0.8.tar.bz2 3 | 4 | cd log4cplus-2.0.8 5 | 6 | ./configure --prefix=/usr/local 7 | 8 | make 9 | 10 | make install 11 | 12 | vim /etc/ld.so.conf 13 | 14 | /usr/local/lib 15 | 16 | ldconfig 17 | 18 | 19 | g++ hello.cpp -o hello -llog4cplus -std=gnu++17 20 | 21 | ./hello -lognumber 1000 22 | 23 | ``` 24 | -------------------------------------------------------------------------------- /c++/hello: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/c++/hello -------------------------------------------------------------------------------- /c++/hello.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "log4cplus/logger.h" 4 | #include "log4cplus/fileappender.h" 5 | #include "log4cplus/loglevel.h" 6 | #include "log4cplus/loggingmacros.h" 7 | #include "log4cplus/initializer.h" 8 | #include 9 | 10 | using namespace std; 11 | using namespace log4cplus; 12 | using namespace chrono; 13 | 14 | int main(int argc, char *argv[]) 15 | { 16 | argparse::ArgumentParser program("hello"); 17 | program.add_argument("-lognumber") 18 | .help("log number") 19 | .default_value(100) 20 | .scan<'i', int>(); 21 | program.add_argument("-length") 22 | .default_value(std::string{"log20"}) // might otherwise be type const char* leading to an error when trying program.get 23 | .help("log length"); 24 | 25 | try { 26 | program.parse_args(argc, argv); 27 | } 28 | catch (const std::runtime_error& err) { 29 | std::cerr << err.what() << std::endl; 30 | std::cerr << program; 31 | std::exit(1); 32 | } 33 | 34 | int log_number = program.get("-lognumber"); 35 | auto log_length = program.get("-length"); 36 | 37 | map dict{{"log20", "This is log hotspot!"}, {"log50", "This is hotspots.arXiEDHcPlfrcKwJxWjrjAYLvWkgzD"}, {"log100", "This is hotspots.bJzsxyrMpzYGcfgKquEUPTHQtiFzeEHJxTEmdAtQQHhpfjtrKClFLsfjacHuTDkgzWNGmSOaZnzLxoUp"}, 38 | {"log200", "This is hotspots.aRdUTDyLjPkZknRfLlCSxcEQLUKQTnbNCaMEnKLhVFmdNSQjInhGjrUMqMHPyScbMKBEPAIkxLukJUfImotUtzxIbhjMOHUpLqbbTYEXsOToClWMkOydioyhXzvcLiXtAMMssTROiXDwLLwInfaZyXptcxnRjPafxftKWtBBPQqodLPdvmEB"}, 39 | {"log500", "This is hotspots.kdeHKrCcqKeCxYqRbXDENnFBuJemiQMuNvresaULGgozUxAugbywwYJEaqQbYGWcISrRlEnXdpoNZNwvdnzxoaXLMWHLxghZixRDiZbVTKynrgQjJisDcaGwanDipFZGRLhJkXqnDlPlcaHBnYYwRQcpnbBWcsRwaOWyVjVORurodxkMGXvUPNoKJKcUrhiDuTpVCQWpmbJYkNYiVyiPYlllLCOMPIJGgLiUCjOSnmFcXtmnUXyKBvDgKTCVRLYZzkTPhtKPApvFMuVVuEcuqPcQLLbHHvumugBgmgOiguyYbdMsfwQLguqOBeakiSjhkSKBFTSjuoMViXPeMCQllfTdvPVJcRpjtACHfxJkhHYrLUyvCTKBvXzNHxikUFbngYdERraSrrvlGRciDalgxLIVTrbdEnORNIomHRtbFfPnJHFWLfAjFxWpKhLHZewgavRdVmbfSIzfHMfFxtIlEOpOyxGyAKnX"}, 40 | {"log1000", "This is hotspots.htTGLdjysnJBmraMbjsSytuKfxduwsdQJyzsIneGUNyiIYLwXBHgWupqOXLfUXOnpYbkJriscvFxGMRXbQsuipuBTsiMjpXKpbNcEoFWDDiMGeORcLWacLLOdcMaEVMcPDTpwNeEvUCgfcAARWsVBogiRstmyNCFwLfyvRdfdESZeSdjVzXDVGIKQztMylXzZfgCSzKIjiVtYwjHNTIwFDvhpyTgjKLfAFOHtJaVKEmzxDxrQJPmENyfBpQTDgFtwJYSwGPJkXBxoWKyhlVgavTnAOIgxSMsupviMzCttszFjubiNpFuierLCDVhsLudMgFbyaRFuapcYfGrvMVrIKWvoDEBHAdgLcGWVxMduuMkhHtAUtnCKHcYFkvXTCbixbGKpZBCuOkpcTCVTsGFZahRgzCwPHmKqaYougbHiOiDDSHatwIAKQDqhKNJEuydQfwcBoIeMotVHIpNzhcIndmUaNiHfYltuJuJNraDdUysIKwBNclJFLUbajVSgVheJbFdPjlTlnafBFAqeKQPvQvgjTLISfYKVmZcfReYOnRRlneMtBuVxCMapyYPZShEWkMsdTIYxLgjEawGXwXvSvUkAvclpOPKSTxrOIfHSMLcmRlfzXlbIPHpmncnfhGWIRfcVsZazehLGuYfVEfjlzuluPxlFKxeYJjmtEzYTVGOAjuMIpUYEwNeFNOeqoyHkJhrzNtsyiYBNHRmlMCZubXmwBRXingWgbYMRyabuABkGUpWQBQLdEtYHyRnCocXgirNamwpkBmdfjuzPzbvxtzdwHwXZrffSSeKPvkRkwLvofZmXBwABRhimRkAiMxFZRKflFdVKDcBjpCBvMZaJVnqyuqxCvfVRWzLijTQAjPIbBBRUTFKeRabpkeUddhfONWuRAhxbghoIKHIYhyvjBxyywIQsvUveuJvZzprbTaBKQJpRBiezwfTAtRDpmitJqPYxWhBhAuvvQjbbyjf"}}; 41 | 42 | log4cplus::Initializer initializer; 43 | SharedAppenderPtr append_1(new FileAppender("c++.log")); 44 | append_1->setName(LOG4CPLUS_TEXT("append_1")); 45 | log4cplus::tstring pattern = LOG4CPLUS_TEXT("%D{%Y/%m/%d %H:%M:%S,%Q} %-5p - [%l] %m %n"); 46 | append_1->setLayout(std::unique_ptr(new PatternLayout(pattern))); 47 | Logger logger_1 = Logger::getInstance(LOG4CPLUS_TEXT("c++")); 48 | logger_1.addAppender(append_1); 49 | logger_1.setLogLevel(log4cplus::WARN_LOG_LEVEL); 50 | 51 | int cycle = 10000; 52 | 53 | while (cycle > 0) 54 | { 55 | cycle = cycle - 1; 56 | LOG4CPLUS_ERROR(logger_1, LOG4CPLUS_TEXT("This is not hotspots")); 57 | auto start = system_clock::now(); 58 | for( int a = 0; a < log_number; a = a + 1 ) 59 | { 60 | LOG4CPLUS_ERROR(logger_1, "" << dict[log_length]); 61 | } 62 | auto end = system_clock::now(); 63 | int duration = duration_cast(end - start).count(); 64 | // LOG4CPLUS_ERROR(logger_1, "Runtime: " << duration); 65 | int sleep = 100000 - duration; // 100ms - duration 66 | usleep(sleep); // sleep us 67 | } 68 | 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /c++/log4cplus-2.0.8.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/c++/log4cplus-2.0.8.tar.bz2 -------------------------------------------------------------------------------- /generate_log.py: -------------------------------------------------------------------------------- 1 | import string 2 | import random 3 | 4 | x = string.ascii_letters 5 | y = [random.choice(x) for i in range(500-20)] 6 | y = "".join(y) 7 | y = "This is hotspots." + y 8 | print(y) 9 | 10 | 11 | print(len("This is log hotspot!")) 12 | 13 | log20 = "This is log hotspot!" 14 | log50 = "This is hotspots.arXiEDHcPlfrcKwJxWjrjAYLvWkgzD" 15 | log100 = "This is hotspots.bJzsxyrMpzYGcfgKquEUPTHQtiFzeEHJxTEmdAtQQHhpfjtrKClFLsfjacHuTDkgzWNGmSOaZnzLxoUp" 16 | log200 = "This is hotspots.aRdUTDyLjPkZknRfLlCSxcEQLUKQTnbNCaMEnKLhVFmdNSQjInhGjrUMqMHPyScbMKBEPAIkxLukJUfImotUtzxIbhjMOHUpLqbbTYEXsOToClWMkOydioyhXzvcLiXtAMMssTROiXDwLLwInfaZyXptcxnRjPafxftKWtBBPQqodLPdvmEB" 17 | log500 = "This is hotspots.kdeHKrCcqKeCxYqRbXDENnFBuJemiQMuNvresaULGgozUxAugbywwYJEaqQbYGWcISrRlEnXdpoNZNwvdnzxoaXLMWHLxghZixRDiZbVTKynrgQjJisDcaGwanDipFZGRLhJkXqnDlPlcaHBnYYwRQcpnbBWcsRwaOWyVjVORurodxkMGXvUPNoKJKcUrhiDuTpVCQWpmbJYkNYiVyiPYlllLCOMPIJGgLiUCjOSnmFcXtmnUXyKBvDgKTCVRLYZzkTPhtKPApvFMuVVuEcuqPcQLLbHHvumugBgmgOiguyYbdMsfwQLguqOBeakiSjhkSKBFTSjuoMViXPeMCQllfTdvPVJcRpjtACHfxJkhHYrLUyvCTKBvXzNHxikUFbngYdERraSrrvlGRciDalgxLIVTrbdEnORNIomHRtbFfPnJHFWLfAjFxWpKhLHZewgavRdVmbfSIzfHMfFxtIlEOpOyxGyAKnX" 18 | log1000 = "This is hotspots.htTGLdjysnJBmraMbjsSytuKfxduwsdQJyzsIneGUNyiIYLwXBHgWupqOXLfUXOnpYbkJriscvFxGMRXbQsuipuBTsiMjpXKpbNcEoFWDDiMGeORcLWacLLOdcMaEVMcPDTpwNeEvUCgfcAARWsVBogiRstmyNCFwLfyvRdfdESZeSdjVzXDVGIKQztMylXzZfgCSzKIjiVtYwjHNTIwFDvhpyTgjKLfAFOHtJaVKEmzxDxrQJPmENyfBpQTDgFtwJYSwGPJkXBxoWKyhlVgavTnAOIgxSMsupviMzCttszFjubiNpFuierLCDVhsLudMgFbyaRFuapcYfGrvMVrIKWvoDEBHAdgLcGWVxMduuMkhHtAUtnCKHcYFkvXTCbixbGKpZBCuOkpcTCVTsGFZahRgzCwPHmKqaYougbHiOiDDSHatwIAKQDqhKNJEuydQfwcBoIeMotVHIpNzhcIndmUaNiHfYltuJuJNraDdUysIKwBNclJFLUbajVSgVheJbFdPjlTlnafBFAqeKQPvQvgjTLISfYKVmZcfReYOnRRlneMtBuVxCMapyYPZShEWkMsdTIYxLgjEawGXwXvSvUkAvclpOPKSTxrOIfHSMLcmRlfzXlbIPHpmncnfhGWIRfcVsZazehLGuYfVEfjlzuluPxlFKxeYJjmtEzYTVGOAjuMIpUYEwNeFNOeqoyHkJhrzNtsyiYBNHRmlMCZubXmwBRXingWgbYMRyabuABkGUpWQBQLdEtYHyRnCocXgirNamwpkBmdfjuzPzbvxtzdwHwXZrffSSeKPvkRkwLvofZmXBwABRhimRkAiMxFZRKflFdVKDcBjpCBvMZaJVnqyuqxCvfVRWzLijTQAjPIbBBRUTFKeRabpkeUddhfONWuRAhxbghoIKHIYhyvjBxyywIQsvUveuJvZzprbTaBKQJpRBiezwfTAtRDpmitJqPYxWhBhAuvvQjbbyjf" 19 | -------------------------------------------------------------------------------- /golang/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/golang/README.md -------------------------------------------------------------------------------- /golang/go.mod: -------------------------------------------------------------------------------- 1 | module golang 2 | 3 | go 1.16 4 | 5 | require ( 6 | go.uber.org/atomic v1.9.0 // indirect 7 | go.uber.org/multierr v1.8.0 // indirect 8 | go.uber.org/zap v1.22.0 // indirect 9 | k8s.io/klog v1.0.0 10 | ) 11 | -------------------------------------------------------------------------------- /golang/go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 5 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 6 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 7 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 8 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 12 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 13 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 14 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 15 | go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= 16 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 17 | go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= 18 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 19 | go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= 20 | go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= 21 | go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0= 22 | go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= 23 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 24 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 25 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 26 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 27 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 28 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 29 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 30 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 31 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 32 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 33 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 34 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 35 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 37 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 39 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 40 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 41 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 42 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 43 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 44 | golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 45 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 46 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 47 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 48 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 49 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 50 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 51 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 52 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 53 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 54 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 55 | -------------------------------------------------------------------------------- /golang/golang: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/golang/golang -------------------------------------------------------------------------------- /golang/hello: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/golang/hello -------------------------------------------------------------------------------- /golang/hello.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | ) 12 | 13 | var sugarLogger *zap.SugaredLogger 14 | 15 | func InitLogger() { 16 | writeSyncer := getLogWriter() 17 | encoder := getEncoder() 18 | core := zapcore.NewCore(encoder, writeSyncer, zapcore.DebugLevel) 19 | 20 | logger := zap.New(core, zap.AddCaller()) 21 | sugarLogger = logger.Sugar() 22 | } 23 | 24 | func getEncoder() zapcore.Encoder { 25 | encoderConfig := zapcore.EncoderConfig{ 26 | TimeKey: "ts", 27 | LevelKey: "level", 28 | NameKey: "logger", 29 | CallerKey: "caller", 30 | FunctionKey: zapcore.OmitKey, 31 | MessageKey: "msg", 32 | StacktraceKey: "stacktrace", 33 | LineEnding: zapcore.DefaultLineEnding, 34 | EncodeLevel: zapcore.LowercaseLevelEncoder, 35 | EncodeTime: zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05.000"), 36 | EncodeDuration: zapcore.SecondsDurationEncoder, 37 | EncodeCaller: zapcore.ShortCallerEncoder, 38 | } 39 | return zapcore.NewJSONEncoder(encoderConfig) 40 | } 41 | 42 | func getLogWriter() zapcore.WriteSyncer { 43 | file, _ := os.Create("/data/home/logbench/golang/golang.log") 44 | return zapcore.AddSync(file) 45 | } 46 | 47 | var ( 48 | h bool 49 | logNumber int 50 | logLength string 51 | ) 52 | 53 | func init() { 54 | const ( 55 | loglenth = "log20" 56 | ) 57 | 58 | flag.BoolVar(&h, "h", false, "this help") 59 | flag.StringVar(&logLength, "length", loglenth, "log length") 60 | flag.IntVar(&logNumber, "lognumber", 100, "log number") 61 | } 62 | 63 | func main() { 64 | // _, err := os.Stat("golang.log") 65 | // fmt.Println(err) 66 | // if err != nil { 67 | // fmt.Println(err) 68 | // } else { 69 | // err = os.Remove("/data/home/bobgbyu/LangBench/golang/golang.log") 70 | // if err != nil { 71 | // fmt.Println(err) 72 | // } 73 | // } 74 | 75 | InitLogger() 76 | defer sugarLogger.Sync() 77 | 78 | printLog := map[string]string{"log20": "This is log hotspot!", "log50": "This is hotspots.arXiEDHcPlfrcKwJxWjrjAYLvWkgzD", "log100": "This is hotspots.bJzsxyrMpzYGcfgKquEUPTHQtiFzeEHJxTEmdAtQQHhpfjtrKClFLsfjacHuTDkgzWNGmSOaZnzLxoUp", 79 | "log200": "This is hotspots.aRdUTDyLjPkZknRfLlCSxcEQLUKQTnbNCaMEnKLhVFmdNSQjInhGjrUMqMHPyScbMKBEPAIkxLukJUfImotUtzxIbhjMOHUpLqbbTYEXsOToClWMkOydioyhXzvcLiXtAMMssTROiXDwLLwInfaZyXptcxnRjPafxftKWtBBPQqodLPdvmEB", 80 | "log500": "This is hotspots.kdeHKrCcqKeCxYqRbXDENnFBuJemiQMuNvresaULGgozUxAugbywwYJEaqQbYGWcISrRlEnXdpoNZNwvdnzxoaXLMWHLxghZixRDiZbVTKynrgQjJisDcaGwanDipFZGRLhJkXqnDlPlcaHBnYYwRQcpnbBWcsRwaOWyVjVORurodxkMGXvUPNoKJKcUrhiDuTpVCQWpmbJYkNYiVyiPYlllLCOMPIJGgLiUCjOSnmFcXtmnUXyKBvDgKTCVRLYZzkTPhtKPApvFMuVVuEcuqPcQLLbHHvumugBgmgOiguyYbdMsfwQLguqOBeakiSjhkSKBFTSjuoMViXPeMCQllfTdvPVJcRpjtACHfxJkhHYrLUyvCTKBvXzNHxikUFbngYdERraSrrvlGRciDalgxLIVTrbdEnORNIomHRtbFfPnJHFWLfAjFxWpKhLHZewgavRdVmbfSIzfHMfFxtIlEOpOyxGyAKnX", 81 | "log1000": "This is hotspots.htTGLdjysnJBmraMbjsSytuKfxduwsdQJyzsIneGUNyiIYLwXBHgWupqOXLfUXOnpYbkJriscvFxGMRXbQsuipuBTsiMjpXKpbNcEoFWDDiMGeORcLWacLLOdcMaEVMcPDTpwNeEvUCgfcAARWsVBogiRstmyNCFwLfyvRdfdESZeSdjVzXDVGIKQztMylXzZfgCSzKIjiVtYwjHNTIwFDvhpyTgjKLfAFOHtJaVKEmzxDxrQJPmENyfBpQTDgFtwJYSwGPJkXBxoWKyhlVgavTnAOIgxSMsupviMzCttszFjubiNpFuierLCDVhsLudMgFbyaRFuapcYfGrvMVrIKWvoDEBHAdgLcGWVxMduuMkhHtAUtnCKHcYFkvXTCbixbGKpZBCuOkpcTCVTsGFZahRgzCwPHmKqaYougbHiOiDDSHatwIAKQDqhKNJEuydQfwcBoIeMotVHIpNzhcIndmUaNiHfYltuJuJNraDdUysIKwBNclJFLUbajVSgVheJbFdPjlTlnafBFAqeKQPvQvgjTLISfYKVmZcfReYOnRRlneMtBuVxCMapyYPZShEWkMsdTIYxLgjEawGXwXvSvUkAvclpOPKSTxrOIfHSMLcmRlfzXlbIPHpmncnfhGWIRfcVsZazehLGuYfVEfjlzuluPxlFKxeYJjmtEzYTVGOAjuMIpUYEwNeFNOeqoyHkJhrzNtsyiYBNHRmlMCZubXmwBRXingWgbYMRyabuABkGUpWQBQLdEtYHyRnCocXgirNamwpkBmdfjuzPzbvxtzdwHwXZrffSSeKPvkRkwLvofZmXBwABRhimRkAiMxFZRKflFdVKDcBjpCBvMZaJVnqyuqxCvfVRWzLijTQAjPIbBBRUTFKeRabpkeUddhfONWuRAhxbghoIKHIYhyvjBxyywIQsvUveuJvZzprbTaBKQJpRBiezwfTAtRDpmitJqPYxWhBhAuvvQjbbyjf"} 82 | 83 | flag.Parse() 84 | 85 | logContent := printLog[logLength] 86 | 87 | startT := time.Now() //计算当前时间 88 | // log_number := 1000 // print time per 100 ms 89 | interval := time.Duration(100) 90 | t := time.NewTicker(interval * time.Millisecond) 91 | 92 | count := 0 93 | cycle := 10000 94 | for { 95 | <-t.C 96 | count++ 97 | sugarLogger.Error("This is not hotspots") 98 | for i := 0; i < logNumber; i++ { 99 | sugarLogger.Error(logContent) 100 | } 101 | if count > cycle { 102 | t.Stop() 103 | tc := time.Since(startT) //计算耗时 104 | fmt.Printf("time cost = %v\n", tc) 105 | } 106 | } 107 | 108 | // for cycle := 1000; cycle > 0; cycle++ { 109 | // sugarLogger.Error("This is not hotspots") 110 | // start := time.Now().UTC() 111 | // for i := 0; i < each_interval_log; i++ { 112 | // sugarLogger.Error("This is not hotspots", i) 113 | // } 114 | // pTime := time.Now().UTC().Sub(start) 115 | // var pSecond = float64(pTime) 116 | // fmt.Println(time.Second*time.Duration(interval) - time.Nanosecond*time.Duration(pSecond)) 117 | // time.Sleep(time.Second*time.Duration(interval) - time.Nanosecond*time.Duration(pSecond)) 118 | // } 119 | 120 | // for i := 0; i < 1000; i++ { 121 | // for j := 0; j < 10000; j++ { 122 | // sugarLogger.Error(i, j) 123 | // } 124 | // } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /java/helloLog/README.md: -------------------------------------------------------------------------------- 1 | ## Java 2 | ``` 3 | mvn clean package 4 | 5 | java -jar /data/home/logbench/java/helloLog/target/helloLog-jar-with-dependencies.jar -l 2000 6 | ``` -------------------------------------------------------------------------------- /java/helloLog/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.yuxiaoba.hello 5 | helloLog 6 | jar 7 | 1.0 8 | helloLog 9 | 10 | 11 | 12 | org.apache.maven.plugins 13 | maven-jar-plugin 14 | 3.2.0 15 | 16 | 17 | 18 | true 19 | 20 | 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-compiler-plugin 26 | 27 | 11 28 | 11 29 | 30 | 31 | 32 | org.apache.maven.plugins 33 | maven-assembly-plugin 34 | 3.3.0 35 | 36 | 37 | hello 38 | package 39 | 40 | 41 | 42 | com.yuxiaoba.hello.Hello 43 | 44 | 45 | 46 | jar-with-dependencies 47 | 48 | helloLog 49 | 50 | 51 | single 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | org.apache.logging.log4j 62 | log4j-api 63 | 2.18.0 64 | compile 65 | 66 | 67 | org.apache.logging.log4j 68 | log4j-core 69 | 2.18.0 70 | compile 71 | 72 | 73 | commons-io 74 | commons-io 75 | 2.8.0 76 | 77 | 78 | commons-cli 79 | commons-cli 80 | 1.5.0 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /java/helloLog/src/main/java/com/yuxiaoba/hello/Hello.java: -------------------------------------------------------------------------------- 1 | package com.yuxiaoba.hello; 2 | 3 | import org.apache.logging.log4j.LogManager; 4 | import org.apache.logging.log4j.Logger; 5 | import java.util.*; 6 | 7 | import org.apache.commons.cli.CommandLineParser; 8 | import org.apache.commons.cli.ParseException; 9 | import org.apache.commons.cli.DefaultParser; 10 | import org.apache.commons.cli.HelpFormatter; 11 | import org.apache.commons.cli.CommandLine; 12 | import org.apache.commons.cli.Options; 13 | import org.apache.commons.cli.Option; 14 | 15 | public class Hello { 16 | public static void main(String[] args) { 17 | CommandLine line = parseArg(args); 18 | int logNumber = 100; 19 | String logLength = "log20"; 20 | 21 | if (line.hasOption('h') == false) { 22 | if (line.hasOption("lognumber") == false) { 23 | System.out.println("Get empty lognumber, user 100 at default!"); 24 | } else { 25 | logNumber = Integer.parseInt(line.getOptionValue("lognumber")); 26 | } 27 | if (line.hasOption("length") == false) { 28 | System.out.println("Get empty log length, user log20 at default!"); 29 | } else { 30 | logLength = line.getOptionValue("length"); 31 | } 32 | } 33 | 34 | Hashtable printLog = new Hashtable(); 35 | printLog.put("log20", "This is log hotspot!"); 36 | printLog.put("log50", "This is hotspots.arXiEDHcPlfrcKwJxWjrjAYLvWkgzD"); 37 | printLog.put("log100", 38 | "This is hotspots.bJzsxyrMpzYGcfgKquEUPTHQtiFzeEHJxTEmdAtQQHhpfjtrKClFLsfjacHuTDkgzWNGmSOaZnzLxoUp"); 39 | printLog.put("log200", 40 | "This is hotspots.aRdUTDyLjPkZknRfLlCSxcEQLUKQTnbNCaMEnKLhVFmdNSQjInhGjrUMqMHPyScbMKBEPAIkxLukJUfImotUtzxIbhjMOHUpLqbbTYEXsOToClWMkOydioyhXzvcLiXtAMMssTROiXDwLLwInfaZyXptcxnRjPafxftKWtBBPQqodLPdvmEB"); 41 | printLog.put("log500", 42 | "This is hotspots.kdeHKrCcqKeCxYqRbXDENnFBuJemiQMuNvresaULGgozUxAugbywwYJEaqQbYGWcISrRlEnXdpoNZNwvdnzxoaXLMWHLxghZixRDiZbVTKynrgQjJisDcaGwanDipFZGRLhJkXqnDlPlcaHBnYYwRQcpnbBWcsRwaOWyVjVORurodxkMGXvUPNoKJKcUrhiDuTpVCQWpmbJYkNYiVyiPYlllLCOMPIJGgLiUCjOSnmFcXtmnUXyKBvDgKTCVRLYZzkTPhtKPApvFMuVVuEcuqPcQLLbHHvumugBgmgOiguyYbdMsfwQLguqOBeakiSjhkSKBFTSjuoMViXPeMCQllfTdvPVJcRpjtACHfxJkhHYrLUyvCTKBvXzNHxikUFbngYdERraSrrvlGRciDalgxLIVTrbdEnORNIomHRtbFfPnJHFWLfAjFxWpKhLHZewgavRdVmbfSIzfHMfFxtIlEOpOyxGyAKnX"); 43 | printLog.put("log1000", 44 | "This is hotspots.htTGLdjysnJBmraMbjsSytuKfxduwsdQJyzsIneGUNyiIYLwXBHgWupqOXLfUXOnpYbkJriscvFxGMRXbQsuipuBTsiMjpXKpbNcEoFWDDiMGeORcLWacLLOdcMaEVMcPDTpwNeEvUCgfcAARWsVBogiRstmyNCFwLfyvRdfdESZeSdjVzXDVGIKQztMylXzZfgCSzKIjiVtYwjHNTIwFDvhpyTgjKLfAFOHtJaVKEmzxDxrQJPmENyfBpQTDgFtwJYSwGPJkXBxoWKyhlVgavTnAOIgxSMsupviMzCttszFjubiNpFuierLCDVhsLudMgFbyaRFuapcYfGrvMVrIKWvoDEBHAdgLcGWVxMduuMkhHtAUtnCKHcYFkvXTCbixbGKpZBCuOkpcTCVTsGFZahRgzCwPHmKqaYougbHiOiDDSHatwIAKQDqhKNJEuydQfwcBoIeMotVHIpNzhcIndmUaNiHfYltuJuJNraDdUysIKwBNclJFLUbajVSgVheJbFdPjlTlnafBFAqeKQPvQvgjTLISfYKVmZcfReYOnRRlneMtBuVxCMapyYPZShEWkMsdTIYxLgjEawGXwXvSvUkAvclpOPKSTxrOIfHSMLcmRlfzXlbIPHpmncnfhGWIRfcVsZazehLGuYfVEfjlzuluPxlFKxeYJjmtEzYTVGOAjuMIpUYEwNeFNOeqoyHkJhrzNtsyiYBNHRmlMCZubXmwBRXingWgbYMRyabuABkGUpWQBQLdEtYHyRnCocXgirNamwpkBmdfjuzPzbvxtzdwHwXZrffSSeKPvkRkwLvofZmXBwABRhimRkAiMxFZRKflFdVKDcBjpCBvMZaJVnqyuqxCvfVRWzLijTQAjPIbBBRUTFKeRabpkeUddhfONWuRAhxbghoIKHIYhyvjBxyywIQsvUveuJvZzprbTaBKQJpRBiezwfTAtRDpmitJqPYxWhBhAuvvQjbbyjf"); 45 | String logContent = printLog.get(logLength).toString(); 46 | System.out.println("Start java log bench!"); 47 | System.out.println(logNumber); 48 | Timer timer = new Timer(); 49 | 50 | // run per 100ms, stop if run 1000 times 51 | timer.schedule(new Task(timer, logNumber, 10000,logContent), 0, 100); 52 | } 53 | 54 | private static CommandLine parseArg(String[] args) { 55 | // create the parser 56 | CommandLineParser parser = new DefaultParser(); 57 | // create Options object 58 | final Options options = new Options(); 59 | options.addOption(new Option("n", "lognumber", true, "Print number of log per 100ms")); 60 | options.addOption(new Option("l", "length", true, "Print the lenghth of log")); 61 | options.addOption(new Option("h", "help", false, "Display help text.")); 62 | 63 | try { 64 | // parse the command line arguments 65 | CommandLine line = parser.parse(options, args); 66 | 67 | if (line.hasOption('h')) { 68 | // automatically generate the help statement 69 | HelpFormatter formatter = new HelpFormatter(); 70 | formatter.printHelp("java log bench help", options); 71 | } 72 | return line; 73 | } catch (ParseException exp) { 74 | // oops, something went wrong 75 | System.err.println("Parsing failed. Reason: " + exp.getMessage()); 76 | 77 | } 78 | return null; 79 | } 80 | 81 | } 82 | 83 | class Task extends TimerTask { 84 | private Timer timer; 85 | private int exeCount; 86 | private int logNumber; 87 | private String logContent; 88 | private static final Logger logger = LogManager.getLogger(Hello.class.getName()); 89 | 90 | public Task(Timer timer, int logNumber, int exeCount, String logContent) { 91 | this.timer = timer; 92 | this.exeCount = exeCount; 93 | this.logNumber = logNumber; 94 | this.logContent = logContent; 95 | } 96 | 97 | private int count = 1; 98 | 99 | @Override 100 | public void run() { 101 | logger.error("This is not log hotspots"); 102 | 103 | for (int i = 0; i < logNumber; i++) { 104 | logger.error(logContent); 105 | } 106 | count = count + 1; 107 | 108 | // 若满足此条件退出此线程 109 | if (count > exeCount) { 110 | this.timer.cancel(); 111 | System.out.println("Java log bench stop"); 112 | } 113 | } 114 | } -------------------------------------------------------------------------------- /java/helloLog/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /java/helloLog/target/classes/com/yuxiaoba/hello/Hello.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/java/helloLog/target/classes/com/yuxiaoba/hello/Hello.class -------------------------------------------------------------------------------- /java/helloLog/target/classes/com/yuxiaoba/hello/Task.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/java/helloLog/target/classes/com/yuxiaoba/hello/Task.class -------------------------------------------------------------------------------- /java/helloLog/target/classes/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /java/helloLog/target/helloLog-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/java/helloLog/target/helloLog-1.0.jar -------------------------------------------------------------------------------- /java/helloLog/target/helloLog-jar-with-dependencies.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/java/helloLog/target/helloLog-jar-with-dependencies.jar -------------------------------------------------------------------------------- /java/helloLog/target/maven-archiver/pom.properties: -------------------------------------------------------------------------------- 1 | artifactId=helloLog 2 | groupId=com.yuxiaoba.hello 3 | version=1.0 4 | -------------------------------------------------------------------------------- /java/helloLog/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst: -------------------------------------------------------------------------------- 1 | com/yuxiaoba/hello/Hello.class 2 | com/yuxiaoba/hello/Task.class 3 | -------------------------------------------------------------------------------- /java/helloLog/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst: -------------------------------------------------------------------------------- 1 | /data/home/logbench/java/helloLog/src/main/java/com/yuxiaoba/hello/Hello.java 2 | -------------------------------------------------------------------------------- /java/helloLog/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/java/helloLog/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst -------------------------------------------------------------------------------- /logReducer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define MAX_NB_PACKETS 1000 6 | #define LEGAL_DIFF_TIMESTAMP_PACKETS 1000000 7 | #define MAX_FILTER_SIZE 4 8 | #define MAX_FILTER_LENGTH 64 9 | #define MAX_FILE_NAME 64 10 | #define MAX_PATH_NAME 128 11 | #define MAX_LOG_SIZE 1024 12 | #define LOG_BLOCK_SIZE 64 13 | struct filterNode 14 | { 15 | char filerExpr[MAX_FILTER_LENGTH]; 16 | }; 17 | 18 | struct listenKey 19 | { 20 | u32 pid; 21 | u32 fd; 22 | }; 23 | 24 | struct listenVal 25 | { 26 | int needListen; 27 | char fileName[MAX_FILE_NAME]; 28 | }; 29 | 30 | struct writePackages 31 | { 32 | u32 pid; 33 | u32 fd; 34 | char contents[LOG_BLOCK_SIZE]; 35 | char fileName[MAX_FILE_NAME]; 36 | }; 37 | 38 | BPF_HASH(filter_map, u32, struct filterNode); 39 | BPF_HASH(start_map, u32, u32); 40 | BPF_HASH(pid_map, u32, u32); 41 | BPF_HASH(file_name_map, u32, const char *); 42 | BPF_HASH(listen_map, struct listenKey, struct listenVal); 43 | 44 | // https://github.com/iovisor/bcc/blob/e83019bdf6c400b589e69c7d18092e38088f89a8/tests/python/test_call1.c 45 | BPF_PROG_ARRAY(jump, 16); 46 | 47 | BPF_PERF_OUTPUT(events_write); 48 | BPF_PERF_OUTPUT(events_find); 49 | 50 | int detect_file_open(struct pt_regs *ctx) 51 | { 52 | // 监听程序准备打开一个文件，这时候我们可以获得文件名，保存起来 53 | u64 tgid = bpf_get_current_pid_tgid(); 54 | u32 pid = tgid >> 32; 55 | 56 | if (!pid_map.lookup(&pid)) 57 | { 58 | return 0; 59 | } 60 | // bpf_trace_printk("open filter pid %d\n", pid); 61 | 62 | // bpf_probe_read_kernel_str in 5.5 or latest 63 | const char *name = (const char __user *)PT_REGS_PARM2(ctx); 64 | bpf_trace_printk("open: %s, pid: %d\n", name, pid); 65 | 66 | // A simple postfix comparision, for postfix without duplicate character 67 | int state = 0; 68 | char postfix[] = ".log"; 69 | for (int i = 0; i < MAX_FILE_NAME; i++) 70 | { 71 | for (int j = sizeof(postfix) - 1; j >= 0; j--) 72 | { 73 | if (state == j) 74 | { 75 | state = (name[i] == postfix[state]) ? state + 1 : 0; 76 | } 77 | } 78 | 79 | if (name[i] == '\0') 80 | { 81 | break; 82 | } 83 | } 84 | 85 | if (state == sizeof(postfix)) 86 | { 87 | // bpf_trace_printk("update\n"); 88 | file_name_map.update(&pid, &name); 89 | } 90 | 91 | return 0; 92 | } 93 | 94 | int detect_file_open_ret(struct pt_regs *ctx) 95 | { 96 | // 监听程序完成了文件打开操作，这时候我们可以获取到文件的fd，保存起来 97 | const char **name; 98 | u64 tgid = bpf_get_current_pid_tgid(); 99 | u32 pid = tgid >> 32; 100 | name = file_name_map.lookup(&pid); 101 | if (!name) 102 | { 103 | return 0; 104 | } 105 | 106 | // bpf_trace_printk("pass\n"); 107 | struct listenKey key = {}; 108 | key.pid = pid; 109 | key.fd = PT_REGS_RC(ctx); 110 | 111 | struct listenVal val = {}; 112 | bpf_probe_read_kernel(val.fileName, MAX_FILE_NAME, *name); 113 | 114 | file_name_map.delete(&pid); 115 | listen_map.update(&key, &val); 116 | 117 | return 0; 118 | } 119 | 120 | int detect_file_write(struct pt_regs *ctx) 121 | { 122 | // 判断是否是监听文件的写入 123 | u64 tgid = bpf_get_current_pid_tgid(); 124 | u32 pid = tgid >> 32; 125 | 126 | if (!pid_map.lookup(&pid)) 127 | { 128 | return 0; 129 | } 130 | 131 | struct pt_regs reg = {}; 132 | char *buffer = (char *)PT_REGS_PARM1(ctx); // 拿出函数调用时的用户态寄存器地址 133 | bpf_probe_read(®, sizeof(struct pt_regs), buffer); // 将寄存器内容拷贝一份到ebpf 134 | struct listenKey key = {}; // 构建哈希表查找key，用于获取当前写入的文件 135 | key.pid = pid; 136 | key.fd = (u32)PT_REGS_PARM1(®); 137 | 138 | // bpf_trace_printk("write pid: %d fd: %d\n", pid, key.fd); 139 | 140 | struct listenVal *keep_listen; 141 | keep_listen = listen_map.lookup(&key); // 判断当前写入文件是否需要监听 142 | if (!keep_listen) 143 | { 144 | // 没有查到监听信息，上报到用户端程序中由它来判断 145 | struct listenVal needListen = {}; 146 | needListen.needListen = 0; 147 | listen_map.update(&key, &needListen); 148 | events_find.perf_submit(ctx, &needListen.needListen, sizeof(int)); 149 | return 0; 150 | } 151 | else if (keep_listen->needListen != 1) 152 | { 153 | // 有监听信息且用户端认为不需要监听，跳过 154 | return 0; 155 | } 156 | 157 | // 该写入需要监听，判断写入内容是否匹配模板 158 | // bpf_trace_printk("pass %d name: %s\n", pid, keep_listen->fileName); 159 | char block_buffer[LOG_BLOCK_SIZE]; 160 | int fail; 161 | struct filterNode *node; 162 | u32 *start; 163 | struct writePackages writePackage = {}; 164 | 165 | // 获取写操作传入内容 166 | void *ptr = (void *)PT_REGS_PARM2(®); 167 | 168 | // 给予count边界，确保循环有界以通过编译 169 | size_t count = (size_t)PT_REGS_PARM3(®); 170 | if (count <= 0) 171 | { 172 | return 0; 173 | } 174 | else if (count > MAX_LOG_SIZE) 175 | { 176 | count = MAX_LOG_SIZE; 177 | } 178 | 179 | // 开始匹配模板 180 | int filter_key = 0; 181 | node = filter_map.lookup(&filter_key); 182 | start = start_map.lookup(&filter_key); 183 | if (!start) 184 | { 185 | return 0; 186 | } 187 | 188 | // bpf_trace_printk("start: %d", *start); 189 | int str = *start; 190 | // bpf_trace_printk("str: %d", str); 191 | // 从 ptr 开始读，读取日志前LOG_BLOCK_SIZE个字符 192 | bpf_probe_read(&block_buffer, sizeof(char) * LOG_BLOCK_SIZE, ptr+str); // 从内核中读取数据 193 | 194 | // bpf_trace_printk("block_buffer: %s"); 195 | 196 | if (!node) 197 | { 198 | return 0; 199 | } 200 | 201 | for (int i = 0; i < LOG_BLOCK_SIZE; i++) 202 | { 203 | if (node->filerExpr[i] == '\0') 204 | { // 筛选成功，拦截 205 | bpf_override_return(ctx, count); 206 | } 207 | 208 | if (block_buffer[i] != node->filerExpr[i]) 209 | { // 不满足前缀，跳过当前配置 210 | break; 211 | } 212 | } 213 | 214 | return 0; 215 | 216 | // success: 217 | // // 上报数据并拦截实际磁盘写入 218 | // writePackage.fd = key.fd; 219 | // writePackage.pid = key.pid; 220 | // __builtin_memcpy(writePackage.fileName, keep_listen->fileName, MAX_FILE_NAME); 221 | // bpf_trace_printk("length %d name: \n", count); 222 | 223 | // for (int cur = 0; cur < count; cur += LOG_BLOCK_SIZE) 224 | // { 225 | // // Use bpf_probe_read_kernel_str instead in 5.5 or later 226 | // if (count - cur <= LOG_BLOCK_SIZE) 227 | // { 228 | // bpf_probe_read(&writePackage.contents, sizeof(char) * (count - cur), ptr + cur); 229 | // events_write.perf_submit(ctx, &writePackage, sizeof(writePackage)); 230 | // break; 231 | // } 232 | // else 233 | // { 234 | // bpf_probe_read(&writePackage.contents, sizeof(char) * LOG_BLOCK_SIZE, ptr + cur); 235 | // events_write.perf_submit(ctx, &writePackage, sizeof(writePackage)); 236 | // } 237 | // } 238 | // bpf_override_return(ctx, count); 239 | 240 | // // 尾调用函数无法调用bpf_override_return，原因不明 241 | // // jump.call(ctx, 2); 242 | // return 0; 243 | 244 | } 245 | 246 | // int upload_and_block(struct pt_regs *ctx) 247 | // { 248 | // bpf_override_return(ctx, count); 249 | // return 0; 250 | // } -------------------------------------------------------------------------------- /logReducer.py: -------------------------------------------------------------------------------- 1 | from ast import arg 2 | from bcc import BPF 3 | # from elasticsearch import Elasticsearch 4 | import ctypes as ct 5 | import datetime 6 | import argparse 7 | import time 8 | import os 9 | import re 10 | import yaml 11 | 12 | 13 | def get_pid(program_name): 14 | get_pid_command = "ps -ef | grep %s | grep -v grep |grep -v program | awk \'{print $2}\'" % program_name 15 | result = os.popen(get_pid_command) 16 | pid = result.read() 17 | 18 | if "\n" in pid: 19 | pid = pid.strip("\n") 20 | 21 | print(pid) 22 | if pid == "": 23 | print("program not found") 24 | return 0 25 | return int(pid) 26 | 27 | def extract_python_template(template): 28 | start = len(template.split("[")[0]) 29 | location = "[" + template.split("[")[1] 30 | print("python", start,location) 31 | return start, location 32 | 33 | def extract_golang_template(template): 34 | start = len(template.split("/")[0]) + 2 35 | location = template.split("/")[1].split(",")[0] 36 | print("golang", start,location) 37 | return start, location 38 | 39 | parser = argparse.ArgumentParser( 40 | description="filter logs of specific program and location", 41 | formatter_class=argparse.RawDescriptionHelpFormatter) 42 | # parser.add_argument("pid", nargs='+', type=int, 43 | # help="only trace those process whose name here") 44 | 45 | # with open("./config.yaml", "r", encoding="utf8") as f: 46 | # config = yaml.load(f) 47 | # print(config) 48 | parser.add_argument("--program", type=str, default="hello", 49 | help="only filter the program named here") 50 | parser.add_argument("--language", type=str, default="python", 51 | help="for filter template") 52 | parser.add_argument("--template", type=str, default="2022-08-15 10:13:22 - ERROR - [/data/home/logbench/python/hello.py:26]", 53 | help="only filter the template named here") 54 | 55 | args = parser.parse_args() 56 | 57 | pid = get_pid(args.program) 58 | 59 | if args.language=="python": 60 | start, location = extract_python_template(args.template) 61 | elif args.language=="golang": 62 | start, location = extract_golang_template(args.template) 63 | 64 | class WritePackage(ct.Structure): 65 | _fields_ = [ 66 | ("pid", ct.c_uint), 67 | ("fd", ct.c_uint), 68 | ("contents", ct.c_char * 64), 69 | ("fileName", ct.c_char * 32) 70 | ] 71 | 72 | 73 | # Loads eBPF program 74 | b = BPF(src_file="logReducer.c") 75 | 76 | # Attach tail call 77 | # https://github.com/iovisor/bcc/blob/e83019bdf6c400b589e69c7d18092e38088f89a8/tests/python/test_call1.py 78 | # https://programs.team/ebpf-from-bpf-to-bpf-calls-to-tail-calls.html 79 | # check_fn = b.load_func("check_template", BPF.KPROBE) 80 | # upload_fn = b.load_func("upload_and_block", BPF.KPROBE) 81 | # jump = b.get_table("jump") 82 | # # jump[ct.c_int(1)] = ct.c_int(check_fn.fd) 83 | # jump[ct.c_int(2)] = ct.c_int(upload_fn.fd) 84 | 85 | # Attach kprobe 86 | # 这里监听了文件打开和写入两部分系统调用 87 | # - 打开是为了获取到文件名和fd的对应关系 88 | # - 写入是实际要监听的内容 89 | b.attach_kprobe(event="do_sys_open", fn_name="detect_file_open") 90 | b.attach_kprobe(event=b.get_syscall_fnname( 91 | "write"), fn_name="detect_file_write") 92 | b.attach_kretprobe(event="do_sys_open", fn_name="detect_file_open_ret") 93 | 94 | # 监听文件触发写入，需要执行的动作写这里 95 | 96 | 97 | def trigger_alert_write(cpu, data, size): 98 | # global table 99 | event = ct.cast(data, ct.POINTER(WritePackage)).contents 100 | contents = event.contents.decode('utf-8', 'ignore') 101 | timestamp = datetime.datetime.now() 102 | print("%-29s (pid:%ld fd:%ld name:%s):%s" % ( 103 | timestamp, 104 | event.pid, 105 | event.fd, 106 | event.fileName, 107 | contents)) 108 | 109 | # # 考虑存一段时间再push 110 | # if es is not None: 111 | # result = es.index(index='logs', doc_type=event.fileName, body={ 112 | # "timestamp": timestamp, 113 | # "pid": event.pid, 114 | # "fd": event.fd, 115 | # "contents": contents 116 | # }) 117 | # print(result) 118 | 119 | # 程序运行中发现新文件写入操作，决策是否监听拦截 120 | 121 | 122 | def trigger_alert_find(cpu, data, size): 123 | global b 124 | listen_map = b["listen_map"] 125 | print("trigger") 126 | for listen_key in listen_map: 127 | listen_val = listen_map[listen_key] 128 | if listen_val.needListen == 0: 129 | path = os.readlink("/proc/%d/fd/%d" % 130 | (listen_key.pid, listen_key.fd)) 131 | filename = path.split("/")[-1] 132 | if re.match(r".+\.log$", filename): 133 | listen_map[listen_key] = listen_map.Leaf( 134 | 1, bytes(filename, encoding="utf-8")) 135 | else: 136 | listen_map[listen_key] = listen_map.Leaf(2, b"") 137 | 138 | 139 | b["events_write"].open_perf_buffer(trigger_alert_write) 140 | b["events_find"].open_perf_buffer(trigger_alert_find) 141 | 142 | # 初始化时遍历监听进程正在写入的文件，决策是否需要监听和拦截 143 | listen_map = b.get_table("listen_map") 144 | pid_map = b.get_table("pid_map") 145 | #for pid in args.pid: 146 | path = '/proc/%d/fd' % pid 147 | for fd in os.listdir(path): 148 | filename = os.readlink("%s/%s" % (path, fd)).split("/")[-1] 149 | if filename: 150 | print(pid, filename, fd) 151 | listen_key = listen_map.Key(pid, int(fd)) 152 | if re.match(r".+\.log$", filename): 153 | listen_map[listen_key] = listen_map.Leaf( 154 | 1, bytes(filename, encoding="utf-8")) 155 | else: 156 | listen_map[listen_key] = listen_map.Leaf(2, b"") 157 | pid_map[ct.c_uint(pid)] = ct.c_uint(pid) 158 | 159 | # 拦截内容匹配，只有匹配上的写入才会进行监听和拦截 160 | filter_map = b.get_table("filter_map") 161 | start_map = b.get_table("start_map") 162 | 163 | 164 | filter_map[ct.c_uint(0)] = filter_map.Leaf( 165 | bytes(location, encoding="utf-8")) 166 | start_map[ct.c_uint(0)] = ct.c_uint(start) 167 | 168 | # for idx, filter in enumerate(config["filter"]): 169 | # filter_map[ct.c_uint(idx)] = filter_map.Leaf( 170 | # bytes(filter, encoding="utf-8")) 171 | # start_map[ct.c_uint(idx)] = ct.c_uint(29) 172 | 173 | print("start reading... ") 174 | # b.trace_print() 175 | while 1: 176 | try: 177 | b.perf_buffer_poll() 178 | except KeyboardInterrupt: 179 | exit() 180 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | 2 | ``` 3 | cd /data/home/logbench/python 4 | python3 hello.py --lognumber 1000 5 | ``` -------------------------------------------------------------------------------- /python/__pycache__/nb_log_config.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IntelligentDDS/LogReducer/173a8b958a4de8292af880f2f102707b76b5362e/python/__pycache__/nb_log_config.cpython-36.pyc -------------------------------------------------------------------------------- /python/hello.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from nb_log import get_logger 3 | import timeit 4 | import time 5 | import argparse 6 | import os 7 | 8 | os.remove("/data/home/logbench/python/python.log") 9 | 10 | # logger = get_logger('python',, log_filename='python.log',formatter_template=5,log_file_handler_type=3,is_add_stream_handler=False) 11 | 12 | print_log = {"log20": "This is log hotspot!", "log50": "This is hotspots.arXiEDHcPlfrcKwJxWjrjAYLvWkgzD", "log100": "This is hotspots.bJzsxyrMpzYGcfgKquEUPTHQtiFzeEHJxTEmdAtQQHhpfjtrKClFLsfjacHuTDkgzWNGmSOaZnzLxoUp", 13 | "log200": "This is hotspots.aRdUTDyLjPkZknRfLlCSxcEQLUKQTnbNCaMEnKLhVFmdNSQjInhGjrUMqMHPyScbMKBEPAIkxLukJUfImotUtzxIbhjMOHUpLqbbTYEXsOToClWMkOydioyhXzvcLiXtAMMssTROiXDwLLwInfaZyXptcxnRjPafxftKWtBBPQqodLPdvmEB", 14 | "log500": "This is hotspots.kdeHKrCcqKeCxYqRbXDENnFBuJemiQMuNvresaULGgozUxAugbywwYJEaqQbYGWcISrRlEnXdpoNZNwvdnzxoaXLMWHLxghZixRDiZbVTKynrgQjJisDcaGwanDipFZGRLhJkXqnDlPlcaHBnYYwRQcpnbBWcsRwaOWyVjVORurodxkMGXvUPNoKJKcUrhiDuTpVCQWpmbJYkNYiVyiPYlllLCOMPIJGgLiUCjOSnmFcXtmnUXyKBvDgKTCVRLYZzkTPhtKPApvFMuVVuEcuqPcQLLbHHvumugBgmgOiguyYbdMsfwQLguqOBeakiSjhkSKBFTSjuoMViXPeMCQllfTdvPVJcRpjtACHfxJkhHYrLUyvCTKBvXzNHxikUFbngYdERraSrrvlGRciDalgxLIVTrbdEnORNIomHRtbFfPnJHFWLfAjFxWpKhLHZewgavRdVmbfSIzfHMfFxtIlEOpOyxGyAKnX", 15 | "log1000": "This is hotspots.htTGLdjysnJBmraMbjsSytuKfxduwsdQJyzsIneGUNyiIYLwXBHgWupqOXLfUXOnpYbkJriscvFxGMRXbQsuipuBTsiMjpXKpbNcEoFWDDiMGeORcLWacLLOdcMaEVMcPDTpwNeEvUCgfcAARWsVBogiRstmyNCFwLfyvRdfdESZeSdjVzXDVGIKQztMylXzZfgCSzKIjiVtYwjHNTIwFDvhpyTgjKLfAFOHtJaVKEmzxDxrQJPmENyfBpQTDgFtwJYSwGPJkXBxoWKyhlVgavTnAOIgxSMsupviMzCttszFjubiNpFuierLCDVhsLudMgFbyaRFuapcYfGrvMVrIKWvoDEBHAdgLcGWVxMduuMkhHtAUtnCKHcYFkvXTCbixbGKpZBCuOkpcTCVTsGFZahRgzCwPHmKqaYougbHiOiDDSHatwIAKQDqhKNJEuydQfwcBoIeMotVHIpNzhcIndmUaNiHfYltuJuJNraDdUysIKwBNclJFLUbajVSgVheJbFdPjlTlnafBFAqeKQPvQvgjTLISfYKVmZcfReYOnRRlneMtBuVxCMapyYPZShEWkMsdTIYxLgjEawGXwXvSvUkAvclpOPKSTxrOIfHSMLcmRlfzXlbIPHpmncnfhGWIRfcVsZazehLGuYfVEfjlzuluPxlFKxeYJjmtEzYTVGOAjuMIpUYEwNeFNOeqoyHkJhrzNtsyiYBNHRmlMCZubXmwBRXingWgbYMRyabuABkGUpWQBQLdEtYHyRnCocXgirNamwpkBmdfjuzPzbvxtzdwHwXZrffSSeKPvkRkwLvofZmXBwABRhimRkAiMxFZRKflFdVKDcBjpCBvMZaJVnqyuqxCvfVRWzLijTQAjPIbBBRUTFKeRabpkeUddhfONWuRAhxbghoIKHIYhyvjBxyywIQsvUveuJvZzprbTaBKQJpRBiezwfTAtRDpmitJqPYxWhBhAuvvQjbbyjf"} 16 | 17 | logger = get_logger(name="pythonbench", is_add_stream_handler=False, 18 | log_path="/data/home/logbench/python", log_filename='python.log', log_file_handler_type=3) 19 | 20 | if __name__ == "__main__": 21 | # number = 10000 # 试验次数 22 | # msg = "The benchmark cannot handler so mush logs at the intervarl, please reduce lognumber" # 日志内容 23 | # time_pass = timeit.timeit('logger.info(msg)', setup='from __main__ import logger, msg', number=number) 24 | # rate = number / time_pass # 每秒最大次数 25 | # print(rate) 26 | parser = argparse.ArgumentParser( 27 | description="generate specific number of logs per second", 28 | formatter_class=argparse.RawDescriptionHelpFormatter) 29 | parser.add_argument("--lognumber", type=int, default="100", 30 | help="number of logs per 100ms (less than 28000 in python)") 31 | parser.add_argument("--length", type=str, default="log20", 32 | help="length of log") 33 | args = parser.parse_args() 34 | 35 | 36 | interval = 0.1 37 | log_number = args.lognumber 38 | cycle = 10000 39 | each_interval_log = int(log_number) 40 | 41 | start = datetime.datetime.now() 42 | while cycle > 0: 43 | logger.error("This is not log hotspot") 44 | begin = time.time() 45 | for i in range(each_interval_log): 46 | logger.error("%s", print_log[args.length]) 47 | if interval-float(time.time()-begin) > 0: 48 | # print(interval-float(time.time()-begin)) 49 | time.sleep(interval-float(time.time()-begin)) 50 | else: 51 | logger.error( 52 | "The benchmark cannot handler so mush logs at the intervarl, please reduce lognumber") 53 | 54 | cycle = cycle - 1 55 | 56 | print((datetime.datetime.now()-start)) 57 | print("Python bench finish") 58 | -------------------------------------------------------------------------------- /python/nb_log_config.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | """ 3 | 此文件nb_log_config.py是自动生成到python项目的根目录的。 4 | 在这里面写的变量会覆盖此文件nb_log_config_default中的值。对nb_log包进行默认的配置。 5 | 但最终配置方式是由get_logger_and_add_handlers方法的各种传参决定，如果方法相应的传参为None则使用这里面的配置。 6 | """ 7 | 8 | """ 9 | 如果反对日志有各种彩色，可以设置 DEFAULUT_USE_COLOR_HANDLER = False 10 | 如果反对日志有块状背景彩色，可以设置 DISPLAY_BACKGROUD_COLOR_IN_CONSOLE = False 11 | 如果想屏蔽nb_log包对怎么设置pycahrm的颜色的提示，可以设置 WARNING_PYCHARM_COLOR_SETINGS = False 12 | 如果想改变日志模板，可以设置 FORMATTER_KIND 参数，只带了7种模板，可以自定义添加喜欢的模板 13 | LOG_PATH 配置文件日志的保存路径的文件夹。 14 | """ 15 | 16 | # noinspection PyUnresolvedReferences 17 | import logging 18 | import os 19 | # noinspection PyUnresolvedReferences 20 | from pathlib import Path # noqa 21 | import socket 22 | from pythonjsonlogger.jsonlogger import JsonFormatter 23 | 24 | 25 | def get_host_ip(): 26 | ip = '' 27 | host_name = '' 28 | # noinspection PyBroadException 29 | try: 30 | sc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 31 | sc.connect(('8.8.8.8', 80)) 32 | ip = sc.getsockname()[0] 33 | host_name = socket.gethostname() 34 | sc.close() 35 | except Exception: 36 | pass 37 | return ip, host_name 38 | 39 | 40 | computer_ip, computer_name = get_host_ip() 41 | 42 | 43 | class JsonFormatterJumpAble(JsonFormatter): 44 | def add_fields(self, log_record, record, message_dict): 45 | # log_record['jump_click'] = f"""File '{record.__dict__.get('pathname')}', line {record.__dict__.get('lineno')}""" 46 | log_record[f"{record.__dict__.get('pathname')}:{record.__dict__.get('lineno')}"] = '' # 加个能点击跳转的字段。 47 | log_record['ip'] = computer_ip 48 | log_record['host_name'] = computer_name 49 | super().add_fields(log_record, record, message_dict) 50 | if 'for_segmentation_color' in log_record: 51 | del log_record['for_segmentation_color'] 52 | 53 | 54 | DING_TALK_TOKEN = '3dd0eexxxxxadab014bd604XXXXXXXXXXXX' # 钉钉报警机器人 55 | 56 | EMAIL_HOST = ('smtp.sohu.com', 465) 57 | EMAIL_FROMADDR = 'aaa0509@sohu.com' # 'matafyhotel-techl@matafy.com', 58 | EMAIL_TOADDRS = ('cccc.cheng@silknets.com', 'yan@dingtalk.com',) 59 | EMAIL_CREDENTIALS = ('aaa0509@sohu.com', 'abcdefg') 60 | 61 | ELASTIC_HOST = '127.0.0.1' 62 | ELASTIC_PORT = 9200 63 | 64 | KAFKA_BOOTSTRAP_SERVERS = ['192.168.199.202:9092'] 65 | ALWAYS_ADD_KAFKA_HANDLER_IN_TEST_ENVIRONENT = False 66 | 67 | MONGO_URL = 'mongodb://myUserAdmin:mimamiama@127.0.0.1:27016/admin' 68 | 69 | DEFAULUT_USE_COLOR_HANDLER = True # 是否默认使用有彩的日志。 70 | DISPLAY_BACKGROUD_COLOR_IN_CONSOLE = True # 在控制台是否显示彩色块状的日志。为False则不使用大块的背景颜色。 71 | AUTO_PATCH_PRINT = True # 是否自动打print的猴子补丁，如果打了猴子补丁，print自动变色和可点击跳转。 72 | SHOW_PYCHARM_COLOR_SETINGS = True # 有的人很反感启动代码时候提示教你怎么优化pycahrm控制台颜色，可以把这里设置为False 73 | 74 | DEFAULT_ADD_MULTIPROCESSING_SAFE_ROATING_FILE_HANDLER = False # 是否默认同时将日志记录到记log文件记事本中。 75 | LOG_FILE_SIZE = 100 # 单位是M,每个文件的切片大小，超过多少后就自动切割 76 | LOG_FILE_BACKUP_COUNT = 3 # 对同一个日志文件，默认最多备份几个文件，超过就删除了。 77 | 78 | LOG_PATH = '/pythonlogs' # 默认的日志文件夹,如果不写明磁盘名，则是项目代码所在磁盘的根目录下的/pythonlogs 79 | # LOG_PATH = Path(__file__).absolute().parent / Path("pythonlogs") #这么配置就会自动在你项目的根目录下创建pythonlogs文件夹了并写入。 80 | if os.name == 'posix': # linux非root用户和mac用户无法操作 /pythonlogs 文件夹，没有权限，默认修改为 home/[username] 下面了。例如你的linux用户名是 xiaomin，那么默认会创建并在 /home/xiaomin/pythonlogs文件夹下写入日志文件。 81 | home_path = os.environ.get("HOME", '/') # 这个是获取linux系统的当前用户的主目录，不需要亲自设置 82 | LOG_PATH = Path(home_path) / Path('pythonlogs') # linux mac 权限很严格，非root权限不能在/pythonlogs写入，修改一下默认值。 83 | 84 | LOG_FILE_HANDLER_TYPE = 1 # 1 2 3 4 5 85 | """ 86 | LOG_FILE_HANDLER_TYPE 这个值可以设置为 1 2 3 4 5 四种值， 87 | 1为使用多进程安全按日志文件大小切割的文件日志,这是本人实现的批量写入日志，减少操作文件锁次数，测试10进程快速写入文件，win上性能比第5种提高了100倍，linux提升5倍 88 | 2为多进程安全按天自动切割的文件日志，同一个文件，每天生成一个新的日志文件。日志文件名字后缀自动加上日期。 89 | 3为不自动切割的单个文件的日志(不切割文件就不会出现所谓进程安不安全的问题) 90 | 4为 WatchedFileHandler，这个是需要在linux下才能使用，需要借助lograte外力进行日志文件的切割，多进程安全。 91 | 5 为第三方的concurrent_log_handler.ConcurrentRotatingFileHandler按日志文件大小切割的文件日志， 92 | 这个是采用了文件锁，多进程安全切割，文件锁在linux上使用fcntl性能还行，win上使用win32con性能非常惨。按大小切割建议不要选第5个个filehandler而是选择第1个。 93 | """ 94 | 95 | LOG_LEVEL_FILTER = logging.DEBUG # 默认日志级别，低于此级别的日志不记录了。例如设置为INFO，那么logger.debug的不会记录，只会记录logger.info以上级别的。 96 | 97 | RUN_ENV = 'test' 98 | 99 | FORMATTER_DICT = { 100 | 1: logging.Formatter( 101 | '日志时间【%(asctime)s】 - 日志名称【%(name)s】 - 文件【%(filename)s】 - 第【%(lineno)d】行 - 日志等级【%(levelname)s】 - 日志信息【%(message)s】', 102 | "%Y-%m-%d %H:%M:%S"), 103 | 2: logging.Formatter( 104 | '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s', 105 | "%Y-%m-%d %H:%M:%S"), 106 | 3: logging.Formatter( 107 | '%(asctime)s - %(name)s - 【 File "%(pathname)s", line %(lineno)d, in %(funcName)s 】 - %(levelname)s - %(message)s', 108 | "%Y-%m-%d %H:%M:%S"), # 一个模仿traceback异常的可跳转到打印日志地方的模板 109 | 4: logging.Formatter( 110 | '%(asctime)s - %(name)s - "%(filename)s" - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s - File "%(pathname)s", line %(lineno)d ', 111 | "%Y-%m-%d %H:%M:%S"), # 这个也支持日志跳转 112 | 5: logging.Formatter( 113 | '%(asctime)s - %(levelname)s - [%(pathname)s:%(lineno)d] - %(message)s',"%Y-%m-%d %H:%M:%S"), # 我认为的最好的模板,推荐 114 | 6: logging.Formatter('%(name)s - %(asctime)-15s - %(filename)s - %(lineno)d - %(levelname)s: %(message)s', 115 | "%Y-%m-%d %H:%M:%S"), 116 | 7: logging.Formatter('%(asctime)s - %(name)s - "%(filename)s:%(lineno)d" - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S"), # 一个只显示简短文件名和所处行数的日志模板 117 | 118 | 8: JsonFormatterJumpAble('%(asctime)s - %(name)s - %(levelname)s - %(message)s - "%(filename)s %(lineno)d -" ', "%Y-%m-%d %H:%M:%S", json_ensure_ascii=False), # 这个是json日志，方便分析. 119 | 120 | 9: logging.Formatter( 121 | '[p%(process)d_t%(thread)d] %(asctime)s - %(name)s - "%(pathname)s:%(lineno)d" - %(funcName)s - %(levelname)s - %(message)s', 122 | "%Y-%m-%d %H:%M:%S"), # 对5改进，带进程和线程显示的日志模板。 123 | 10: logging.Formatter( 124 | '[p%(process)d_t%(thread)d] %(asctime)s - %(name)s - "%(filename)s:%(lineno)d" - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S"), # 对7改进，带进程和线程显示的日志模板。 125 | 11: logging.Formatter( 126 | f'({computer_ip},{computer_name})-[p%(process)d_t%(thread)d] %(asctime)s - %(name)s - "%(filename)s:%(lineno)d" - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S"), # 对7改进，带进程和线程显示的日志模板以及ip和主机名。 127 | } 128 | 129 | FORMATTER_KIND = 5 # 如果get_logger不指定日志模板，则默认选择第几个模板 130 | -------------------------------------------------------------------------------- /run_loglength.sh: -------------------------------------------------------------------------------- 1 | # bin/bash 2 | 3 | log_array=(20 50 100 200 500 1000) # per 100ms 4 | sleep_time=5m 5 | 6 | for element in ${log_array[@]} 7 | do 8 | 9 | front="log" 10 | loglength="$front$element" 11 | 12 | str="start" 13 | print_str="$str $lognumber" 14 | echo $print_str 15 | 16 | ## python 17 | echo "start python" 18 | file="python" 19 | 20 | nohup python3 python/hello.py --lognumber=1000 --length=$loglength> ./log/benchmark.log 2>&1 & 21 | echo python3 python/hello.py --lognumber=1000 --length=$loglength 22 | sleep 5s 23 | echo "start benchmark" 24 | 25 | nohup python3 logReducer.py --program hello.py --template '2022-08-16 20:11:31 - ERROR - [python/hello.py:46]' --language python > ./log/reduer.log 2>&1 & 26 | sleep 5s 27 | echo "start log reducer" 28 | 29 | sleep $sleep_time 30 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 31 | ps -ef | grep hello.py | grep -v grep | awk '{print $2}' | xargs kill -9 32 | sleep 10s 33 | 34 | 35 | ## java 36 | echo "start java" 37 | file="java" 38 | 39 | java -jar ./java/helloLog/target/helloLog-jar-with-dependencies.jar -l $loglength -n 1000 > ./log/benchmark.log 2>&1 & 40 | echo java -jar ./java/helloLog/target/helloLog-jar-with-dependencies.jar -l $loglength -n 1000 41 | sleep 5s 42 | echo "start benchmark" 43 | 44 | nohup python3 logReducer.py --program helloLog --template '2022-08-16 19:47:26,505 ERROR com.yuxiaoba.hello.Hello [Hello.java:104]' --language python > ./log/reduer.log 2>&1 & 45 | sleep 5s 46 | echo "start log reducer" 47 | 48 | 49 | sleep $sleep_time 50 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 51 | ps -ef | grep helloLog | grep -v grep | awk '{print $2}' | xargs kill -9 52 | sleep 10s 53 | 54 | ## go 55 | echo "start go" 56 | file="go" 57 | 58 | ./golang/hello --lognumber=1000 --length=$loglength > ./log/benchmark.log 2>&1 & 59 | echo ./golang/hello --lognumber=1000 --length=$loglength 60 | sleep 5s 61 | echo "start benchmark" 62 | 63 | nohup python3 logReducer.py --program hello --template '"level":"error","ts":"2022-08-16 19:49:04.210","caller":"golang/hello.go:99"' --language golang > ./log/reduer.log 2>&1 & 64 | sleep 5s 65 | echo "start log reducer" 66 | 67 | sleep $sleep_time 68 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 69 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 70 | sleep 10s 71 | 72 | ## 73 | echo "start c++" 74 | file="c++" 75 | 76 | ./c++/hello -lognumber 1000 -length $loglength > ./log/benchmark.log 2>&1 & 77 | echo ./c++/hello -lognumber 1000 -length $loglength 78 | sleep 5s 79 | echo "start benchmark" 80 | 81 | nohup python3 logReducer.py --program hello --template '2022/08/24 08:29:03,534.244 ERROR - [hello.cpp:60]' --language python > ./log/reduer.log 2>&1 & 82 | sleep 5s 83 | echo "start log reducer" 84 | 85 | sleep $sleep_time 86 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 87 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 88 | 89 | done -------------------------------------------------------------------------------- /run_lognumber.sh: -------------------------------------------------------------------------------- 1 | # bin/bash 2 | 3 | log_array=(50 100 1000 2000 5000 10000) # per 100ms 4 | sleep_time=5m 5 | 6 | for element in ${log_array[@]} 7 | do 8 | 9 | lognumber=`expr $element \* 10` 10 | 11 | str="start" 12 | print_str="$str $lognumber" 13 | echo $print_str 14 | 15 | if (($lognumber < 20001)) 16 | then 17 | ## python 18 | echo "start python" 19 | file="python" 20 | 21 | nohup python3 python/hello.py --lognumber=$element > ./log/benchmark.log 2>&1 & 22 | sleep 5s 23 | echo "start benchmark" 24 | 25 | nohup python3 logReducer.py --program hello.py --template '2022-08-16 20:11:31 - ERROR - [python/hello.py:37]' --language python > ./log/reduer.log 2>&1 & 26 | sleep 5s 27 | echo "start log reducer" 28 | 29 | sleep $sleep_time 30 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 31 | ps -ef | grep hello.py | grep -v grep | awk '{print $2}' | xargs kill -9 32 | sleep 10s 33 | 34 | fi 35 | 36 | ## java 37 | echo "start java" 38 | file="java" 39 | 40 | java -jar ./java/helloLog/target/helloLog-jar-with-dependencies.jar -n $element > ./log/benchmark.log 2>&1 & 41 | sleep 5s 42 | echo "start benchmark" 43 | 44 | nohup python3 logReducer.py --program helloLog --template '2022-08-16 19:47:26,505 ERROR com.yuxiaoba.hello.Hello [Hello.java:90]' --language python > ./log/reduer.log 2>&1 & 45 | sleep 5s 46 | echo "start log reducer" 47 | 48 | 49 | sleep $sleep_time 50 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 51 | ps -ef | grep helloLog | grep -v grep | awk '{print $2}' | xargs kill -9 52 | sleep 10s 53 | 54 | ## go 55 | echo "start go" 56 | file="go" 57 | 58 | ./golang/hello --lognumber=$element > ./log/benchmark.log 2>&1 & 59 | sleep 5s 60 | echo "start benchmark" 61 | 62 | nohup python3 logReducer.py --program hello --template '"level":"error","ts":"2022-08-16 19:49:04.210","caller":"golang/hello.go:86"' --language golang > ./log/reduer.log 2>&1 & 63 | sleep 5s 64 | echo "start log reducer" 65 | 66 | sleep $sleep_time 67 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 68 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 69 | sleep 10s 70 | 71 | ## 72 | echo "start c++" 73 | file="c++" 74 | 75 | ./c++/hello -lognumber $element > ./log/benchmark.log 2>&1 & 76 | sleep 5s 77 | echo "start benchmark" 78 | 79 | nohup python3 logReducer.py --program hello --template '2022/08/16 19:32:16,512.177 ERROR - This is log hotspots 6 [hello.cpp:51]' --language python > ./log/reduer.log 2>&1 & 80 | sleep 5s 81 | echo "start log reducer" 82 | 83 | sleep $sleep_time 84 | ps -ef | grep logReducer.py | grep -v grep | awk '{print $2}' | xargs kill -9 85 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 86 | 87 | done -------------------------------------------------------------------------------- /run_userspace.sh: -------------------------------------------------------------------------------- 1 | # bin/bash 2 | 3 | log_array=(50 100 1000 2000 5000 10000) # per 100ms 4 | sleep_time=10m 5 | 6 | for element in ${log_array[@]} 7 | do 8 | 9 | lognumber=`expr $element \* 10` 10 | 11 | str="start" 12 | print_str="$str $lognumber" 13 | echo $print_str 14 | 15 | if (($lognumber < 20001)) 16 | then 17 | ## python 18 | echo "start python" 19 | file="python" 20 | 21 | nohup python3 python/hello.py --lognumber=$element > ./log/benchmark.log 2>&1 & 22 | echo python3 python/hello.py --lognumber=$element > ./log/benchmark.log 23 | sleep 5s 24 | echo "start benchmark" 25 | 26 | nohup python3 logReducer.py --program hello.py --template "not" --logfile '/data/home/logbench/python/python.log' --language $file --lognumber $lognumber > ./log/reducer.log 2>&1 & 27 | sleep 5s 28 | echo python3 logReducer.py --program hello.py --template "not" --logfile '/data/home/logbench/python/python.log' --language $file --lognumber $lognumber 29 | echo "start log reducer" 30 | 31 | sleep $sleep_time 32 | ps -ef | grep logReader.py | grep -v grep | awk '{print $2}' | xargs kill -9 33 | ps -ef | grep hello.py | grep -v grep | awk '{print $2}' | xargs kill -9 34 | sleep 10s 35 | 36 | fi 37 | 38 | ## java 39 | echo "start java" 40 | file="java" 41 | 42 | java -jar ./java/helloLog/target/helloLog-jar-with-dependencies.jar -l $element > ./log/benchmark.log 2>&1 & 43 | sleep 5s 44 | echo "start benchmark" 45 | 46 | nohup python3 logReducer.py --program helloLog --template "not" --logfile '/data/home/logbench/java/java.log' --language $file --lognumber $lognumber > ./log/reducer.log 2>&1 & 47 | sleep 5s 48 | echo "start log reducer" 49 | 50 | sleep $sleep_time 51 | ps -ef | grep logReader.py | grep -v grep | awk '{print $2}' | xargs kill -9 52 | ps -ef | grep helloLog | grep -v grep | awk '{print $2}' | xargs kill -9 53 | 54 | sleep 10s 55 | 56 | ## go 57 | echo "start go" 58 | file="go" 59 | 60 | ./golang/hello --lognumber=$element > ./log/benchmark.log 2>&1 & 61 | sleep 5s 62 | echo "start benchmark" 63 | 64 | nohup python3 logReducer.py --program hello --template "not" --logfile '/data/home/logbench/golang/golang.log' --language $file --lognumber $lognumber > ./log/reducer.log 2>&1 & 65 | sleep 5s 66 | echo "start log reducer" 67 | 68 | sleep $sleep_time 69 | ps -ef | grep logReader.py | grep -v grep | awk '{print $2}' | xargs kill -9 70 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 71 | sleep 10s 72 | 73 | ## 74 | echo "start c++" 75 | file="c++" 76 | 77 | ./c++/hello -lognumber $element > ./log/benchmark.log 2>&1 & 78 | sleep 5s 79 | echo "start benchmark" 80 | 81 | nohup python3 logReducer.py --program hello --template "not" --logfile '/data/home/logbench/c++/c++.log' --language $file --lognumber $lognumber > ./log/reducer.log 2>&1 & 82 | sleep 5s 83 | echo "start log reducer" 84 | 85 | sleep $sleep_time 86 | ps -ef | grep logReader.py | grep -v grep | awk '{print $2}' | xargs kill -9 87 | ps -ef | grep hello | grep -v grep | awk '{print $2}' | xargs kill -9 88 | 89 | done --------------------------------------------------------------------------------