├── .gitignore ├── .vscode └── settings.json ├── Dockerfile ├── LICENSE ├── README.md ├── app-go ├── db │ └── models.go ├── go.mod ├── go.sum ├── main.go ├── query.sql └── routes │ ├── request.go │ ├── response.go │ ├── routes.go │ └── search.go ├── app ├── .gitignore ├── main.py ├── models.py ├── query.py ├── requirements.txt ├── user.py └── util.py ├── assets ├── 1.jpeg └── 2.jpeg ├── config └── clickhouse │ └── default.xml ├── docker-compose.yml ├── install.md └── ui ├── .eslintrc.cjs ├── .gitignore ├── README.md ├── index.html ├── package.json ├── pnpm-lock.yaml ├── postcss.config.js ├── src ├── App.jsx ├── components │ ├── GridItem.tsx │ ├── TimeFilter.tsx │ ├── back.jsx │ ├── grid-old.tsx │ ├── grid.tsx │ ├── metricview.jsx │ └── search.jsx ├── index.css ├── main.jsx ├── pages │ ├── logid.tsx │ └── metrics.jsx └── util │ ├── index.js │ └── logger.ts ├── tailwind.config.js └── vite.config.js /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | logs.db 3 | logs.db.wal 4 | /venv 5 | /app/.venv 6 | 7 | # Added by cargo 8 | # 9 | # already existing elements were commented out 10 | 11 | #/target 12 | 13 | .DS_Store 14 | logs/ 15 | 16 | .venv/ -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[python]": { 3 | "editor.defaultFormatter": "ms-python.black-formatter" 4 | }, 5 | "python.formatting.provider": "none" 6 | } 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | EXPOSE 8000 4 | 5 | RUN pip install fastapi ujson structlog chdb uvicorn luqum async_tail 6 | 7 | COPY ./app/ ./app/ 8 | 9 | CMD ["python3", "-m", "uvicorn", "app.main:app"] 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Nevin Puri. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Erlog 2 | 3 | #### A Log Platform which runs on a $4 VPS 4 | 5 | ![img1](./assets/1.jpeg) 6 | 7 | ## Features 8 | 9 | - Ingest as many logs as you want from an http endpoint or file 10 | - Query logs with a nice syntax 11 | - Support for `parent_id`, so logs can have "children" 12 | 13 | ## Sending Logs 14 | 15 | Just send a POST request to erlog with JSON 16 | 17 | ``` 18 | { 19 | "timestamp": "1675955819", 20 | "level": "debug", 21 | "service": "my_service", 22 | "key": "value", 23 | "data": { 24 | "another_key": "another value" 25 | } 26 | } 27 | ``` 28 | 29 | Alternatively, run `export LOGS=file1.txt file2.txt` and those files will be tailed and ingested in to erlog. 30 | 31 | ## Viewing Logs 32 | 33 | ![img2](./assets/2.jpeg) 34 | 35 | ## Querying 36 | 37 | Erql is extremely simple. Here are some examples 38 | 39 | ```rb 40 | # search for "foo" in any log (case sensitive) 41 | foo 42 | "foo" 43 | 44 | # search where name = "foo" or name = "foo bar" 45 | name:foo 46 | name:"foo bar" 47 | 48 | # search for {name: {first: 'foo'}} 49 | name.first:foo 50 | name.first:"foo bar" 51 | 52 | # search for name[0] = "item" 53 | name.0:item 54 | 55 | # member = true 56 | member:true 57 | member:false 58 | 59 | # null search 60 | member:null 61 | 62 | # search for height =, >, >=, <, <= 63 | height:=100 64 | height:>100 65 | height:>=100 66 | height:<100 67 | height:<=100 68 | 69 | # search for height in range (inclusive, exclusive) 70 | height:[100 TO 200] 71 | height:{100 TO 200} 72 | 73 | # AND/OR operators 74 | name:foo AND height:=100 75 | name:foo OR name:bar 76 | 77 | # grouping 78 | name:foo AND (bio:bar OR bio:baz) 79 | ``` 80 | 81 | ## Instrumenting 82 | 83 | Just have your structured logger log in json, and then forward those logs to a file. A collector is coming soon which just `tail`s a file and forwards the requests to your erlog instance. 84 | 85 | Look here for an example of an instrumented function. 86 | 87 | ## Example Project 88 | 89 | For an example project, see app/main.py. Erlog is instrumented using structlog. 90 | If you run `export LOGS=file1.txt` and `python3 -m uvicorn main:app > file1.txt`, 91 | you will start seeing erlogs logs in erlog. 92 | 93 | ## Using `parent_id` 94 | 95 | ```python 96 | import uuid 97 | from structlog import get_logger 98 | 99 | logger = get_logger() 100 | 101 | # don't forget str() 102 | # making the id a Uuid type won't work 103 | # as it'll be printed as Uuid('iaodjfoiasjdijdsaiojf') 104 | # instead of 'iaodjfoiasjdijdsaiojf' 105 | 106 | id = str(uuid.uuid4()) 107 | logger.log("root log", id=id) 108 | logger.log("child of root", parent_id=id) 109 | ``` 110 | 111 | This will show the log "root log" in the erlog ui. 112 | Once you click on "root log", you will be able to see "child of root" 113 | in the log viewer. 114 | 115 | This is useful for when you want to capture logs across many different services, and have them all be in one location. 116 | 117 | ## Todo 118 | 119 | - clear db on 'reset_erlog' message (for development) 120 | - stream logs from db 121 | - get support for traces using `parentId` and `duration` or `start` `end` in ms 122 | - show parents whenever you click on a child 123 | - on log submit, if log level is error, then with id == parent_id field to be error=true 124 | - same thing with warning 125 | - add toggle for parent id null or not 126 | 127 | - for each log, make a call to the db with the info to try and merge the events together 128 | - or just do that on every other log call where whenver there's a new info you 129 | merge it into the current log, and an error you merge into the corresponding error log 130 | 131 | The idea is that if they both have the same parent id, do the merge 132 | 133 | - start reporting logs from the ui, set up service_name for both 134 | 135 | ##### installation packages 136 | 137 | - install openssl 138 | - install swig 139 | 140 | to install m2crypto on mac 141 | 142 | ``` 143 | env LDFLAGS="-L$(brew --prefix openssl)/lib" \ 144 | CFLAGS="-I$(brew --prefix openssl)/include" \ 145 | SWIG_FEATURES="-cpperraswarn -includeall -I$(brew --prefix openssl)/include" \ 146 | pip3 install m2crypto 147 | ``` 148 | 149 | We get it, monitoring your applications in production is difficult. Most logging platforms are designed for large applications where people need to \_\_. 150 | 151 | However, that's not you. You're just building a small project and want to view your logs from your next js api routes, formatted, in one place, without configuring any additional collector. Erlog is really great at doing that. 152 | 153 | ## Most logging platforms are built for enterprise 154 | 155 | ## Most logging platforms aren't built for people like you 156 | 157 | (aren't built for people like you) 158 | 159 | They have every single use case, a convoluted ui, and follow a standard which is too complicated 160 | 161 | ours: 162 | 163 | - simple, one menu 164 | - zero configuration 165 | - works with your code 166 | 167 | - you dont need to worry about infastructure 168 | - you pay per month 169 | 170 | ## Make a filter be logs with children updated recently 171 | 172 | ## Alow filtering to be done on title + messsage 173 | 174 | ## Filtering will be main way we can differentiate from discord, also having this extra data from the client libraries 175 | 176 | - keep the client libraries, say ok put this here, and after you send in, you can send MORE data (not just message titel simple shit etc but full json, and then filter on that, so you can have really advanced filters and charts from this thing as oppoesd to discord/slack) 177 | 178 | ## 10X experience 179 | 180 | currently there isn't that much, but mainly for users the killer pain point will be the metrics 181 | 182 | 183 | ## Installing on mac 184 | CFLAGS="-I$(brew --prefix openssl)/include" \ 185 | SWIG_FEATURES="-cpperraswarn -includeall -I$(brew --prefix openssl)/include" \ -------------------------------------------------------------------------------- /app-go/db/models.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "os" 8 | "time" 9 | 10 | "github.com/ClickHouse/clickhouse-go/v2" 11 | "github.com/ClickHouse/clickhouse-go/v2/lib/driver" 12 | ) 13 | 14 | var Conn driver.Conn 15 | 16 | func ConnectDB() { 17 | dialCount := 0 18 | conn, err := clickhouse.Open(&clickhouse.Options{ 19 | Addr: []string{"127.0.0.1:19000"}, 20 | Auth: clickhouse.Auth{ 21 | Database: "default", 22 | Username: "default", 23 | Password: "test123", 24 | }, 25 | DialContext: func(ctx context.Context, addr string) (net.Conn, error) { 26 | dialCount++ 27 | var d net.Dialer 28 | return d.DialContext(ctx, "tcp", addr) 29 | }, 30 | Debug: true, 31 | Debugf: func(format string, v ...any) { 32 | fmt.Printf(format+"\n", v...) 33 | }, 34 | Settings: clickhouse.Settings{ 35 | "max_execution_time": 60, 36 | }, 37 | Compression: &clickhouse.Compression{ 38 | Method: clickhouse.CompressionLZ4, 39 | }, 40 | DialTimeout: time.Second * 30, 41 | MaxOpenConns: 5, 42 | MaxIdleConns: 5, 43 | ConnMaxLifetime: time.Duration(10) * time.Minute, 44 | ConnOpenStrategy: clickhouse.ConnOpenInOrder, 45 | BlockBufferSize: 10, 46 | MaxCompressionBuffer: 10240, 47 | ClientInfo: clickhouse.ClientInfo{ // optional, please see Client info section in the README.md 48 | Products: []struct { 49 | Name string 50 | Version string 51 | }{ 52 | {Name: "my-app", Version: "0.1"}, 53 | }, 54 | }, 55 | }) 56 | 57 | if err != nil { 58 | fmt.Printf("%v\n", err.Error()) 59 | os.Exit(0) 60 | } 61 | 62 | conn.Ping(context.Background()) 63 | 64 | // so will basically be 65 | // metrics: 66 | // id, name, timestamp 67 | // report("user signed up") 68 | 69 | fmt.Println("Creating Metrics table") 70 | err = conn.Exec(context.Background(), "CREATE TABLE IF NOT EXISTS metrics (id UUID, name String, timestamp DateTime) Engine = MergeTree PRIMARY KEY (id, timestamp, name) ORDER BY (id, timestamp, name);") 71 | if err != nil { 72 | fmt.Printf("%v\n", err.Error()) 73 | os.Exit(0) 74 | } 75 | fmt.Println("Done creating Metrics Table") 76 | 77 | Conn = conn 78 | } -------------------------------------------------------------------------------- /app-go/go.mod: -------------------------------------------------------------------------------- 1 | module erlog 2 | 3 | go 1.22.1 4 | 5 | require ( 6 | github.com/ClickHouse/ch-go v0.61.5 // indirect 7 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2 // indirect 8 | github.com/andybalholm/brotli v1.1.0 // indirect 9 | github.com/bytedance/sonic v1.11.3 // indirect 10 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect 11 | github.com/chenzhuoyu/iasm v0.9.1 // indirect 12 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 13 | github.com/gin-contrib/cors v1.7.0 // indirect 14 | github.com/gin-contrib/sse v0.1.0 // indirect 15 | github.com/gin-gonic/gin v1.9.1 // indirect 16 | github.com/go-chi/chi/v5 v5.0.12 // indirect 17 | github.com/go-faster/city v1.0.1 // indirect 18 | github.com/go-faster/errors v0.7.1 // indirect 19 | github.com/go-playground/locales v0.14.1 // indirect 20 | github.com/go-playground/universal-translator v0.18.1 // indirect 21 | github.com/go-playground/validator/v10 v10.19.0 // indirect 22 | github.com/goccy/go-json v0.10.2 // indirect 23 | github.com/google/uuid v1.6.0 // indirect 24 | github.com/json-iterator/go v1.1.12 // indirect 25 | github.com/klauspost/compress v1.17.7 // indirect 26 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 27 | github.com/leodido/go-urn v1.4.0 // indirect 28 | github.com/mattn/go-isatty v0.0.20 // indirect 29 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 30 | github.com/modern-go/reflect2 v1.0.2 // indirect 31 | github.com/paulmach/orb v0.11.1 // indirect 32 | github.com/pelletier/go-toml/v2 v2.2.0 // indirect 33 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 34 | github.com/pkg/errors v0.9.1 // indirect 35 | github.com/segmentio/asm v1.2.0 // indirect 36 | github.com/shopspring/decimal v1.3.1 // indirect 37 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 38 | github.com/ugorji/go/codec v1.2.12 // indirect 39 | go.opentelemetry.io/otel v1.24.0 // indirect 40 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 41 | golang.org/x/arch v0.7.0 // indirect 42 | golang.org/x/crypto v0.21.0 // indirect 43 | golang.org/x/net v0.22.0 // indirect 44 | golang.org/x/sys v0.18.0 // indirect 45 | golang.org/x/text v0.14.0 // indirect 46 | google.golang.org/protobuf v1.33.0 // indirect 47 | gopkg.in/yaml.v3 v3.0.1 // indirect 48 | ) 49 | -------------------------------------------------------------------------------- /app-go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= 2 | github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= 3 | github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= 4 | github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= 5 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc= 6 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE= 7 | github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= 8 | github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= 9 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= 10 | github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= 11 | github.com/bytedance/sonic v1.11.3 h1:jRN+yEjakWh8aK5FzrciUHG8OFXK+4/KrAX/ysEtHAA= 12 | github.com/bytedance/sonic v1.11.3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= 13 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= 14 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= 15 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= 16 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= 17 | github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= 18 | github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= 19 | github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= 20 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= 23 | github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= 24 | github.com/gin-contrib/cors v1.7.0 h1:wZX2wuZ0o7rV2/1i7gb4Jn+gW7HBqaP91fizJkBUJOA= 25 | github.com/gin-contrib/cors v1.7.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= 26 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 27 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 28 | github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= 29 | github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= 30 | github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= 31 | github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= 32 | github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= 33 | github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= 34 | github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= 35 | github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= 36 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 37 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 38 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 39 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 40 | github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= 41 | github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= 42 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= 43 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 44 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 45 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 46 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 47 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 48 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 49 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 50 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 51 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 52 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 53 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 54 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 55 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 56 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 57 | github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= 58 | github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 59 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 60 | github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= 61 | github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 62 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 63 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 64 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 65 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 66 | github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= 67 | github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= 68 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 69 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 70 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 71 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 72 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 73 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 74 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 75 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 76 | github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= 77 | github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= 78 | github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= 79 | github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= 80 | github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= 81 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 82 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 83 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 84 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 85 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 86 | github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= 87 | github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= 88 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= 89 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= 90 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 91 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 92 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 93 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 94 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 95 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 96 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 97 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 98 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 99 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 100 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 101 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 102 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 103 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 104 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 105 | github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= 106 | github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= 107 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 108 | github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= 109 | github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= 110 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 111 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 112 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 113 | go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= 114 | go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= 115 | go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= 116 | go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= 117 | go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= 118 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 119 | golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= 120 | golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= 121 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 122 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 123 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 124 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 125 | golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= 126 | golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= 127 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 128 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 129 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 130 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 131 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 132 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 133 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 134 | golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= 135 | golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 136 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 137 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 138 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 139 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 141 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 142 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 143 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 144 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 145 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 146 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 147 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 148 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= 149 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 150 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 151 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 152 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 153 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 154 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 155 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 156 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 157 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 158 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 159 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 160 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 161 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 162 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 163 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 164 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 165 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 166 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 167 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 168 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 169 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 170 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 171 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 172 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 173 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 174 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 175 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 176 | -------------------------------------------------------------------------------- /app-go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "erlog/db" 5 | "erlog/routes" 6 | "net/http" 7 | 8 | "github.com/gin-contrib/cors" 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | func main() { 13 | db.ConnectDB() 14 | r := gin.Default() 15 | r.Use(cors.Default()) 16 | 17 | r.GET("/", func (c *gin.Context) { 18 | c.JSON(200, gin.H{ 19 | "message": "pong", 20 | }) 21 | }) 22 | 23 | r.POST("/report", routes.Report) 24 | r.POST("/search", routes.Search) 25 | 26 | http.ListenAndServe(":8000", r) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /app-go/query.sql: -------------------------------------------------------------------------------- 1 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfYear(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute; 2 | 3 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfYear(timestamp) as date, toHour(timestamp) as hour, COUNT(*) as count from metrics GROUP BY hour, date, month, year ORDER BY year, month, date, hour; 4 | 5 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute; -------------------------------------------------------------------------------- /app-go/routes/request.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | type ReportRequestBody struct { 4 | Name string `json:"name"` 5 | } 6 | 7 | type SearchRequestBody struct { 8 | Per string `json:"per"` 9 | } 10 | -------------------------------------------------------------------------------- /app-go/routes/response.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | type SearchResponse struct { 4 | DateTime string `json:"dateTime"` 5 | Count uint64 `json:"count"` 6 | } -------------------------------------------------------------------------------- /app-go/routes/routes.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "context" 5 | "erlog/db" 6 | "fmt" 7 | 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | func Search(c *gin.Context) { 12 | var searchQuery SearchRequestBody 13 | err := c.BindJSON(&searchQuery) 14 | 15 | if err != nil { 16 | fmt.Printf("%v\n", err.Error()) 17 | c.JSON(400, gin.H{ 18 | "error": err.Error(), 19 | }) 20 | return 21 | } 22 | 23 | data, err := ExecSearch(searchQuery.Per, "asdf") 24 | 25 | if err != nil { 26 | fmt.Printf("%v\n", err.Error()) 27 | c.JSON(400, gin.H{ 28 | "error": err.Error(), 29 | }) 30 | return 31 | } 32 | 33 | c.JSON(200, data) 34 | } 35 | 36 | func Report(c *gin.Context) { 37 | var body ReportRequestBody 38 | err := c.BindJSON(&body) 39 | 40 | if err != nil { 41 | fmt.Printf("%v\n", err.Error()) 42 | c.JSON(400, gin.H{ 43 | "error": err.Error(), 44 | }) 45 | return 46 | } 47 | 48 | db.Conn.Exec(context.Background(), "INSERT INTO metrics VALUES (generateUUIDv4(), ?, now('Africa/Abidjan'))", body.Name) 49 | c.JSON(200, gin.H{"status": "ok"}) 50 | } -------------------------------------------------------------------------------- /app-go/routes/search.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "context" 5 | "erlog/db" 6 | "fmt" 7 | 8 | "github.com/ClickHouse/clickhouse-go/v2/lib/driver" 9 | ) 10 | 11 | 12 | var perHour = "select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute;" 13 | var perDay = "select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, COUNT(*) as count from metrics GROUP BY date, month, year, hour ORDER BY year, month, date, hour;" 14 | 15 | func ExecSearch(per string, name string) ([]SearchResponse, error) { 16 | var query string 17 | if per == "hour" { 18 | query = perHour 19 | } else if per == "day" { 20 | query = perDay 21 | } else { 22 | fmt.Printf("Not implemented\n") 23 | return nil, fmt.Errorf("not implemented query") 24 | } 25 | 26 | result, err := db.Conn.Query(context.Background(), query) 27 | 28 | if err != nil { 29 | fmt.Printf("%v\n", err.Error()) 30 | return nil, err 31 | } 32 | 33 | data, err := GetDataFor(per, result) 34 | 35 | if err != nil { 36 | fmt.Printf("%v\n", err.Error()) 37 | return nil, err 38 | } 39 | 40 | return data, nil 41 | } 42 | 43 | // gets the data 44 | func GetDataFor(per string, rows driver.Rows) ([]SearchResponse, error) { 45 | var data []SearchResponse 46 | var err error 47 | if per == "hour" { 48 | data, err = GetDataPerHour(rows, data) 49 | } else if per == "day" { 50 | data, err = GetDataPerDay(rows, data) 51 | } else { 52 | return nil, fmt.Errorf("not implemented") 53 | } 54 | 55 | if err != nil { 56 | fmt.Printf("%v\n", err) 57 | return nil, err 58 | } 59 | 60 | return data, nil 61 | } 62 | 63 | // returns data in per minute intervals 64 | func GetDataPerHour(rows driver.Rows, data []SearchResponse) ([]SearchResponse, error) { 65 | for rows.Next() { 66 | var year uint16 67 | var month uint8 68 | var date uint8 69 | var hour uint8 70 | var minute uint8 71 | var count uint64 72 | 73 | if err := rows.Scan(&year, &month, &date, &hour, &minute, &count); err != nil { 74 | fmt.Printf("%v\n", err.Error()) 75 | return nil, err 76 | } 77 | 78 | concatted := fmt.Sprint(date) + " " + fmt.Sprint(hour) + ":" + fmt.Sprint(minute) 79 | fmt.Printf("%v:%v\n", concatted, count) 80 | // data[concatted] = count 81 | data = append(data, SearchResponse{ 82 | DateTime: concatted, 83 | Count: count, 84 | }) 85 | } 86 | 87 | return data, nil 88 | } 89 | 90 | // returns data in per hour intervals 91 | func GetDataPerDay(rows driver.Rows, data []SearchResponse) ([]SearchResponse, error) { 92 | for rows.Next() { 93 | var year uint16 94 | var month uint8 95 | var date uint8 96 | var hour uint8 97 | var count uint64 98 | 99 | if err := rows.Scan(&year, &month, &date, &hour, &count); err != nil { 100 | fmt.Printf("%v\n", err.Error()) 101 | return nil, err 102 | } 103 | 104 | concatted := fmt.Sprint(year) + "-" + fmt.Sprint(month) + "-" + fmt.Sprint(date) + " " + fmt.Sprint(hour) 105 | fmt.Printf("%v:%v\n", concatted, count) 106 | data = append(data, SearchResponse{ 107 | DateTime: concatted, 108 | Count: count, 109 | }) 110 | } 111 | 112 | return data, nil 113 | } -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | file1.txt 3 | file2.txt -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, HTTPException, Depends 2 | from fastapi import Request, HTTPException 3 | from fastapi.middleware.cors import CORSMiddleware 4 | from app.util import flatten 5 | from app.models import ErLog, User, get_current_user 6 | from app.query import QBuilder 7 | import structlog 8 | from structlog import get_logger 9 | import ujson 10 | import uuid 11 | import secrets 12 | from typing import Annotated 13 | from fastapi.security import OAuth2PasswordBearer 14 | from clickhouse_driver import Client 15 | 16 | api_keys = ["ek-QldMOqfEWSpG_u6VCJv3ng_OD97OiXPDh5Luqvc"] 17 | 18 | # todo: use flask_httpauth 19 | # https://stackoverflow.com/questions/817882/unique-session-id-in-python/6092448#6092448 20 | # https://medium.com/@anubabajide/rest-api-authentication-in-flask-481518a7479b 21 | # instead of token generation use this qhwer u query the db 22 | 23 | structlog.configure(processors=[structlog.processors.JSONRenderer()]) 24 | 25 | 26 | logger = get_logger() 27 | 28 | def insert_log(log): 29 | logger = get_logger() 30 | id = str(uuid.uuid4()) 31 | if log == "": 32 | return False 33 | 34 | try: 35 | l = ujson.loads(log) 36 | except Exception as e: 37 | logger.error("Failed parsing log json", parent_id=id, e=str(e)) 38 | return False 39 | 40 | try: 41 | flattened = flatten(l) 42 | except Exception as e: 43 | logger.error("Failed flattening json", parent_id=id, e=str(e)) 44 | return False 45 | 46 | erlog = ErLog(log) 47 | erlog.parse_log(flattened) 48 | 49 | if erlog._id == None: 50 | erlog._id = str(uuid.uuid4()) 51 | 52 | if erlog._parent_id == None: 53 | erlog._parent_id = str("00000000-0000-0000-0000-000000000000") 54 | else: 55 | print("updating child logs") 56 | client.execute( 57 | "ALTER TABLE erlogs UPDATE child_logs = child_logs + 1 WHERE id = %(parent_id)s", 58 | {"parent_id": str(erlog._parent_id)}, 59 | ) 60 | 61 | client.execute( 62 | "INSERT INTO erlogs VALUES", 63 | [ 64 | [ 65 | str(erlog._id), 66 | str(erlog._parent_id), 67 | erlog._timestamp, 68 | erlog._string_keys, 69 | erlog._string_values, 70 | erlog._bool_keys, 71 | erlog._bool_values, 72 | erlog._number_keys, 73 | erlog._number_values, 74 | erlog._raw_log, 75 | erlog._child_logs, 76 | ] 77 | ], 78 | ) 79 | 80 | return erlog._id 81 | 82 | 83 | client = Client(host="localhost", password="test123") 84 | 85 | print("Creating tables..") 86 | # client.execute("CREATE DATABASE IF NOT EXISTS e ENGINE = Atomic;") 87 | res = client.execute( 88 | "CREATE TABLE IF NOT EXISTS erlogs (id UUID primary key, parent_id UUID, timestamp DOUBLE, string_keys Array(String), string_values Array(String), bool_keys Array(String), bool_values Array(Boolean), number_keys Array(String), number_values Array(Double), raw_log String, child_logs UInt32) Engine = MergeTree;" 89 | ) 90 | client.execute( 91 | "CREATE TABLE IF NOT EXISTS users (id UUID primary key, email String, hashed_password String) Engine = MergeTree;" 92 | ) 93 | 94 | client.execute( 95 | "CREATE TABLE IF NOT EXISTS sessions (id UUID primary key, session_id String, user_id Int32) Engine = MergeTree;" 96 | ) 97 | 98 | print("Finished creating tables") 99 | 100 | fake_users_db = { 101 | "johndoe": { 102 | "username": "johndoe", 103 | "full_name": "John Doe", 104 | "email": "johndoe@example.com", 105 | "hashed_password": "fakehashedsecret", 106 | "disabled": False, 107 | }, 108 | "alice": { 109 | "username": "alice", 110 | "full_name": "Alice Wonderson", 111 | "email": "alice@example.com", 112 | "hashed_password": "fakehashedsecret2", 113 | "disabled": True, 114 | }, 115 | } 116 | 117 | 118 | def hash_password(password: str): 119 | return "hashed" + password 120 | 121 | 122 | def get_user(session_id: str): 123 | user_ids = client.execute( 124 | "SELECT user_id FROM sessions WHERE session_id = %(session_id)s;", 125 | {"session_id": session_id}, 126 | ) 127 | print(user_ids) 128 | pass 129 | 130 | 131 | app = FastAPI() 132 | 133 | oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") 134 | 135 | 136 | @app.get("/users/me") 137 | async def read_me(current_user: Annotated[User, Depends(get_current_user)]): 138 | return current_user 139 | 140 | 141 | @app.get("/items/") 142 | async def items(token: Annotated[str, Depends(oauth2_scheme)]): 143 | return {"token": token} 144 | 145 | 146 | # @app.on_event("startup") 147 | # async def read_logs(): 148 | # if not "LOGS" in os.environ: 149 | # logger = get_logger() 150 | # logger.info("No logs in os.environ", l=len(os.environ)) 151 | # return 152 | 153 | # loop = asyncio.get_event_loop() 154 | # loop.create_task(read_from_file()) 155 | 156 | 157 | origins = [ 158 | "http://localhost:5173", 159 | "http://localhost:3000", 160 | "http://localhost", 161 | "http://localhost:59971", 162 | ] 163 | 164 | app.add_middleware( 165 | CORSMiddleware, 166 | allow_origins=origins, 167 | allow_credentials=True, 168 | allow_methods=["*"], 169 | allow_headers=["*"], 170 | ) 171 | 172 | 173 | @app.post("/api_key") 174 | async def gen_api_key(request: Request): 175 | key = "ek-" + secrets.token_urlsafe(29) 176 | api_keys.append(key) 177 | return key 178 | 179 | 180 | @app.post("/metrics") 181 | async def metrics(request: Request): 182 | return "ok!" 183 | 184 | 185 | @app.post("/test") 186 | async def test_log(request: Request): 187 | logger = get_logger() 188 | logger.info("hello!") 189 | return "ok" 190 | 191 | 192 | @app.post("/search") 193 | async def root(request: Request): 194 | try: 195 | body = await request.json() 196 | if not isinstance(body, object) or isinstance(body, str): 197 | logger.error("Invalid json", error_details=str(body)) 198 | raise HTTPException(status_code=400, detail="Invalid json") 199 | 200 | user_query = body.get("query", "") 201 | page = body.get("page", 0) 202 | show_children = body.get("showChildren", False) 203 | time_range = body.get("timeRange", "all") 204 | 205 | # Convert show_children string to boolean if needed 206 | if isinstance(show_children, str): 207 | show_children = show_children.lower() == "true" 208 | 209 | try: 210 | p = int(page) 211 | except Exception: 212 | logger.error("Invalid page number", page_value=str(page)) 213 | raise HTTPException(status_code=400, detail="Page is invalid") 214 | 215 | logger.info("Building query", 216 | user_query=str(user_query), 217 | page=str(p), 218 | show_children=str(show_children), 219 | time_range=str(time_range)) 220 | 221 | q = QBuilder() 222 | q.parse(user_query, p, show_children, time_range) 223 | query, params = q.query, q.params 224 | 225 | logger.info("Executing query", query=str(query)) 226 | l = client.execute(query, params) 227 | 228 | logs = [ 229 | {"id": log[0], "timestamp": log[1], "log": log[2], "child_logs": log[3]} 230 | for log in l 231 | ] 232 | 233 | return logs 234 | 235 | except Exception as e: 236 | logger.error("Search error", error_message=str(e)) 237 | raise HTTPException(status_code=500, detail=str(e)) 238 | 239 | 240 | @app.post("/get") 241 | async def get_log(request: Request): 242 | body = await request.json() 243 | 244 | if isinstance(body, str): 245 | raise HTTPException(status_code=400, detail="Invalid json") 246 | 247 | id = body["id"] 248 | 249 | if id is None: 250 | raise HTTPException(status_code=400, detail="Invalid json") 251 | 252 | h = client.execute( 253 | "SELECT id, parent_id, timestamp, raw_log from erlogs WHERE id = %(s)s", 254 | {"s": id}, 255 | ) 256 | 257 | # TODO: check if less than one 258 | log = h[0] 259 | 260 | # print(log[]) 261 | # if log[1] != None: 262 | c = client.execute( 263 | "SELECT id, parent_id, timestamp, raw_log from erlogs WHERE parent_id = %(s)s ORDER BY timestamp ASC", 264 | {"s": id}, 265 | ) 266 | 267 | children = [] 268 | clogs = c 269 | for c in clogs: 270 | children.append({"id": c[0], "parent_id": c[1], "timestamp": c[2], "log": c[3]}) 271 | # print("hi") 272 | # print(len(child)) 273 | 274 | return {"id": log[0], "timestamp": log[2], "log": log[3], "children": children} 275 | 276 | 277 | @app.post("/") 278 | async def log(request: Request): 279 | # if not "Authorization" in request.headers: 280 | # raise HTTPException(401, "Unauthorized") 281 | 282 | # key = request.headers["Authorization"] 283 | # key = key.replace("Bearer ", "") 284 | 285 | # if not key in api_keys: 286 | # raise HTTPException(401, "Unauthorized") 287 | 288 | body = await request.json() 289 | s = ujson.dumps(body) 290 | status = insert_log(s) 291 | 292 | # if isinstance(body, str): 293 | # raise HTTPException(status_code=400, detail="Invalid json") 294 | 295 | # flattened = flatten(body) 296 | # erlog.parse_log(flattened) 297 | 298 | # # todo, use appender or add tis to a batch 299 | # if erlog._id == None: 300 | # erlog._id = uuid.uuid4() 301 | 302 | # conn.execute( 303 | # "INSERT INTO erlogs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 304 | # [ 305 | # erlog._id, 306 | # erlog._timestamp, 307 | # erlog._parent_id, 308 | # erlog._string_keys, 309 | # erlog._string_values, 310 | # erlog._bool_keys, 311 | # erlog._bool_values, 312 | # erlog._number_keys, 313 | # erlog._number_values, 314 | # erlog._raw_log, 315 | # ], 316 | # ) 317 | return {"status": status} 318 | 319 | 320 | @app.get("/children/{log_id}") 321 | async def get_children(log_id: str): 322 | children = client.execute( 323 | "SELECT id, timestamp, raw_log, child_logs FROM erlogs WHERE parent_id = %(parent_id)s ORDER BY timestamp ASC", 324 | {"parent_id": log_id} 325 | ) 326 | 327 | return [ 328 | { 329 | "id": child[0], 330 | "timestamp": child[1], 331 | "log": child[2], 332 | "child_logs": child[3] 333 | } 334 | for child in children 335 | ] 336 | 337 | 338 | @app.get("/preview/{log_id}") 339 | async def get_log_preview(log_id: str): 340 | log = client.execute( 341 | """ 342 | SELECT id, timestamp, raw_log, child_logs 343 | FROM erlogs 344 | WHERE id = %(log_id)s 345 | LIMIT 1 346 | """, 347 | {"log_id": log_id} 348 | ) 349 | 350 | if not log: 351 | raise HTTPException(status_code=404, detail="Log not found") 352 | 353 | return { 354 | "id": log[0][0], 355 | "timestamp": log[0][1], 356 | "log": log[0][2], 357 | "child_logs": log[0][3] 358 | } 359 | 360 | 361 | if __name__ == "__main__": 362 | import uvicorn 363 | 364 | uvicorn.run(app) 365 | -------------------------------------------------------------------------------- /app/models.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timezone 2 | from pydantic import BaseModel 3 | import uuid 4 | 5 | 6 | class User(BaseModel): 7 | username: str 8 | email: str | None = None 9 | 10 | 11 | class DBUser(User): 12 | hashed_password: str 13 | 14 | 15 | def decode_token(token): 16 | return User(username=token + "fakecoded", email="hi@example.com") 17 | 18 | 19 | def get_current_user(token): 20 | user = decode_token(token) 21 | return user 22 | 23 | 24 | class ErLog: 25 | def __init__(self, raw_log): 26 | self._id = None 27 | self._timestamp = 0.00 28 | 29 | self._string_keys = [] 30 | self._string_values = [] 31 | 32 | self._number_keys = [] 33 | self._number_values = [] 34 | 35 | self._bool_keys = [] 36 | self._bool_values = [] 37 | 38 | self._raw_log = raw_log 39 | self._parent_id = None 40 | self._child_logs = 0 41 | 42 | def parse_log(self, log): 43 | for k, v in log.items(): 44 | if k == "timestamp": 45 | self._timestamp = float(v) 46 | continue 47 | 48 | if k == "parentId" or k == "parent_id": 49 | self._parent_id = str(v) 50 | continue 51 | 52 | if k == "id": 53 | # uuid parse 54 | self._id = str(v) 55 | continue 56 | 57 | if isinstance(v, str): 58 | self._string_keys.append(k) 59 | self._string_values.append(v) 60 | 61 | elif ( 62 | isinstance(v, int) 63 | and not isinstance(v, bool) 64 | or isinstance(v, float) 65 | and not isinstance(v, bool) 66 | ): 67 | self._number_keys.append(k) 68 | self._number_values.append(v) 69 | 70 | elif isinstance(v, bool): 71 | self._bool_keys.append(k) 72 | self._bool_values.append(v) 73 | 74 | if self._timestamp == 0.00: 75 | self._timestamp = datetime.now().timestamp() 76 | -------------------------------------------------------------------------------- /app/query.py: -------------------------------------------------------------------------------- 1 | from luqum.parser import parser 2 | from luqum.tree import ( 3 | SearchField, 4 | Word, 5 | Phrase, 6 | From, 7 | To, 8 | AndOperation, 9 | OrOperation, 10 | Group, 11 | ) 12 | import uuid 13 | import time 14 | import structlog 15 | 16 | logger = structlog.get_logger() 17 | 18 | class QBuilder: 19 | def __init__(self): 20 | self.query = "SELECT id, timestamp, raw_log, child_logs FROM erlogs" 21 | self.params = {} 22 | self.where_conditions = [] 23 | 24 | def parse(self, user_query, page, show_children, time_range="all"): 25 | try: 26 | # Add time filter 27 | time_filter = self.build_time_filter(time_range) 28 | if time_filter: 29 | self.where_conditions.append(time_filter) 30 | 31 | # Add parent filter 32 | if not show_children: 33 | self.where_conditions.append("parent_id = '00000000-0000-0000-0000-000000000000'") 34 | 35 | # Parse user query if present 36 | if user_query: 37 | try: 38 | tree = parser.parse(user_query) 39 | condition = self.parse_node(tree) 40 | if condition: 41 | self.where_conditions.append(condition) 42 | except Exception as e: 43 | logger.error("Query parse error", error_message=str(e)) 44 | 45 | # Combine WHERE conditions 46 | if self.where_conditions: 47 | self.query += " WHERE " + " AND ".join(self.where_conditions) 48 | 49 | # Add ordering and pagination 50 | self.query += " ORDER BY timestamp DESC" 51 | page_param = str(uuid.uuid4()) 52 | self.query += f" LIMIT 50 OFFSET %({page_param})s" 53 | self.params[page_param] = int(page * 50) 54 | 55 | return self.query, self.params 56 | 57 | except Exception as e: 58 | logger.error("Query build error", error_message=str(e)) 59 | raise 60 | 61 | def parse_node(self, node): 62 | if isinstance(node, AndOperation): 63 | left = self.parse_node(node.children[0]) 64 | right = self.parse_node(node.children[1]) 65 | return f"({left} AND {right})" 66 | 67 | if isinstance(node, OrOperation): 68 | left = self.parse_node(node.children[0]) 69 | right = self.parse_node(node.children[1]) 70 | return f"({left} OR {right})" 71 | 72 | if isinstance(node, Group): 73 | return self.parse_node(node.children[0]) 74 | 75 | if isinstance(node, SearchField): 76 | return self.parse_field(node) 77 | 78 | if isinstance(node, Word): 79 | return self.parse_word(node.value) 80 | 81 | if isinstance(node, Phrase): 82 | return self.parse_word(node.value.strip('"\'')) 83 | 84 | return None 85 | 86 | def parse_field(self, node): 87 | field = node.name 88 | value = node.children[0].value.strip('"\'') 89 | 90 | # Handle special fields 91 | if field in ['id', 'parent_id', 'timestamp']: 92 | param_id = str(uuid.uuid4()) 93 | self.params[param_id] = value 94 | return f"{field} = %({param_id})s" 95 | 96 | # Handle array fields 97 | param_id = str(uuid.uuid4()) 98 | self.params[param_id] = field 99 | value_param = str(uuid.uuid4()) 100 | self.params[value_param] = value 101 | 102 | return f""" 103 | (arrayExists(x -> x = %({param_id})s, string_keys) AND 104 | arrayFirst(i -> string_keys[i] = %({param_id})s, arrayEnumerate(string_keys)) > 0 AND 105 | string_values[arrayFirst(i -> string_keys[i] = %({param_id})s, arrayEnumerate(string_keys))] = %({value_param})s) 106 | OR 107 | (arrayExists(x -> x = %({param_id})s, number_keys) AND 108 | arrayFirst(i -> number_keys[i] = %({param_id})s, arrayEnumerate(number_keys)) > 0 AND 109 | toString(number_values[arrayFirst(i -> number_keys[i] = %({param_id})s, arrayEnumerate(number_keys))]) = %({value_param})s) 110 | OR 111 | (arrayExists(x -> x = %({param_id})s, bool_keys) AND 112 | arrayFirst(i -> bool_keys[i] = %({param_id})s, arrayEnumerate(bool_keys)) > 0 AND 113 | toString(bool_values[arrayFirst(i -> bool_keys[i] = %({param_id})s, arrayEnumerate(bool_keys))]) = %({value_param})s) 114 | """ 115 | 116 | def parse_word(self, value): 117 | param_id = str(uuid.uuid4()) 118 | self.params[param_id] = f"%{value}%" 119 | 120 | return f""" 121 | arrayExists(x -> position(lower(x), lower(%({param_id})s)) > 0, string_values) OR 122 | arrayExists(x -> position(lower(x), lower(%({param_id})s)) > 0, string_keys) OR 123 | position(lower(raw_log), lower(%({param_id})s)) > 0 124 | """ 125 | 126 | def build_time_filter(self, time_range): 127 | if time_range == "all": 128 | return None 129 | 130 | current_time = time.time() 131 | time_filters = { 132 | "1h": current_time - 3600, 133 | "24h": current_time - 86400, 134 | "7d": current_time - 604800, 135 | "30d": current_time - 2592000 136 | } 137 | 138 | if time_range in time_filters: 139 | param_id = str(uuid.uuid4()) 140 | self.params[param_id] = time_filters[time_range] 141 | return f"timestamp >= %({param_id})s" 142 | 143 | return None 144 | 145 | 146 | if __name__ == "__main__": 147 | QBuilder().parse("event:null", 1) 148 | # print(type(f)) 149 | # print(dir(f)) 150 | # print(f.value) 151 | # print(repr(parser.parse('title:"foo bar"'))) 152 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.6.0 2 | anyio==3.7.1 3 | async-tail==0.2.0 4 | click==8.1.7 5 | fastapi==0.103.2 6 | h11==0.14.0 7 | idna==3.4 8 | luqum==0.13.0 9 | maturin==1.3.0 10 | ply==3.11 11 | pydantic==2.4.2 12 | pydantic_core==2.10.1 13 | sh==2.0.6 14 | sniffio==1.3.0 15 | starlette==0.27.0 16 | structlog==23.2.0 17 | typing_extensions==4.8.0 18 | ujson==5.8.0 19 | uvicorn==0.23.2 20 | clickhouse-driver==0.2.8 21 | Flask-Login==0.6.3 22 | M2Crypto==0.41.0 -------------------------------------------------------------------------------- /app/user.py: -------------------------------------------------------------------------------- 1 | class User: 2 | def is_authenticated(): 3 | return False 4 | 5 | def is_active(): 6 | return True 7 | 8 | def is_anonymous(): 9 | return True 10 | 11 | # returns str 12 | def get_id(): 13 | pass 14 | 15 | pass 16 | -------------------------------------------------------------------------------- /app/util.py: -------------------------------------------------------------------------------- 1 | from collections.abc import MutableMapping 2 | 3 | 4 | def flatten(dictionary, parent_key=False, separator="."): 5 | """ 6 | Turn a nested dictionary into a flattened dictionary 7 | :param dictionary: The dictionary to flatten 8 | :param parent_key: The string to prepend to dictionary's keys 9 | :param separator: The string used to separate flattened keys 10 | :return: A flattened dictionary 11 | """ 12 | 13 | items = [] 14 | for key, value in dictionary.items(): 15 | new_key = str(parent_key) + separator + key if parent_key else key 16 | if isinstance(value, MutableMapping): 17 | if not value.items(): 18 | items.append((new_key, None)) 19 | else: 20 | items.extend(flatten(value, new_key, separator).items()) 21 | elif isinstance(value, list): 22 | if len(value): 23 | for k, v in enumerate(value): 24 | items.extend(flatten({str(k): v}, new_key, separator).items()) 25 | else: 26 | items.append((new_key, None)) 27 | else: 28 | items.append((new_key, value)) 29 | return dict(items) 30 | 31 | 32 | def isint(s): 33 | try: 34 | int(s) 35 | except ValueError: 36 | return False 37 | else: 38 | return True 39 | 40 | 41 | def isfloat(s): 42 | try: 43 | float(s) 44 | except ValueError: 45 | return False 46 | else: 47 | return True 48 | -------------------------------------------------------------------------------- /assets/1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nevinpuri/erlog/da5ec06ed19bac01bc6faf92761dee128701b553/assets/1.jpeg -------------------------------------------------------------------------------- /assets/2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nevinpuri/erlog/da5ec06ed19bac01bc6faf92761dee128701b553/assets/2.jpeg -------------------------------------------------------------------------------- /config/clickhouse/default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 1 9 | 10 | 11 | 12 | 13 | 1 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 56 | test123 57 | 58 | 78 | 79 | ::/0 80 | 81 | 82 | 83 | default 84 | 85 | 86 | default 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 3600 101 | 102 | 103 | 0 104 | 0 105 | 0 106 | 0 107 | 0 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | db: 4 | image: "clickhouse/clickhouse-server" 5 | ports: 6 | - "8123:8123" 7 | - "9000:9000" 8 | ulimits: 9 | nofile: 10 | soft: "262144" 11 | hard: "262144" 12 | volumes: 13 | - "./config/clickhouse/default.xml:/etc/clickhouse-server/users.d/default.xml" 14 | -------------------------------------------------------------------------------- /install.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nevinpuri/erlog/da5ec06ed19bac01bc6faf92761dee128701b553/install.md -------------------------------------------------------------------------------- /ui/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | 'eslint:recommended', 6 | 'plugin:react/recommended', 7 | 'plugin:react/jsx-runtime', 8 | 'plugin:react-hooks/recommended', 9 | ], 10 | ignorePatterns: ['dist', '.eslintrc.cjs'], 11 | parserOptions: { ecmaVersion: 'latest', sourceType: 'module' }, 12 | settings: { react: { version: '18.2' } }, 13 | plugins: ['react-refresh'], 14 | rules: { 15 | 'react-refresh/only-export-components': [ 16 | 'warn', 17 | { allowConstantExport: true }, 18 | ], 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /ui/README.md: -------------------------------------------------------------------------------- 1 | # React + Vite 2 | 3 | This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. 4 | 5 | Currently, two official plugins are available: 6 | 7 | - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh 8 | - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh 9 | -------------------------------------------------------------------------------- /ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ErLog 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "my-vue-app", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@headlessui/react": "^1.7.18", 14 | "@headlessui/tailwindcss": "^0.2.0", 15 | "@remixicon/react": "^4.2.0", 16 | "@tremor/react": "^3.14.1", 17 | "react": "^18.2.0", 18 | "react-dom": "^18.2.0", 19 | "react-router-dom": "^6.16.0", 20 | "recharts": "^2.8.0", 21 | "winston": "^3.13.0" 22 | }, 23 | "devDependencies": { 24 | "@tailwindcss/forms": "^0.5.6", 25 | "@types/react": "^18.2.15", 26 | "@types/react-dom": "^18.2.7", 27 | "@vitejs/plugin-react": "^4.0.3", 28 | "autoprefixer": "^10.4.16", 29 | "daisyui": "^4.12.2", 30 | "eslint": "^8.45.0", 31 | "eslint-plugin-react": "^7.32.2", 32 | "eslint-plugin-react-hooks": "^4.6.0", 33 | "eslint-plugin-react-refresh": "^0.4.3", 34 | "postcss": "^8.4.31", 35 | "tailwindcss": "^3.3.3", 36 | "vite": "^4.4.5" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /ui/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /ui/src/App.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | import Search from "./components/search"; 3 | import { useEffect } from "react"; 4 | import Grid from "./components/grid"; 5 | import { useLocation, useNavigate } from "react-router-dom"; 6 | import TimeFilter from "./components/TimeFilter"; 7 | 8 | const fetchLogs = async (query, page, showChildren, timeRange) => { 9 | try { 10 | let q = query.replace(" and ", " AND "); 11 | q = q.replace(" or ", " OR "); 12 | const response = await fetch("http://localhost:8000/search", { 13 | method: "POST", 14 | headers: { 15 | "Content-Type": "application/json", 16 | }, 17 | body: JSON.stringify({ 18 | query: q, 19 | page, 20 | showChildren, 21 | timeRange 22 | }), 23 | }); 24 | 25 | if (!response.ok) { 26 | const errorData = await response.json().catch(() => ({ detail: 'Unknown error' })); 27 | return { logs: null, err: errorData.detail || `Error: ${response.status}` }; 28 | } 29 | 30 | const data = await response.json(); 31 | return { logs: data, err: null }; 32 | } catch (error) { 33 | console.error('Fetch error:', error); 34 | return { logs: null, err: 'Failed to connect to the server' }; 35 | } 36 | }; 37 | 38 | export function useQuery() { 39 | const { search } = useLocation(); 40 | 41 | return React.useMemo(() => new URLSearchParams(search), [search]); 42 | } 43 | 44 | function App() { 45 | const router = useNavigate(); 46 | const q = useQuery(); 47 | 48 | const [err, setErr] = useState(null); 49 | const [logs, setLogs] = useState(null); 50 | const [isLoading, setIsLoading] = useState(true); 51 | const [showChildren, setShowChildren] = useState(() => { 52 | const c = q.get("children"); 53 | return c === "true"; 54 | }); 55 | const [timeRange, setTimeRange] = useState(() => { 56 | return q.get("time") || "all"; 57 | }); 58 | 59 | // Handle URL updates when filters change 60 | useEffect(() => { 61 | const currentQuery = q.get("query") || ""; 62 | const currentPage = q.get("page") || "0"; 63 | router(`/?query=${currentQuery}&page=${currentPage}&children=${showChildren}&time=${timeRange}`); 64 | }, [showChildren, timeRange]); 65 | 66 | // Fetch logs when URL params change 67 | useEffect(() => { 68 | console.log("Fetching logs with params:", { 69 | query: q.get("query"), 70 | children: q.get("children"), 71 | page: q.get("page"), 72 | time: q.get("time") 73 | }); 74 | f(); 75 | }, [q.get("query"), q.get("children"), q.get("page"), q.get("time")]); 76 | 77 | const f = async () => { 78 | setIsLoading(true); 79 | try { 80 | let page = q.get("page"); 81 | let query = q.get("query"); 82 | let showChildren = q.get("children"); 83 | let timeRange = q.get("time"); 84 | 85 | if (!query) query = ""; 86 | if (!page) page = 0; 87 | if (!showChildren) showChildren = false; 88 | if (!timeRange) timeRange = "all"; 89 | 90 | const { logs, err } = await fetchLogs( 91 | query, 92 | page, 93 | showChildren, 94 | timeRange 95 | ); 96 | setLogs(logs); 97 | setErr(err); 98 | } catch (error) { 99 | setErr("An error occurred while fetching logs"); 100 | console.error(error); 101 | } finally { 102 | setIsLoading(false); 103 | } 104 | }; 105 | 106 | return ( 107 |
108 |
109 | { 112 | router(`/?query=${e}&page=0&children=${showChildren}&time=${timeRange}`); 113 | }} 114 | showChildren={showChildren} 115 | onShowChildrenChange={(e) => { 116 | setShowChildren(e); 117 | }} 118 | timeRange={timeRange} 119 | onTimeRangeChange={(range) => { 120 | setTimeRange(range); 121 | }} 122 | /> 123 |
124 |

{err}

125 | {isLoading ? ( 126 |
127 |
128 |
129 | ) : ( 130 | 131 | )} 132 |
133 | ); 134 | } 135 | 136 | export default App; 137 | -------------------------------------------------------------------------------- /ui/src/components/GridItem.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useCallback } from "react"; 2 | import { cvtReadable, timeConverter } from "../util"; 3 | 4 | function getColor(logLevel: any) { 5 | // console.log('Log level:', logLevel, typeof logLevel); 6 | 7 | if (!logLevel) { 8 | return ''; 9 | } 10 | 11 | switch (logLevel.toString().toLowerCase()) { 12 | case "info": 13 | return "bg-green-400/20"; 14 | case "warning": 15 | return "bg-yellow-400/20"; 16 | case "error": 17 | return "bg-red-400/20"; 18 | case "critical": 19 | return "bg-red-400/40"; 20 | case "debug": 21 | return "bg-purple-400/20"; 22 | default: 23 | // console.log('No match for log level:', logLevel); 24 | return ""; 25 | } 26 | } 27 | 28 | function getLink(log: any) { 29 | if (!log) return '#'; 30 | 31 | const id = log.id || log._id; 32 | if (!id) return '#'; 33 | 34 | // const parentId = log.parentId || log.parent_id || log._parent_id; 35 | 36 | // if (parentId && parentId !== '00000000-0000-0000-0000-000000000000') { 37 | // return `/${parentId}#${id}`; 38 | // } 39 | 40 | return `/${id}`; 41 | } 42 | 43 | interface Props { 44 | item: Item; 45 | onHover: (id: string) => void; 46 | } 47 | 48 | interface Item { 49 | id: string; 50 | child_logs: number; 51 | timestamp: string; 52 | log: string; 53 | } 54 | 55 | interface LogPreview { 56 | timestamp: string; 57 | childCount: number; 58 | fields: { key: string; value: string }[]; 59 | } 60 | 61 | export default function GridItem({ item, onHover }: Props) { 62 | const [expanded, setExpanded] = useState(false); 63 | const [childLogs, setChildLogs] = useState(null); 64 | const [isLoading, setIsLoading] = useState(false); 65 | const log = JSON.parse(item.log); 66 | const bgColor = getColor(log.level); 67 | const linkUrl = getLink(item); 68 | 69 | const fetchChildLogs = async () => { 70 | setIsLoading(true); 71 | try { 72 | const response = await fetch(`http://localhost:8000/children/${item.id}`); 73 | if (!response.ok) { 74 | throw new Error('Failed to fetch children'); 75 | } 76 | const data = await response.json(); 77 | setChildLogs(data); 78 | } catch (error) { 79 | console.error('Failed to fetch child logs:', error); 80 | setChildLogs([]); 81 | } finally { 82 | setIsLoading(false); 83 | } 84 | }; 85 | 86 | const handleExpand = async (e) => { 87 | e.preventDefault(); 88 | if (!expanded && !childLogs) { 89 | await fetchChildLogs(); 90 | } 91 | setExpanded(!expanded); 92 | }; 93 | 94 | return ( 95 |
96 |
onHover(item.id)} 99 | > 100 |
101 | {item.child_logs > 0 && ( 102 |
103 | 104 | {item.child_logs} 105 | 106 | 130 |
131 | )} 132 | { 137 | if (linkUrl === '#') { 138 | e.preventDefault(); 139 | } 140 | }} 141 | > 142 | {cvtReadable(log)} 143 | 144 |
145 | {timeConverter(item.timestamp)} 146 |
147 | 148 | {expanded && ( 149 |
150 | {childLogs && childLogs.map(childLog => ( 151 | onHover(id)} 155 | /> 156 | ))} 157 |
158 | )} 159 |
160 | ); 161 | } 162 | -------------------------------------------------------------------------------- /ui/src/components/TimeFilter.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export type TimeRange = 'all' | '1h' | '24h' | '7d' | '30d'; 4 | 5 | interface TimeFilterProps { 6 | value: TimeRange; 7 | onChange: (range: TimeRange) => void; 8 | } 9 | 10 | export default function TimeFilter({ value, onChange }: TimeFilterProps) { 11 | return ( 12 | 23 | ); 24 | } -------------------------------------------------------------------------------- /ui/src/components/back.jsx: -------------------------------------------------------------------------------- 1 | import { useNavigate } from "react-router-dom"; 2 | 3 | export default function BackButton() { 4 | const router = useNavigate(); 5 | return ( 6 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /ui/src/components/grid-old.tsx: -------------------------------------------------------------------------------- 1 | import { LogView } from "../pages/logid"; 2 | import { cvtReadable, timeConverter } from "../util/index"; 3 | import React from "react"; 4 | 5 | interface IProps { 6 | logs: any; 7 | } 8 | 9 | export default function GridOld({ logs }: IProps) { 10 | return ( 11 | <> 12 |
13 | 14 | {logs.map((log) => ( 15 | 16 |
17 | 18 | 23 | 🎓 24 |
25 |

26 | {cvtReadable(JSON.parse(log.log))}{" "} 27 | 28 | +1 29 | 30 |

31 | 32 | {timeConverter(log.timestamp)} 33 | 34 |
35 |
36 |
37 | <> 38 | 39 | 40 | View Children 41 | 42 | 43 |
44 | 45 | ))} 46 | 47 |
48 | 49 | ); 50 | } 51 | -------------------------------------------------------------------------------- /ui/src/components/grid.tsx: -------------------------------------------------------------------------------- 1 | import { LogView } from "../pages/logid"; 2 | import { cvtReadable, timeConverter } from "../util/index"; 3 | import React, { useState } from "react"; 4 | import GridItem from "./GridItem"; 5 | 6 | interface LogPreview { 7 | timestamp: string; 8 | childCount: number; 9 | fields: { key: string; value: string }[]; 10 | } 11 | 12 | interface IProps { 13 | logs: any; 14 | } 15 | 16 | export default function Grid({ logs }: IProps) { 17 | const [showPreview, setShowPreview] = useState(false); 18 | const [previewData, setPreviewData] = useState(null); 19 | const [isLoadingPreview, setIsLoadingPreview] = useState(false); 20 | const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 }); 21 | const [activeItemId, setActiveItemId] = useState(null); 22 | 23 | const handleMouseMove = (e: React.MouseEvent) => { 24 | setMousePosition({ x: e.clientX, y: e.clientY }); 25 | }; 26 | 27 | const handleMouseLeave = () => { 28 | setShowPreview(false); 29 | setActiveItemId(null); 30 | }; 31 | 32 | const fetchPreviewData = async (id: string) => { 33 | if (id === activeItemId) return; 34 | setActiveItemId(id); 35 | setIsLoadingPreview(true); 36 | try { 37 | const response = await fetch(`http://localhost:8000/preview/${id}`); 38 | if (!response.ok) throw new Error('Failed to fetch preview'); 39 | const data = await response.json(); 40 | 41 | const parsedLog = JSON.parse(data.log); 42 | const fields = Object.entries(parsedLog) 43 | .filter(([key]) => key !== 'timestamp' && key !== 'level') 44 | .map(([key, value]) => ({ 45 | key, 46 | value: typeof value === 'object' ? JSON.stringify(value) : String(value) 47 | })); 48 | 49 | setPreviewData({ 50 | timestamp: data.timestamp, 51 | childCount: data.child_logs, 52 | fields 53 | }); 54 | setShowPreview(true); 55 | } catch (error) { 56 | console.error('Failed to fetch preview:', error); 57 | } finally { 58 | setIsLoadingPreview(false); 59 | } 60 | }; 61 | 62 | return ( 63 |
68 | {logs.map((log) => ( 69 |
70 | fetchPreviewData(id)} 73 | /> 74 |
75 | ))} 76 | 77 | {showPreview && ( 78 |
86 |
87 | {isLoadingPreview ? ( 88 |
89 |
90 |
91 | ) : previewData && ( 92 |
93 |
94 | 95 | {timeConverter(previewData.timestamp)} 96 | 97 | {previewData.childCount > 0 && ( 98 | 99 | {previewData.childCount} children 100 | 101 | )} 102 |
103 |
104 | {previewData.fields.map(({ key, value }, index) => ( 105 |
106 | {key}: 107 | {value} 108 |
109 | ))} 110 |
111 |
112 | )} 113 |
114 |
115 | )} 116 |
117 | ); 118 | } 119 | -------------------------------------------------------------------------------- /ui/src/components/metricview.jsx: -------------------------------------------------------------------------------- 1 | import { 2 | Line, 3 | CartesianGrid, 4 | XAxis, 5 | YAxis, 6 | Tooltip, 7 | ResponsiveContainer, 8 | } from "recharts"; 9 | import { LineChart } from "@tremor/react"; 10 | 11 | export default function MetricView({ title, data, per }) { 12 | return ( 13 |
14 | 22 | {/* 23 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | */} 36 | {/*

Send a POST requests with

37 |

/metrics "id": "cpu_usage", value: "ifjasdiojf"

*/} 38 |
39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /ui/src/components/search.jsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import TimeFilter from "./TimeFilter"; 3 | 4 | export default function Search({ 5 | onSubmit, 6 | onChange, 7 | defaultValue, 8 | onEnter, 9 | onShowChildrenChange, 10 | showChildren, 11 | timeRange, 12 | onTimeRangeChange, 13 | }) { 14 | const [v, setV] = useState(""); 15 | 16 | const handleTimeRangeChange = (range) => { 17 | if (onTimeRangeChange) { 18 | onTimeRangeChange(range); 19 | } 20 | }; 21 | 22 | const handleShowChildrenChange = (checked) => { 23 | if (onShowChildrenChange) { 24 | onShowChildrenChange(checked); 25 | } 26 | }; 27 | 28 | return ( 29 |
e.preventDefault()}> 30 | 71 | 72 |
73 | 77 |
78 | 79 |
80 |
81 | 90 |
91 |
92 |
93 | ); 94 | } 95 | -------------------------------------------------------------------------------- /ui/src/index.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css2?family=Inconsolata:wght@400;500;600&display=swap"); 2 | 3 | @tailwind base; 4 | @tailwind components; 5 | @tailwind utilities; 6 | 7 | * { 8 | font-family: "Inconsolata", monospace; 9 | } 10 | 11 | @keyframes fadeIn { 12 | from { 13 | opacity: 0; 14 | transform: translateY(-10px); 15 | } 16 | to { 17 | opacity: 1; 18 | transform: translateY(0); 19 | } 20 | } 21 | 22 | .animate-fadeIn { 23 | animation: fadeIn 0.2s ease-out forwards; 24 | } 25 | 26 | @keyframes noise { 27 | 0%, 100% { transform: translate(0, 0); } 28 | 10% { transform: translate(-5%, -5%); } 29 | 20% { transform: translate(-10%, 5%); } 30 | 30% { transform: translate(5%, -10%); } 31 | 40% { transform: translate(-5%, 15%); } 32 | 50% { transform: translate(-10%, 5%); } 33 | 60% { transform: translate(15%, 0); } 34 | 70% { transform: translate(0, 10%); } 35 | 80% { transform: translate(-15%, 0); } 36 | 90% { transform: translate(10%, 5%); } 37 | } 38 | 39 | .bg-noise { 40 | background-image: url("data:image/svg+xml,%3Csvg viewBox='0 0 200 200' xmlns='http://www.w3.org/2000/svg'%3E%3Cfilter id='noise'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.65' numOctaves='3' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%' height='100%' filter='url(%23noise)'/%3E%3C/svg%3E"); 41 | filter: contrast(350%) brightness(1000%); 42 | animation: noise 1s steps(2) infinite; 43 | } 44 | -------------------------------------------------------------------------------- /ui/src/main.jsx: -------------------------------------------------------------------------------- 1 | import "./index.css"; 2 | import React from "react"; 3 | import ReactDOM from "react-dom/client"; 4 | import App from "./App.jsx"; 5 | import { LogId, loader as logIdLoader } from "./pages/logid"; 6 | import { createBrowserRouter, RouterProvider } from "react-router-dom"; 7 | import { Metrics } from "./pages/metrics"; 8 | 9 | const router = createBrowserRouter([ 10 | { 11 | path: "/", 12 | element: , 13 | }, 14 | { 15 | path: "/:id", 16 | element: , 17 | loader: logIdLoader, 18 | }, 19 | { 20 | path: "/metrics", 21 | element: , 22 | }, 23 | ]); 24 | 25 | const selected = (route) => { 26 | console.log(window.location.href); 27 | if (window.location.pathname === route) { 28 | return "text-blue-600"; 29 | } 30 | 31 | return ""; 32 | }; 33 | 34 | ReactDOM.createRoot(document.getElementById("root")).render( 35 | 36 |
37 | 82 |
83 | 84 |
85 |
86 | 87 | {/* */} 88 |
89 | ); 90 | -------------------------------------------------------------------------------- /ui/src/pages/logid.tsx: -------------------------------------------------------------------------------- 1 | import { useLoaderData, useLocation, useParams } from "react-router-dom"; 2 | import BackButton from "../components/back"; 3 | import { timeConverter } from "../util"; 4 | import React from "react"; 5 | 6 | const fetchLog = async (id) => { 7 | const response = await fetch("http://localhost:8000/get", { 8 | method: "POST", 9 | headers: { 10 | "Content-Type": "application/json", 11 | }, 12 | body: JSON.stringify({ id }), 13 | }); 14 | 15 | const d = await response.json(); 16 | return { log: d }; 17 | }; 18 | 19 | export async function loader({ params }) { 20 | const { log } = await fetchLog(params.id); 21 | return { log }; 22 | } 23 | 24 | export function LogId() { 25 | const { log } = useLoaderData(); 26 | const location = useLocation(); 27 | const f = location.hash.substring(1); 28 | 29 | return ( 30 |
31 | 32 |

{timeConverter(log.timestamp)}

33 |
34 | 35 | {/* */} 36 |

Children

37 |
38 | {log.children.map((c) => ( 39 | 51 | ))} 52 |
53 |
54 | ); 55 | } 56 | 57 | interface ILogViewProps { 58 | data: any; 59 | } 60 | 61 | export const LogView = ({ data }: ILogViewProps) => { 62 | return ( 63 |
64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | {Object.entries(data).map(([key, value]) => ( 73 | <> 74 | {key !== "timestamp" ? ( 75 | 76 | 77 | 78 | 79 | ) : ( 80 | <> 81 | )} 82 | 83 | ))} 84 | 85 |
FieldValue
{key}{value}
86 |
87 | ); 88 | }; 89 | 90 | const PrettyPrintJson = ({ data }) => { 91 | // (destructured) data could be a prop for example 92 | return ( 93 |
94 |
{JSON.stringify(data, null, 2)}
95 |
96 | ); 97 | }; 98 | -------------------------------------------------------------------------------- /ui/src/pages/metrics.jsx: -------------------------------------------------------------------------------- 1 | import { 2 | LineChart, 3 | Line, 4 | CartesianGrid, 5 | XAxis, 6 | YAxis, 7 | Tooltip, 8 | } from "recharts"; 9 | import MetricView from "../components/metricview"; 10 | import { useEffect, useState } from "react"; 11 | 12 | export function Metrics() { 13 | const [filter, setFilter] = useState("hour"); 14 | const [data, setData] = useState(); 15 | 16 | useEffect(() => { 17 | fetch("http://localhost:8000/search", { 18 | method: "POST", 19 | body: JSON.stringify({ 20 | per: filter, 21 | }), 22 | }).then(async (e) => { 23 | const d = await e.json(); 24 | console.log(d); 25 | setData(d); 26 | }); 27 | }, [filter]); 28 | 29 | // fuck me, this whole add shit to that object shit is really really good 30 | return ( 31 |
32 |
33 |

Metrics — Coming Soon

34 |

35 | Follow me on{" "} 36 | 42 | Twitter 43 | {" "} 44 | to stay updated 45 |

46 | {/*
47 | 48 | 56 |
*/} 57 |
58 | {/* */} 59 | {/* 60 | 61 | */} 62 | {/* 66 | + New Metric 67 | */} 68 |
69 | //
70 | //

Active Users Per Hour

71 | // 77 | // 78 | // 79 | // 80 | // 81 | // 82 | // 83 | //
84 | ); 85 | } 86 | 87 | /*

Send a POST requests with

88 |

/metrics "id": "cpu_usage", value: "ifjasdiojf"

*/ 89 | -------------------------------------------------------------------------------- /ui/src/util/index.js: -------------------------------------------------------------------------------- 1 | export function timeConverter(UNIX_timestamp) { 2 | var a = new Date(UNIX_timestamp * 1000); 3 | var months = [ 4 | "Jan", 5 | "Feb", 6 | "Mar", 7 | "Apr", 8 | "May", 9 | "Jun", 10 | "Jul", 11 | "Aug", 12 | "Sep", 13 | "Oct", 14 | "Nov", 15 | "Dec", 16 | ]; 17 | var year = a.getFullYear(); 18 | var month = months[a.getMonth()]; 19 | var date = a.getDate(); 20 | var hour = a.getHours(); 21 | // var min = a.getMinutes(); 22 | // var sec = a.getSeconds(); 23 | 24 | var min = "0" + a.getMinutes(); 25 | // Seconds part from the timestamp 26 | var sec = "0" + a.getSeconds(); 27 | 28 | var ms = a.getMilliseconds(); 29 | var time = 30 | date + 31 | " " + 32 | month + 33 | " " + 34 | year + 35 | " " + 36 | hour + 37 | ":" + 38 | min.slice(-2) + 39 | ":" + 40 | sec.slice(-2) + 41 | "." + 42 | ms; 43 | 44 | return time; 45 | } 46 | 47 | export function cvtReadable(obj) { 48 | let out = ""; 49 | for (const [key, value] of Object.entries(obj)) { 50 | if (key === "id" || key === "parent_id") { 51 | continue; 52 | } 53 | out += ` ${key}=${value}`; 54 | } 55 | 56 | return out.trim(); 57 | } 58 | -------------------------------------------------------------------------------- /ui/src/util/logger.ts: -------------------------------------------------------------------------------- 1 | import winston from "winston"; 2 | 3 | class Logger { 4 | private logger = winston.createLogger({ 5 | transports: [ 6 | new winston.transports.Http({ 7 | host: "localhost", 8 | port: 8000, 9 | }), 10 | ], 11 | }); 12 | 13 | constructor() {} 14 | 15 | public info(data: object) { 16 | this.logger.info("fkas", data); 17 | } 18 | } 19 | 20 | export function getLogger() { 21 | return new Logger(); 22 | } 23 | 24 | let logUrl: string; 25 | export function init(_logUrl: string) { 26 | if (_logUrl.endsWith("/")) { 27 | logUrl = _logUrl.slice(0, _logUrl.length - 1); 28 | } else { 29 | logUrl = _logUrl; 30 | } 31 | } 32 | 33 | /** 34 | * You would use it like 35 | * report("post_created", {userId: 3}) 36 | * @param args 37 | */ 38 | export function report( 39 | name: string, 40 | args: object, 41 | throwError: boolean = false 42 | ) { 43 | // just queue it up or some thing or just send the post request asynchronously 44 | // return a promise, you can await it if you want 45 | return new Promise(async (resolve, reject) => { 46 | if (!logUrl) { 47 | return reject("Error: please initialize logger"); 48 | } 49 | try { 50 | await fetch(logUrl + "/metrics", { 51 | method: "POST", 52 | body: JSON.stringify({ 53 | name, 54 | args, 55 | }), 56 | }); 57 | } catch (exception) { 58 | if (throwError === true) { 59 | return reject(exception); 60 | } 61 | return resolve(); 62 | } 63 | return resolve(); 64 | }); 65 | } 66 | 67 | /** 68 | * Supports all the parent_id shit. Main feature, creating metrics in next js apps in the api route 69 | */ 70 | export function log() {} 71 | -------------------------------------------------------------------------------- /ui/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | import daisyui from "daisyui"; 3 | import colors from "tailwindcss/colors"; 4 | 5 | export default { 6 | content: [ 7 | "./index.html", 8 | "./src/**/*.{js,ts,jsx,tsx}", 9 | "./node_modules/@tremor/**/*.{js,ts,jsx,tsx}", 10 | ], 11 | theme: { 12 | transparent: "transparent", 13 | current: "currentColor", 14 | extend: { 15 | colors: { 16 | // light mode 17 | tremor: { 18 | brand: { 19 | faint: colors.blue[50], 20 | muted: colors.blue[200], 21 | subtle: colors.blue[400], 22 | DEFAULT: colors.blue[500], 23 | emphasis: colors.blue[700], 24 | inverted: colors.white, 25 | }, 26 | background: { 27 | muted: colors.gray[50], 28 | subtle: colors.gray[100], 29 | DEFAULT: colors.white, 30 | emphasis: colors.gray[700], 31 | }, 32 | border: { 33 | DEFAULT: colors.gray[200], 34 | }, 35 | ring: { 36 | DEFAULT: colors.gray[200], 37 | }, 38 | content: { 39 | subtle: colors.gray[400], 40 | DEFAULT: colors.gray[500], 41 | emphasis: colors.gray[700], 42 | strong: colors.gray[900], 43 | inverted: colors.white, 44 | }, 45 | }, 46 | // dark mode 47 | "dark-tremor": { 48 | brand: { 49 | faint: "#0B1229", 50 | muted: colors.blue[950], 51 | subtle: colors.blue[800], 52 | DEFAULT: colors.blue[500], 53 | emphasis: colors.blue[400], 54 | inverted: colors.blue[950], 55 | }, 56 | background: { 57 | muted: "#131A2B", 58 | subtle: colors.gray[800], 59 | DEFAULT: colors.gray[900], 60 | emphasis: colors.gray[300], 61 | }, 62 | border: { 63 | DEFAULT: colors.gray[800], 64 | }, 65 | ring: { 66 | DEFAULT: colors.gray[800], 67 | }, 68 | content: { 69 | subtle: colors.gray[600], 70 | DEFAULT: colors.gray[500], 71 | emphasis: colors.gray[200], 72 | strong: colors.gray[50], 73 | inverted: colors.gray[950], 74 | }, 75 | }, 76 | }, 77 | boxShadow: { 78 | // light 79 | "tremor-input": "0 1px 2px 0 rgb(0 0 0 / 0.05)", 80 | "tremor-card": 81 | "0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)", 82 | "tremor-dropdown": 83 | "0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)", 84 | // dark 85 | "dark-tremor-input": "0 1px 2px 0 rgb(0 0 0 / 0.05)", 86 | "dark-tremor-card": 87 | "0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)", 88 | "dark-tremor-dropdown": 89 | "0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)", 90 | }, 91 | borderRadius: { 92 | "tremor-small": "0.375rem", 93 | "tremor-default": "0.5rem", 94 | "tremor-full": "9999px", 95 | }, 96 | fontSize: { 97 | "tremor-label": ["0.75rem", { lineHeight: "1rem" }], 98 | "tremor-default": ["0.875rem", { lineHeight: "1.25rem" }], 99 | "tremor-title": ["1.125rem", { lineHeight: "1.75rem" }], 100 | "tremor-metric": ["1.875rem", { lineHeight: "2.25rem" }], 101 | }, 102 | }, 103 | }, 104 | safelist: [ 105 | { 106 | pattern: 107 | /^(bg-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 108 | variants: ["hover", "ui-selected"], 109 | }, 110 | { 111 | pattern: 112 | /^(text-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 113 | variants: ["hover", "ui-selected"], 114 | }, 115 | { 116 | pattern: 117 | /^(border-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 118 | variants: ["hover", "ui-selected"], 119 | }, 120 | { 121 | pattern: 122 | /^(ring-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 123 | }, 124 | { 125 | pattern: 126 | /^(stroke-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 127 | }, 128 | { 129 | pattern: 130 | /^(fill-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 131 | }, 132 | ], 133 | plugins: [ 134 | // require("@tailwindcss/forms"), 135 | require("@headlessui/tailwindcss"), 136 | require("daisyui"), 137 | ], 138 | // retro, forest 139 | daisyui: { 140 | themes: ["retro"], 141 | }, 142 | }; 143 | -------------------------------------------------------------------------------- /ui/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import react from "@vitejs/plugin-react"; 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | define: { "process.env": {} }, 8 | }); 9 | --------------------------------------------------------------------------------