├── .gitignore
├── .vscode
└── settings.json
├── Dockerfile
├── LICENSE
├── README.md
├── app-go
├── db
│ └── models.go
├── go.mod
├── go.sum
├── main.go
├── query.sql
└── routes
│ ├── request.go
│ ├── response.go
│ ├── routes.go
│ └── search.go
├── app
├── .gitignore
├── main.py
├── models.py
├── query.py
├── requirements.txt
├── user.py
└── util.py
├── assets
├── 1.jpeg
└── 2.jpeg
├── config
└── clickhouse
│ └── default.xml
├── docker-compose.yml
├── install.md
└── ui
├── .eslintrc.cjs
├── .gitignore
├── README.md
├── index.html
├── package.json
├── pnpm-lock.yaml
├── postcss.config.js
├── src
├── App.jsx
├── components
│ ├── GridItem.tsx
│ ├── TimeFilter.tsx
│ ├── back.jsx
│ ├── grid-old.tsx
│ ├── grid.tsx
│ ├── metricview.jsx
│ └── search.jsx
├── index.css
├── main.jsx
├── pages
│ ├── logid.tsx
│ └── metrics.jsx
└── util
│ ├── index.js
│ └── logger.ts
├── tailwind.config.js
└── vite.config.js
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | logs.db
3 | logs.db.wal
4 | /venv
5 | /app/.venv
6 |
7 | # Added by cargo
8 | #
9 | # already existing elements were commented out
10 |
11 | #/target
12 |
13 | .DS_Store
14 | logs/
15 |
16 | .venv/
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "[python]": {
3 | "editor.defaultFormatter": "ms-python.black-formatter"
4 | },
5 | "python.formatting.provider": "none"
6 | }
7 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11-slim
2 |
3 | EXPOSE 8000
4 |
5 | RUN pip install fastapi ujson structlog chdb uvicorn luqum async_tail
6 |
7 | COPY ./app/ ./app/
8 |
9 | CMD ["python3", "-m", "uvicorn", "app.main:app"]
10 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Nevin Puri.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## Erlog
2 |
3 | #### A Log Platform which runs on a $4 VPS
4 |
5 | 
6 |
7 | ## Features
8 |
9 | - Ingest as many logs as you want from an http endpoint or file
10 | - Query logs with a nice syntax
11 | - Support for `parent_id`, so logs can have "children"
12 |
13 | ## Sending Logs
14 |
15 | Just send a POST request to erlog with JSON
16 |
17 | ```
18 | {
19 | "timestamp": "1675955819",
20 | "level": "debug",
21 | "service": "my_service",
22 | "key": "value",
23 | "data": {
24 | "another_key": "another value"
25 | }
26 | }
27 | ```
28 |
29 | Alternatively, run `export LOGS=file1.txt file2.txt` and those files will be tailed and ingested in to erlog.
30 |
31 | ## Viewing Logs
32 |
33 | 
34 |
35 | ## Querying
36 |
37 | Erql is extremely simple. Here are some examples
38 |
39 | ```rb
40 | # search for "foo" in any log (case sensitive)
41 | foo
42 | "foo"
43 |
44 | # search where name = "foo" or name = "foo bar"
45 | name:foo
46 | name:"foo bar"
47 |
48 | # search for {name: {first: 'foo'}}
49 | name.first:foo
50 | name.first:"foo bar"
51 |
52 | # search for name[0] = "item"
53 | name.0:item
54 |
55 | # member = true
56 | member:true
57 | member:false
58 |
59 | # null search
60 | member:null
61 |
62 | # search for height =, >, >=, <, <=
63 | height:=100
64 | height:>100
65 | height:>=100
66 | height:<100
67 | height:<=100
68 |
69 | # search for height in range (inclusive, exclusive)
70 | height:[100 TO 200]
71 | height:{100 TO 200}
72 |
73 | # AND/OR operators
74 | name:foo AND height:=100
75 | name:foo OR name:bar
76 |
77 | # grouping
78 | name:foo AND (bio:bar OR bio:baz)
79 | ```
80 |
81 | ## Instrumenting
82 |
83 | Just have your structured logger log in json, and then forward those logs to a file. A collector is coming soon which just `tail`s a file and forwards the requests to your erlog instance.
84 |
85 | Look here for an example of an instrumented function.
86 |
87 | ## Example Project
88 |
89 | For an example project, see app/main.py. Erlog is instrumented using structlog.
90 | If you run `export LOGS=file1.txt` and `python3 -m uvicorn main:app > file1.txt`,
91 | you will start seeing erlogs logs in erlog.
92 |
93 | ## Using `parent_id`
94 |
95 | ```python
96 | import uuid
97 | from structlog import get_logger
98 |
99 | logger = get_logger()
100 |
101 | # don't forget str()
102 | # making the id a Uuid type won't work
103 | # as it'll be printed as Uuid('iaodjfoiasjdijdsaiojf')
104 | # instead of 'iaodjfoiasjdijdsaiojf'
105 |
106 | id = str(uuid.uuid4())
107 | logger.log("root log", id=id)
108 | logger.log("child of root", parent_id=id)
109 | ```
110 |
111 | This will show the log "root log" in the erlog ui.
112 | Once you click on "root log", you will be able to see "child of root"
113 | in the log viewer.
114 |
115 | This is useful for when you want to capture logs across many different services, and have them all be in one location.
116 |
117 | ## Todo
118 |
119 | - clear db on 'reset_erlog' message (for development)
120 | - stream logs from db
121 | - get support for traces using `parentId` and `duration` or `start` `end` in ms
122 | - show parents whenever you click on a child
123 | - on log submit, if log level is error, then with id == parent_id field to be error=true
124 | - same thing with warning
125 | - add toggle for parent id null or not
126 |
127 | - for each log, make a call to the db with the info to try and merge the events together
128 | - or just do that on every other log call where whenver there's a new info you
129 | merge it into the current log, and an error you merge into the corresponding error log
130 |
131 | The idea is that if they both have the same parent id, do the merge
132 |
133 | - start reporting logs from the ui, set up service_name for both
134 |
135 | ##### installation packages
136 |
137 | - install openssl
138 | - install swig
139 |
140 | to install m2crypto on mac
141 |
142 | ```
143 | env LDFLAGS="-L$(brew --prefix openssl)/lib" \
144 | CFLAGS="-I$(brew --prefix openssl)/include" \
145 | SWIG_FEATURES="-cpperraswarn -includeall -I$(brew --prefix openssl)/include" \
146 | pip3 install m2crypto
147 | ```
148 |
149 | We get it, monitoring your applications in production is difficult. Most logging platforms are designed for large applications where people need to \_\_.
150 |
151 | However, that's not you. You're just building a small project and want to view your logs from your next js api routes, formatted, in one place, without configuring any additional collector. Erlog is really great at doing that.
152 |
153 | ## Most logging platforms are built for enterprise
154 |
155 | ## Most logging platforms aren't built for people like you
156 |
157 | (aren't built for people like you)
158 |
159 | They have every single use case, a convoluted ui, and follow a standard which is too complicated
160 |
161 | ours:
162 |
163 | - simple, one menu
164 | - zero configuration
165 | - works with your code
166 |
167 | - you dont need to worry about infastructure
168 | - you pay per month
169 |
170 | ## Make a filter be logs with children updated recently
171 |
172 | ## Alow filtering to be done on title + messsage
173 |
174 | ## Filtering will be main way we can differentiate from discord, also having this extra data from the client libraries
175 |
176 | - keep the client libraries, say ok put this here, and after you send in, you can send MORE data (not just message titel simple shit etc but full json, and then filter on that, so you can have really advanced filters and charts from this thing as oppoesd to discord/slack)
177 |
178 | ## 10X experience
179 |
180 | currently there isn't that much, but mainly for users the killer pain point will be the metrics
181 |
182 |
183 | ## Installing on mac
184 | CFLAGS="-I$(brew --prefix openssl)/include" \
185 | SWIG_FEATURES="-cpperraswarn -includeall -I$(brew --prefix openssl)/include" \
--------------------------------------------------------------------------------
/app-go/db/models.go:
--------------------------------------------------------------------------------
1 | package db
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "net"
7 | "os"
8 | "time"
9 |
10 | "github.com/ClickHouse/clickhouse-go/v2"
11 | "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
12 | )
13 |
14 | var Conn driver.Conn
15 |
16 | func ConnectDB() {
17 | dialCount := 0
18 | conn, err := clickhouse.Open(&clickhouse.Options{
19 | Addr: []string{"127.0.0.1:19000"},
20 | Auth: clickhouse.Auth{
21 | Database: "default",
22 | Username: "default",
23 | Password: "test123",
24 | },
25 | DialContext: func(ctx context.Context, addr string) (net.Conn, error) {
26 | dialCount++
27 | var d net.Dialer
28 | return d.DialContext(ctx, "tcp", addr)
29 | },
30 | Debug: true,
31 | Debugf: func(format string, v ...any) {
32 | fmt.Printf(format+"\n", v...)
33 | },
34 | Settings: clickhouse.Settings{
35 | "max_execution_time": 60,
36 | },
37 | Compression: &clickhouse.Compression{
38 | Method: clickhouse.CompressionLZ4,
39 | },
40 | DialTimeout: time.Second * 30,
41 | MaxOpenConns: 5,
42 | MaxIdleConns: 5,
43 | ConnMaxLifetime: time.Duration(10) * time.Minute,
44 | ConnOpenStrategy: clickhouse.ConnOpenInOrder,
45 | BlockBufferSize: 10,
46 | MaxCompressionBuffer: 10240,
47 | ClientInfo: clickhouse.ClientInfo{ // optional, please see Client info section in the README.md
48 | Products: []struct {
49 | Name string
50 | Version string
51 | }{
52 | {Name: "my-app", Version: "0.1"},
53 | },
54 | },
55 | })
56 |
57 | if err != nil {
58 | fmt.Printf("%v\n", err.Error())
59 | os.Exit(0)
60 | }
61 |
62 | conn.Ping(context.Background())
63 |
64 | // so will basically be
65 | // metrics:
66 | // id, name, timestamp
67 | // report("user signed up")
68 |
69 | fmt.Println("Creating Metrics table")
70 | err = conn.Exec(context.Background(), "CREATE TABLE IF NOT EXISTS metrics (id UUID, name String, timestamp DateTime) Engine = MergeTree PRIMARY KEY (id, timestamp, name) ORDER BY (id, timestamp, name);")
71 | if err != nil {
72 | fmt.Printf("%v\n", err.Error())
73 | os.Exit(0)
74 | }
75 | fmt.Println("Done creating Metrics Table")
76 |
77 | Conn = conn
78 | }
--------------------------------------------------------------------------------
/app-go/go.mod:
--------------------------------------------------------------------------------
1 | module erlog
2 |
3 | go 1.22.1
4 |
5 | require (
6 | github.com/ClickHouse/ch-go v0.61.5 // indirect
7 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2 // indirect
8 | github.com/andybalholm/brotli v1.1.0 // indirect
9 | github.com/bytedance/sonic v1.11.3 // indirect
10 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
11 | github.com/chenzhuoyu/iasm v0.9.1 // indirect
12 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect
13 | github.com/gin-contrib/cors v1.7.0 // indirect
14 | github.com/gin-contrib/sse v0.1.0 // indirect
15 | github.com/gin-gonic/gin v1.9.1 // indirect
16 | github.com/go-chi/chi/v5 v5.0.12 // indirect
17 | github.com/go-faster/city v1.0.1 // indirect
18 | github.com/go-faster/errors v0.7.1 // indirect
19 | github.com/go-playground/locales v0.14.1 // indirect
20 | github.com/go-playground/universal-translator v0.18.1 // indirect
21 | github.com/go-playground/validator/v10 v10.19.0 // indirect
22 | github.com/goccy/go-json v0.10.2 // indirect
23 | github.com/google/uuid v1.6.0 // indirect
24 | github.com/json-iterator/go v1.1.12 // indirect
25 | github.com/klauspost/compress v1.17.7 // indirect
26 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect
27 | github.com/leodido/go-urn v1.4.0 // indirect
28 | github.com/mattn/go-isatty v0.0.20 // indirect
29 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
30 | github.com/modern-go/reflect2 v1.0.2 // indirect
31 | github.com/paulmach/orb v0.11.1 // indirect
32 | github.com/pelletier/go-toml/v2 v2.2.0 // indirect
33 | github.com/pierrec/lz4/v4 v4.1.21 // indirect
34 | github.com/pkg/errors v0.9.1 // indirect
35 | github.com/segmentio/asm v1.2.0 // indirect
36 | github.com/shopspring/decimal v1.3.1 // indirect
37 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
38 | github.com/ugorji/go/codec v1.2.12 // indirect
39 | go.opentelemetry.io/otel v1.24.0 // indirect
40 | go.opentelemetry.io/otel/trace v1.24.0 // indirect
41 | golang.org/x/arch v0.7.0 // indirect
42 | golang.org/x/crypto v0.21.0 // indirect
43 | golang.org/x/net v0.22.0 // indirect
44 | golang.org/x/sys v0.18.0 // indirect
45 | golang.org/x/text v0.14.0 // indirect
46 | google.golang.org/protobuf v1.33.0 // indirect
47 | gopkg.in/yaml.v3 v3.0.1 // indirect
48 | )
49 |
--------------------------------------------------------------------------------
/app-go/go.sum:
--------------------------------------------------------------------------------
1 | github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
2 | github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
3 | github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
4 | github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
5 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc=
6 | github.com/ClickHouse/clickhouse-go/v2 v2.22.2/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE=
7 | github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
8 | github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
9 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
10 | github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
11 | github.com/bytedance/sonic v1.11.3 h1:jRN+yEjakWh8aK5FzrciUHG8OFXK+4/KrAX/ysEtHAA=
12 | github.com/bytedance/sonic v1.11.3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
13 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
14 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
15 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
16 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
17 | github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
18 | github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
19 | github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
20 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
22 | github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
23 | github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
24 | github.com/gin-contrib/cors v1.7.0 h1:wZX2wuZ0o7rV2/1i7gb4Jn+gW7HBqaP91fizJkBUJOA=
25 | github.com/gin-contrib/cors v1.7.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro=
26 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
27 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
28 | github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
29 | github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
30 | github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
31 | github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
32 | github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
33 | github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
34 | github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
35 | github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
36 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
37 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
38 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
39 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
40 | github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
41 | github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
42 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
43 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
44 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
45 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
46 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
47 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
48 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
49 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
50 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
51 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
52 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
53 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
54 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
55 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
56 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
57 | github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
58 | github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
59 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
60 | github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
61 | github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
62 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
63 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
64 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
65 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
66 | github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
67 | github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
68 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
69 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
70 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
71 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
72 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
73 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
74 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
75 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
76 | github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
77 | github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
78 | github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
79 | github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo=
80 | github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
81 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
82 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
83 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
84 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
85 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
86 | github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
87 | github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
88 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
89 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
90 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
91 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
92 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
93 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
94 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
95 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
96 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
97 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
98 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
99 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
100 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
101 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
102 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
103 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
104 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
105 | github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
106 | github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
107 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
108 | github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
109 | github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
110 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
111 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
112 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
113 | go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
114 | go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
115 | go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
116 | go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
117 | go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
118 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
119 | golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
120 | golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
121 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
122 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
123 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
124 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
125 | golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
126 | golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
127 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
128 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
129 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
130 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
131 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
132 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
133 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
134 | golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
135 | golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
136 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
137 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
138 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
139 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
140 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
141 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
142 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
143 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
144 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
145 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
146 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
147 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
148 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
149 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
150 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
151 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
152 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
153 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
154 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
155 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
156 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
157 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
158 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
159 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
160 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
161 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
162 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
163 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
164 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
165 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
166 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
167 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
168 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
169 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
170 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
171 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
172 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
173 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
174 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
175 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
176 |
--------------------------------------------------------------------------------
/app-go/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "erlog/db"
5 | "erlog/routes"
6 | "net/http"
7 |
8 | "github.com/gin-contrib/cors"
9 | "github.com/gin-gonic/gin"
10 | )
11 |
12 | func main() {
13 | db.ConnectDB()
14 | r := gin.Default()
15 | r.Use(cors.Default())
16 |
17 | r.GET("/", func (c *gin.Context) {
18 | c.JSON(200, gin.H{
19 | "message": "pong",
20 | })
21 | })
22 |
23 | r.POST("/report", routes.Report)
24 | r.POST("/search", routes.Search)
25 |
26 | http.ListenAndServe(":8000", r)
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/app-go/query.sql:
--------------------------------------------------------------------------------
1 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfYear(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute;
2 |
3 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfYear(timestamp) as date, toHour(timestamp) as hour, COUNT(*) as count from metrics GROUP BY hour, date, month, year ORDER BY year, month, date, hour;
4 |
5 | select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute;
--------------------------------------------------------------------------------
/app-go/routes/request.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | type ReportRequestBody struct {
4 | Name string `json:"name"`
5 | }
6 |
7 | type SearchRequestBody struct {
8 | Per string `json:"per"`
9 | }
10 |
--------------------------------------------------------------------------------
/app-go/routes/response.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | type SearchResponse struct {
4 | DateTime string `json:"dateTime"`
5 | Count uint64 `json:"count"`
6 | }
--------------------------------------------------------------------------------
/app-go/routes/routes.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | import (
4 | "context"
5 | "erlog/db"
6 | "fmt"
7 |
8 | "github.com/gin-gonic/gin"
9 | )
10 |
11 | func Search(c *gin.Context) {
12 | var searchQuery SearchRequestBody
13 | err := c.BindJSON(&searchQuery)
14 |
15 | if err != nil {
16 | fmt.Printf("%v\n", err.Error())
17 | c.JSON(400, gin.H{
18 | "error": err.Error(),
19 | })
20 | return
21 | }
22 |
23 | data, err := ExecSearch(searchQuery.Per, "asdf")
24 |
25 | if err != nil {
26 | fmt.Printf("%v\n", err.Error())
27 | c.JSON(400, gin.H{
28 | "error": err.Error(),
29 | })
30 | return
31 | }
32 |
33 | c.JSON(200, data)
34 | }
35 |
36 | func Report(c *gin.Context) {
37 | var body ReportRequestBody
38 | err := c.BindJSON(&body)
39 |
40 | if err != nil {
41 | fmt.Printf("%v\n", err.Error())
42 | c.JSON(400, gin.H{
43 | "error": err.Error(),
44 | })
45 | return
46 | }
47 |
48 | db.Conn.Exec(context.Background(), "INSERT INTO metrics VALUES (generateUUIDv4(), ?, now('Africa/Abidjan'))", body.Name)
49 | c.JSON(200, gin.H{"status": "ok"})
50 | }
--------------------------------------------------------------------------------
/app-go/routes/search.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | import (
4 | "context"
5 | "erlog/db"
6 | "fmt"
7 |
8 | "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
9 | )
10 |
11 |
12 | var perHour = "select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, toMinute(timestamp) as minute, COUNT(*) as count from metrics GROUP BY minute, hour, date, month, year ORDER BY year, month, date, hour, minute;"
13 | var perDay = "select toYear(timestamp) as year, toMonth(timestamp) as month, toDayOfMonth(timestamp) as date, toHour(timestamp) as hour, COUNT(*) as count from metrics GROUP BY date, month, year, hour ORDER BY year, month, date, hour;"
14 |
15 | func ExecSearch(per string, name string) ([]SearchResponse, error) {
16 | var query string
17 | if per == "hour" {
18 | query = perHour
19 | } else if per == "day" {
20 | query = perDay
21 | } else {
22 | fmt.Printf("Not implemented\n")
23 | return nil, fmt.Errorf("not implemented query")
24 | }
25 |
26 | result, err := db.Conn.Query(context.Background(), query)
27 |
28 | if err != nil {
29 | fmt.Printf("%v\n", err.Error())
30 | return nil, err
31 | }
32 |
33 | data, err := GetDataFor(per, result)
34 |
35 | if err != nil {
36 | fmt.Printf("%v\n", err.Error())
37 | return nil, err
38 | }
39 |
40 | return data, nil
41 | }
42 |
43 | // gets the data
44 | func GetDataFor(per string, rows driver.Rows) ([]SearchResponse, error) {
45 | var data []SearchResponse
46 | var err error
47 | if per == "hour" {
48 | data, err = GetDataPerHour(rows, data)
49 | } else if per == "day" {
50 | data, err = GetDataPerDay(rows, data)
51 | } else {
52 | return nil, fmt.Errorf("not implemented")
53 | }
54 |
55 | if err != nil {
56 | fmt.Printf("%v\n", err)
57 | return nil, err
58 | }
59 |
60 | return data, nil
61 | }
62 |
63 | // returns data in per minute intervals
64 | func GetDataPerHour(rows driver.Rows, data []SearchResponse) ([]SearchResponse, error) {
65 | for rows.Next() {
66 | var year uint16
67 | var month uint8
68 | var date uint8
69 | var hour uint8
70 | var minute uint8
71 | var count uint64
72 |
73 | if err := rows.Scan(&year, &month, &date, &hour, &minute, &count); err != nil {
74 | fmt.Printf("%v\n", err.Error())
75 | return nil, err
76 | }
77 |
78 | concatted := fmt.Sprint(date) + " " + fmt.Sprint(hour) + ":" + fmt.Sprint(minute)
79 | fmt.Printf("%v:%v\n", concatted, count)
80 | // data[concatted] = count
81 | data = append(data, SearchResponse{
82 | DateTime: concatted,
83 | Count: count,
84 | })
85 | }
86 |
87 | return data, nil
88 | }
89 |
90 | // returns data in per hour intervals
91 | func GetDataPerDay(rows driver.Rows, data []SearchResponse) ([]SearchResponse, error) {
92 | for rows.Next() {
93 | var year uint16
94 | var month uint8
95 | var date uint8
96 | var hour uint8
97 | var count uint64
98 |
99 | if err := rows.Scan(&year, &month, &date, &hour, &count); err != nil {
100 | fmt.Printf("%v\n", err.Error())
101 | return nil, err
102 | }
103 |
104 | concatted := fmt.Sprint(year) + "-" + fmt.Sprint(month) + "-" + fmt.Sprint(date) + " " + fmt.Sprint(hour)
105 | fmt.Printf("%v:%v\n", concatted, count)
106 | data = append(data, SearchResponse{
107 | DateTime: concatted,
108 | Count: count,
109 | })
110 | }
111 |
112 | return data, nil
113 | }
--------------------------------------------------------------------------------
/app/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | file1.txt
3 | file2.txt
--------------------------------------------------------------------------------
/app/main.py:
--------------------------------------------------------------------------------
1 | from fastapi import FastAPI, HTTPException, Depends
2 | from fastapi import Request, HTTPException
3 | from fastapi.middleware.cors import CORSMiddleware
4 | from app.util import flatten
5 | from app.models import ErLog, User, get_current_user
6 | from app.query import QBuilder
7 | import structlog
8 | from structlog import get_logger
9 | import ujson
10 | import uuid
11 | import secrets
12 | from typing import Annotated
13 | from fastapi.security import OAuth2PasswordBearer
14 | from clickhouse_driver import Client
15 |
16 | api_keys = ["ek-QldMOqfEWSpG_u6VCJv3ng_OD97OiXPDh5Luqvc"]
17 |
18 | # todo: use flask_httpauth
19 | # https://stackoverflow.com/questions/817882/unique-session-id-in-python/6092448#6092448
20 | # https://medium.com/@anubabajide/rest-api-authentication-in-flask-481518a7479b
21 | # instead of token generation use this qhwer u query the db
22 |
23 | structlog.configure(processors=[structlog.processors.JSONRenderer()])
24 |
25 |
26 | logger = get_logger()
27 |
28 | def insert_log(log):
29 | logger = get_logger()
30 | id = str(uuid.uuid4())
31 | if log == "":
32 | return False
33 |
34 | try:
35 | l = ujson.loads(log)
36 | except Exception as e:
37 | logger.error("Failed parsing log json", parent_id=id, e=str(e))
38 | return False
39 |
40 | try:
41 | flattened = flatten(l)
42 | except Exception as e:
43 | logger.error("Failed flattening json", parent_id=id, e=str(e))
44 | return False
45 |
46 | erlog = ErLog(log)
47 | erlog.parse_log(flattened)
48 |
49 | if erlog._id == None:
50 | erlog._id = str(uuid.uuid4())
51 |
52 | if erlog._parent_id == None:
53 | erlog._parent_id = str("00000000-0000-0000-0000-000000000000")
54 | else:
55 | print("updating child logs")
56 | client.execute(
57 | "ALTER TABLE erlogs UPDATE child_logs = child_logs + 1 WHERE id = %(parent_id)s",
58 | {"parent_id": str(erlog._parent_id)},
59 | )
60 |
61 | client.execute(
62 | "INSERT INTO erlogs VALUES",
63 | [
64 | [
65 | str(erlog._id),
66 | str(erlog._parent_id),
67 | erlog._timestamp,
68 | erlog._string_keys,
69 | erlog._string_values,
70 | erlog._bool_keys,
71 | erlog._bool_values,
72 | erlog._number_keys,
73 | erlog._number_values,
74 | erlog._raw_log,
75 | erlog._child_logs,
76 | ]
77 | ],
78 | )
79 |
80 | return erlog._id
81 |
82 |
83 | client = Client(host="localhost", password="test123")
84 |
85 | print("Creating tables..")
86 | # client.execute("CREATE DATABASE IF NOT EXISTS e ENGINE = Atomic;")
87 | res = client.execute(
88 | "CREATE TABLE IF NOT EXISTS erlogs (id UUID primary key, parent_id UUID, timestamp DOUBLE, string_keys Array(String), string_values Array(String), bool_keys Array(String), bool_values Array(Boolean), number_keys Array(String), number_values Array(Double), raw_log String, child_logs UInt32) Engine = MergeTree;"
89 | )
90 | client.execute(
91 | "CREATE TABLE IF NOT EXISTS users (id UUID primary key, email String, hashed_password String) Engine = MergeTree;"
92 | )
93 |
94 | client.execute(
95 | "CREATE TABLE IF NOT EXISTS sessions (id UUID primary key, session_id String, user_id Int32) Engine = MergeTree;"
96 | )
97 |
98 | print("Finished creating tables")
99 |
100 | fake_users_db = {
101 | "johndoe": {
102 | "username": "johndoe",
103 | "full_name": "John Doe",
104 | "email": "johndoe@example.com",
105 | "hashed_password": "fakehashedsecret",
106 | "disabled": False,
107 | },
108 | "alice": {
109 | "username": "alice",
110 | "full_name": "Alice Wonderson",
111 | "email": "alice@example.com",
112 | "hashed_password": "fakehashedsecret2",
113 | "disabled": True,
114 | },
115 | }
116 |
117 |
118 | def hash_password(password: str):
119 | return "hashed" + password
120 |
121 |
122 | def get_user(session_id: str):
123 | user_ids = client.execute(
124 | "SELECT user_id FROM sessions WHERE session_id = %(session_id)s;",
125 | {"session_id": session_id},
126 | )
127 | print(user_ids)
128 | pass
129 |
130 |
131 | app = FastAPI()
132 |
133 | oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
134 |
135 |
136 | @app.get("/users/me")
137 | async def read_me(current_user: Annotated[User, Depends(get_current_user)]):
138 | return current_user
139 |
140 |
141 | @app.get("/items/")
142 | async def items(token: Annotated[str, Depends(oauth2_scheme)]):
143 | return {"token": token}
144 |
145 |
146 | # @app.on_event("startup")
147 | # async def read_logs():
148 | # if not "LOGS" in os.environ:
149 | # logger = get_logger()
150 | # logger.info("No logs in os.environ", l=len(os.environ))
151 | # return
152 |
153 | # loop = asyncio.get_event_loop()
154 | # loop.create_task(read_from_file())
155 |
156 |
157 | origins = [
158 | "http://localhost:5173",
159 | "http://localhost:3000",
160 | "http://localhost",
161 | "http://localhost:59971",
162 | ]
163 |
164 | app.add_middleware(
165 | CORSMiddleware,
166 | allow_origins=origins,
167 | allow_credentials=True,
168 | allow_methods=["*"],
169 | allow_headers=["*"],
170 | )
171 |
172 |
173 | @app.post("/api_key")
174 | async def gen_api_key(request: Request):
175 | key = "ek-" + secrets.token_urlsafe(29)
176 | api_keys.append(key)
177 | return key
178 |
179 |
180 | @app.post("/metrics")
181 | async def metrics(request: Request):
182 | return "ok!"
183 |
184 |
185 | @app.post("/test")
186 | async def test_log(request: Request):
187 | logger = get_logger()
188 | logger.info("hello!")
189 | return "ok"
190 |
191 |
192 | @app.post("/search")
193 | async def root(request: Request):
194 | try:
195 | body = await request.json()
196 | if not isinstance(body, object) or isinstance(body, str):
197 | logger.error("Invalid json", error_details=str(body))
198 | raise HTTPException(status_code=400, detail="Invalid json")
199 |
200 | user_query = body.get("query", "")
201 | page = body.get("page", 0)
202 | show_children = body.get("showChildren", False)
203 | time_range = body.get("timeRange", "all")
204 |
205 | # Convert show_children string to boolean if needed
206 | if isinstance(show_children, str):
207 | show_children = show_children.lower() == "true"
208 |
209 | try:
210 | p = int(page)
211 | except Exception:
212 | logger.error("Invalid page number", page_value=str(page))
213 | raise HTTPException(status_code=400, detail="Page is invalid")
214 |
215 | logger.info("Building query",
216 | user_query=str(user_query),
217 | page=str(p),
218 | show_children=str(show_children),
219 | time_range=str(time_range))
220 |
221 | q = QBuilder()
222 | q.parse(user_query, p, show_children, time_range)
223 | query, params = q.query, q.params
224 |
225 | logger.info("Executing query", query=str(query))
226 | l = client.execute(query, params)
227 |
228 | logs = [
229 | {"id": log[0], "timestamp": log[1], "log": log[2], "child_logs": log[3]}
230 | for log in l
231 | ]
232 |
233 | return logs
234 |
235 | except Exception as e:
236 | logger.error("Search error", error_message=str(e))
237 | raise HTTPException(status_code=500, detail=str(e))
238 |
239 |
240 | @app.post("/get")
241 | async def get_log(request: Request):
242 | body = await request.json()
243 |
244 | if isinstance(body, str):
245 | raise HTTPException(status_code=400, detail="Invalid json")
246 |
247 | id = body["id"]
248 |
249 | if id is None:
250 | raise HTTPException(status_code=400, detail="Invalid json")
251 |
252 | h = client.execute(
253 | "SELECT id, parent_id, timestamp, raw_log from erlogs WHERE id = %(s)s",
254 | {"s": id},
255 | )
256 |
257 | # TODO: check if less than one
258 | log = h[0]
259 |
260 | # print(log[])
261 | # if log[1] != None:
262 | c = client.execute(
263 | "SELECT id, parent_id, timestamp, raw_log from erlogs WHERE parent_id = %(s)s ORDER BY timestamp ASC",
264 | {"s": id},
265 | )
266 |
267 | children = []
268 | clogs = c
269 | for c in clogs:
270 | children.append({"id": c[0], "parent_id": c[1], "timestamp": c[2], "log": c[3]})
271 | # print("hi")
272 | # print(len(child))
273 |
274 | return {"id": log[0], "timestamp": log[2], "log": log[3], "children": children}
275 |
276 |
277 | @app.post("/")
278 | async def log(request: Request):
279 | # if not "Authorization" in request.headers:
280 | # raise HTTPException(401, "Unauthorized")
281 |
282 | # key = request.headers["Authorization"]
283 | # key = key.replace("Bearer ", "")
284 |
285 | # if not key in api_keys:
286 | # raise HTTPException(401, "Unauthorized")
287 |
288 | body = await request.json()
289 | s = ujson.dumps(body)
290 | status = insert_log(s)
291 |
292 | # if isinstance(body, str):
293 | # raise HTTPException(status_code=400, detail="Invalid json")
294 |
295 | # flattened = flatten(body)
296 | # erlog.parse_log(flattened)
297 |
298 | # # todo, use appender or add tis to a batch
299 | # if erlog._id == None:
300 | # erlog._id = uuid.uuid4()
301 |
302 | # conn.execute(
303 | # "INSERT INTO erlogs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
304 | # [
305 | # erlog._id,
306 | # erlog._timestamp,
307 | # erlog._parent_id,
308 | # erlog._string_keys,
309 | # erlog._string_values,
310 | # erlog._bool_keys,
311 | # erlog._bool_values,
312 | # erlog._number_keys,
313 | # erlog._number_values,
314 | # erlog._raw_log,
315 | # ],
316 | # )
317 | return {"status": status}
318 |
319 |
320 | @app.get("/children/{log_id}")
321 | async def get_children(log_id: str):
322 | children = client.execute(
323 | "SELECT id, timestamp, raw_log, child_logs FROM erlogs WHERE parent_id = %(parent_id)s ORDER BY timestamp ASC",
324 | {"parent_id": log_id}
325 | )
326 |
327 | return [
328 | {
329 | "id": child[0],
330 | "timestamp": child[1],
331 | "log": child[2],
332 | "child_logs": child[3]
333 | }
334 | for child in children
335 | ]
336 |
337 |
338 | @app.get("/preview/{log_id}")
339 | async def get_log_preview(log_id: str):
340 | log = client.execute(
341 | """
342 | SELECT id, timestamp, raw_log, child_logs
343 | FROM erlogs
344 | WHERE id = %(log_id)s
345 | LIMIT 1
346 | """,
347 | {"log_id": log_id}
348 | )
349 |
350 | if not log:
351 | raise HTTPException(status_code=404, detail="Log not found")
352 |
353 | return {
354 | "id": log[0][0],
355 | "timestamp": log[0][1],
356 | "log": log[0][2],
357 | "child_logs": log[0][3]
358 | }
359 |
360 |
361 | if __name__ == "__main__":
362 | import uvicorn
363 |
364 | uvicorn.run(app)
365 |
--------------------------------------------------------------------------------
/app/models.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime, timezone
2 | from pydantic import BaseModel
3 | import uuid
4 |
5 |
6 | class User(BaseModel):
7 | username: str
8 | email: str | None = None
9 |
10 |
11 | class DBUser(User):
12 | hashed_password: str
13 |
14 |
15 | def decode_token(token):
16 | return User(username=token + "fakecoded", email="hi@example.com")
17 |
18 |
19 | def get_current_user(token):
20 | user = decode_token(token)
21 | return user
22 |
23 |
24 | class ErLog:
25 | def __init__(self, raw_log):
26 | self._id = None
27 | self._timestamp = 0.00
28 |
29 | self._string_keys = []
30 | self._string_values = []
31 |
32 | self._number_keys = []
33 | self._number_values = []
34 |
35 | self._bool_keys = []
36 | self._bool_values = []
37 |
38 | self._raw_log = raw_log
39 | self._parent_id = None
40 | self._child_logs = 0
41 |
42 | def parse_log(self, log):
43 | for k, v in log.items():
44 | if k == "timestamp":
45 | self._timestamp = float(v)
46 | continue
47 |
48 | if k == "parentId" or k == "parent_id":
49 | self._parent_id = str(v)
50 | continue
51 |
52 | if k == "id":
53 | # uuid parse
54 | self._id = str(v)
55 | continue
56 |
57 | if isinstance(v, str):
58 | self._string_keys.append(k)
59 | self._string_values.append(v)
60 |
61 | elif (
62 | isinstance(v, int)
63 | and not isinstance(v, bool)
64 | or isinstance(v, float)
65 | and not isinstance(v, bool)
66 | ):
67 | self._number_keys.append(k)
68 | self._number_values.append(v)
69 |
70 | elif isinstance(v, bool):
71 | self._bool_keys.append(k)
72 | self._bool_values.append(v)
73 |
74 | if self._timestamp == 0.00:
75 | self._timestamp = datetime.now().timestamp()
76 |
--------------------------------------------------------------------------------
/app/query.py:
--------------------------------------------------------------------------------
1 | from luqum.parser import parser
2 | from luqum.tree import (
3 | SearchField,
4 | Word,
5 | Phrase,
6 | From,
7 | To,
8 | AndOperation,
9 | OrOperation,
10 | Group,
11 | )
12 | import uuid
13 | import time
14 | import structlog
15 |
16 | logger = structlog.get_logger()
17 |
18 | class QBuilder:
19 | def __init__(self):
20 | self.query = "SELECT id, timestamp, raw_log, child_logs FROM erlogs"
21 | self.params = {}
22 | self.where_conditions = []
23 |
24 | def parse(self, user_query, page, show_children, time_range="all"):
25 | try:
26 | # Add time filter
27 | time_filter = self.build_time_filter(time_range)
28 | if time_filter:
29 | self.where_conditions.append(time_filter)
30 |
31 | # Add parent filter
32 | if not show_children:
33 | self.where_conditions.append("parent_id = '00000000-0000-0000-0000-000000000000'")
34 |
35 | # Parse user query if present
36 | if user_query:
37 | try:
38 | tree = parser.parse(user_query)
39 | condition = self.parse_node(tree)
40 | if condition:
41 | self.where_conditions.append(condition)
42 | except Exception as e:
43 | logger.error("Query parse error", error_message=str(e))
44 |
45 | # Combine WHERE conditions
46 | if self.where_conditions:
47 | self.query += " WHERE " + " AND ".join(self.where_conditions)
48 |
49 | # Add ordering and pagination
50 | self.query += " ORDER BY timestamp DESC"
51 | page_param = str(uuid.uuid4())
52 | self.query += f" LIMIT 50 OFFSET %({page_param})s"
53 | self.params[page_param] = int(page * 50)
54 |
55 | return self.query, self.params
56 |
57 | except Exception as e:
58 | logger.error("Query build error", error_message=str(e))
59 | raise
60 |
61 | def parse_node(self, node):
62 | if isinstance(node, AndOperation):
63 | left = self.parse_node(node.children[0])
64 | right = self.parse_node(node.children[1])
65 | return f"({left} AND {right})"
66 |
67 | if isinstance(node, OrOperation):
68 | left = self.parse_node(node.children[0])
69 | right = self.parse_node(node.children[1])
70 | return f"({left} OR {right})"
71 |
72 | if isinstance(node, Group):
73 | return self.parse_node(node.children[0])
74 |
75 | if isinstance(node, SearchField):
76 | return self.parse_field(node)
77 |
78 | if isinstance(node, Word):
79 | return self.parse_word(node.value)
80 |
81 | if isinstance(node, Phrase):
82 | return self.parse_word(node.value.strip('"\''))
83 |
84 | return None
85 |
86 | def parse_field(self, node):
87 | field = node.name
88 | value = node.children[0].value.strip('"\'')
89 |
90 | # Handle special fields
91 | if field in ['id', 'parent_id', 'timestamp']:
92 | param_id = str(uuid.uuid4())
93 | self.params[param_id] = value
94 | return f"{field} = %({param_id})s"
95 |
96 | # Handle array fields
97 | param_id = str(uuid.uuid4())
98 | self.params[param_id] = field
99 | value_param = str(uuid.uuid4())
100 | self.params[value_param] = value
101 |
102 | return f"""
103 | (arrayExists(x -> x = %({param_id})s, string_keys) AND
104 | arrayFirst(i -> string_keys[i] = %({param_id})s, arrayEnumerate(string_keys)) > 0 AND
105 | string_values[arrayFirst(i -> string_keys[i] = %({param_id})s, arrayEnumerate(string_keys))] = %({value_param})s)
106 | OR
107 | (arrayExists(x -> x = %({param_id})s, number_keys) AND
108 | arrayFirst(i -> number_keys[i] = %({param_id})s, arrayEnumerate(number_keys)) > 0 AND
109 | toString(number_values[arrayFirst(i -> number_keys[i] = %({param_id})s, arrayEnumerate(number_keys))]) = %({value_param})s)
110 | OR
111 | (arrayExists(x -> x = %({param_id})s, bool_keys) AND
112 | arrayFirst(i -> bool_keys[i] = %({param_id})s, arrayEnumerate(bool_keys)) > 0 AND
113 | toString(bool_values[arrayFirst(i -> bool_keys[i] = %({param_id})s, arrayEnumerate(bool_keys))]) = %({value_param})s)
114 | """
115 |
116 | def parse_word(self, value):
117 | param_id = str(uuid.uuid4())
118 | self.params[param_id] = f"%{value}%"
119 |
120 | return f"""
121 | arrayExists(x -> position(lower(x), lower(%({param_id})s)) > 0, string_values) OR
122 | arrayExists(x -> position(lower(x), lower(%({param_id})s)) > 0, string_keys) OR
123 | position(lower(raw_log), lower(%({param_id})s)) > 0
124 | """
125 |
126 | def build_time_filter(self, time_range):
127 | if time_range == "all":
128 | return None
129 |
130 | current_time = time.time()
131 | time_filters = {
132 | "1h": current_time - 3600,
133 | "24h": current_time - 86400,
134 | "7d": current_time - 604800,
135 | "30d": current_time - 2592000
136 | }
137 |
138 | if time_range in time_filters:
139 | param_id = str(uuid.uuid4())
140 | self.params[param_id] = time_filters[time_range]
141 | return f"timestamp >= %({param_id})s"
142 |
143 | return None
144 |
145 |
146 | if __name__ == "__main__":
147 | QBuilder().parse("event:null", 1)
148 | # print(type(f))
149 | # print(dir(f))
150 | # print(f.value)
151 | # print(repr(parser.parse('title:"foo bar"')))
152 |
--------------------------------------------------------------------------------
/app/requirements.txt:
--------------------------------------------------------------------------------
1 | annotated-types==0.6.0
2 | anyio==3.7.1
3 | async-tail==0.2.0
4 | click==8.1.7
5 | fastapi==0.103.2
6 | h11==0.14.0
7 | idna==3.4
8 | luqum==0.13.0
9 | maturin==1.3.0
10 | ply==3.11
11 | pydantic==2.4.2
12 | pydantic_core==2.10.1
13 | sh==2.0.6
14 | sniffio==1.3.0
15 | starlette==0.27.0
16 | structlog==23.2.0
17 | typing_extensions==4.8.0
18 | ujson==5.8.0
19 | uvicorn==0.23.2
20 | clickhouse-driver==0.2.8
21 | Flask-Login==0.6.3
22 | M2Crypto==0.41.0
--------------------------------------------------------------------------------
/app/user.py:
--------------------------------------------------------------------------------
1 | class User:
2 | def is_authenticated():
3 | return False
4 |
5 | def is_active():
6 | return True
7 |
8 | def is_anonymous():
9 | return True
10 |
11 | # returns str
12 | def get_id():
13 | pass
14 |
15 | pass
16 |
--------------------------------------------------------------------------------
/app/util.py:
--------------------------------------------------------------------------------
1 | from collections.abc import MutableMapping
2 |
3 |
4 | def flatten(dictionary, parent_key=False, separator="."):
5 | """
6 | Turn a nested dictionary into a flattened dictionary
7 | :param dictionary: The dictionary to flatten
8 | :param parent_key: The string to prepend to dictionary's keys
9 | :param separator: The string used to separate flattened keys
10 | :return: A flattened dictionary
11 | """
12 |
13 | items = []
14 | for key, value in dictionary.items():
15 | new_key = str(parent_key) + separator + key if parent_key else key
16 | if isinstance(value, MutableMapping):
17 | if not value.items():
18 | items.append((new_key, None))
19 | else:
20 | items.extend(flatten(value, new_key, separator).items())
21 | elif isinstance(value, list):
22 | if len(value):
23 | for k, v in enumerate(value):
24 | items.extend(flatten({str(k): v}, new_key, separator).items())
25 | else:
26 | items.append((new_key, None))
27 | else:
28 | items.append((new_key, value))
29 | return dict(items)
30 |
31 |
32 | def isint(s):
33 | try:
34 | int(s)
35 | except ValueError:
36 | return False
37 | else:
38 | return True
39 |
40 |
41 | def isfloat(s):
42 | try:
43 | float(s)
44 | except ValueError:
45 | return False
46 | else:
47 | return True
48 |
--------------------------------------------------------------------------------
/assets/1.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nevinpuri/erlog/da5ec06ed19bac01bc6faf92761dee128701b553/assets/1.jpeg
--------------------------------------------------------------------------------
/assets/2.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nevinpuri/erlog/da5ec06ed19bac01bc6faf92761dee128701b553/assets/2.jpeg
--------------------------------------------------------------------------------
/config/clickhouse/default.xml:
--------------------------------------------------------------------------------
1 |
{err}
125 | {isLoading ? ( 126 |26 | {cvtReadable(JSON.parse(log.log))}{" "} 27 | 28 | +1 29 | 30 |
31 | 32 | {timeConverter(log.timestamp)} 33 | 34 |/metrics "id": "cpu_usage", value: "ifjasdiojf"
*/} 38 |Field | 68 |Value | 69 |
---|---|
{key} | 77 |{value} | 78 |
{JSON.stringify(data, null, 2)}95 |
/metrics "id": "cpu_usage", value: "ifjasdiojf"
*/ 89 | -------------------------------------------------------------------------------- /ui/src/util/index.js: -------------------------------------------------------------------------------- 1 | export function timeConverter(UNIX_timestamp) { 2 | var a = new Date(UNIX_timestamp * 1000); 3 | var months = [ 4 | "Jan", 5 | "Feb", 6 | "Mar", 7 | "Apr", 8 | "May", 9 | "Jun", 10 | "Jul", 11 | "Aug", 12 | "Sep", 13 | "Oct", 14 | "Nov", 15 | "Dec", 16 | ]; 17 | var year = a.getFullYear(); 18 | var month = months[a.getMonth()]; 19 | var date = a.getDate(); 20 | var hour = a.getHours(); 21 | // var min = a.getMinutes(); 22 | // var sec = a.getSeconds(); 23 | 24 | var min = "0" + a.getMinutes(); 25 | // Seconds part from the timestamp 26 | var sec = "0" + a.getSeconds(); 27 | 28 | var ms = a.getMilliseconds(); 29 | var time = 30 | date + 31 | " " + 32 | month + 33 | " " + 34 | year + 35 | " " + 36 | hour + 37 | ":" + 38 | min.slice(-2) + 39 | ":" + 40 | sec.slice(-2) + 41 | "." + 42 | ms; 43 | 44 | return time; 45 | } 46 | 47 | export function cvtReadable(obj) { 48 | let out = ""; 49 | for (const [key, value] of Object.entries(obj)) { 50 | if (key === "id" || key === "parent_id") { 51 | continue; 52 | } 53 | out += ` ${key}=${value}`; 54 | } 55 | 56 | return out.trim(); 57 | } 58 | -------------------------------------------------------------------------------- /ui/src/util/logger.ts: -------------------------------------------------------------------------------- 1 | import winston from "winston"; 2 | 3 | class Logger { 4 | private logger = winston.createLogger({ 5 | transports: [ 6 | new winston.transports.Http({ 7 | host: "localhost", 8 | port: 8000, 9 | }), 10 | ], 11 | }); 12 | 13 | constructor() {} 14 | 15 | public info(data: object) { 16 | this.logger.info("fkas", data); 17 | } 18 | } 19 | 20 | export function getLogger() { 21 | return new Logger(); 22 | } 23 | 24 | let logUrl: string; 25 | export function init(_logUrl: string) { 26 | if (_logUrl.endsWith("/")) { 27 | logUrl = _logUrl.slice(0, _logUrl.length - 1); 28 | } else { 29 | logUrl = _logUrl; 30 | } 31 | } 32 | 33 | /** 34 | * You would use it like 35 | * report("post_created", {userId: 3}) 36 | * @param args 37 | */ 38 | export function report( 39 | name: string, 40 | args: object, 41 | throwError: boolean = false 42 | ) { 43 | // just queue it up or some thing or just send the post request asynchronously 44 | // return a promise, you can await it if you want 45 | return new Promise