├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── RAILWAY.md ├── README.md ├── commands ├── create │ ├── create_test.go │ ├── handler.go │ └── sercive.go ├── grant │ ├── grant_test.go │ ├── handler.go │ └── service.go ├── link │ ├── handler.go │ ├── link_test.go │ └── service.go ├── migrate │ ├── handler.go │ └── migrate_test.go └── update │ ├── handler.go │ ├── service.go │ └── update_test.go ├── connect ├── connect_test.go ├── handler.go └── service.go ├── docker-compose.yml ├── globals └── global.go ├── go.mod ├── go.sum ├── image ├── Drawing 2025-04-04 12.25.23.excalidraw.png ├── connect.svg └── create.svg ├── internal ├── migration.go └── query.go ├── login ├── handler.go ├── login_test.go └── service.go ├── main ├── main.go └── router.go ├── pool ├── pool.go ├── pool_test.go ├── replacer.go └── replacer_test.go ├── railway.toml ├── request_handler ├── request_handler.go └── request_handler_test.go ├── response ├── error.go ├── statements.go └── success.go ├── scripts └── pre-commit ├── server └── config.go ├── storge └── server │ ├── audit.sql │ └── scheme.sql └── utils ├── connection.go ├── password.go ├── stringsUtils.go ├── test.go └── types.go /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | storge/users/* 3 | storge/server/*.sqlite3 4 | temp/ 5 | TCP-Duckdb 6 | build/* 7 | vendor -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24 2 | 3 | WORKDIR /app 4 | 5 | # Install SQLite tools 6 | RUN apt-get update && apt-get install -y sqlite3 && apt-get clean 7 | 8 | # Copy go mod and sum files 9 | COPY go.mod go.sum ./ 10 | 11 | # Download dependencies 12 | RUN go mod download 13 | 14 | # Copy the source code 15 | COPY . . 16 | 17 | # Build the application 18 | RUN go build -o /app/build/server main/* 19 | 20 | # Create directories for storage 21 | RUN mkdir -p /app/storge/server 22 | 23 | # Create SQLite database file and initialize with schema 24 | RUN touch /app/storge/server/db.sqlite3 && \ 25 | sqlite3 /app/storge/server/db.sqlite3 < /app/storge/server/scheme.sql 26 | 27 | # Set environment variables 28 | ENV ServerPort=4000 29 | ENV ServerAddr=0.0.0.0 30 | ENV DBdir=/app/storge/ 31 | ENV ServerDbFile=db.sqlite3 32 | ENV ENCRYPTION_KEY=A15pG0m3hwf0tfpVW6m92eZ6vRmAQA3C 33 | 34 | # Expose the server port 35 | EXPOSE 4000 36 | 37 | # Run the server 38 | CMD ["/app/build/server"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 TCP-DuckDB 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BUILD_DIR := build 2 | SCRIPTS_DIR := scripts 3 | MSG ?= "Default commit message" 4 | 5 | build: 6 | go build -o $(BUILD_DIR)/server main/* 7 | 8 | run: 9 | cd $(BUILD_DIR) && ./server 10 | 11 | runserver: build run 12 | 13 | format: 14 | ./$(SCRIPTS_DIR)/pre-commit 15 | 16 | commit: format 17 | echo "\033[32mstagging changes...\033[0m" 18 | @git add . 19 | echo "\033[32mcommiting changes...\033[0m" 20 | @git commit -m "$(MSG)" 21 | 22 | push: commit 23 | echo "\033[32mpushing to remote repo...\033[0m" 24 | @git push git@github.com:rag-nar1/TCP-Duckdb.git 25 | 26 | clean: 27 | rm -rf $(BUILD_DIR)/* 28 | 29 | .PHONY: build run clean -------------------------------------------------------------------------------- /RAILWAY.md: -------------------------------------------------------------------------------- 1 | # Railway Deployment Instructions 2 | 3 | ## Setup 4 | 5 | 1. Install the Railway CLI: 6 | ``` 7 | npm i -g @railway/cli 8 | ``` 9 | 10 | 2. Login to Railway: 11 | ``` 12 | railway login 13 | ``` 14 | 15 | 3. Link your project: 16 | ``` 17 | railway link 18 | ``` 19 | 20 | 4. Deploy: 21 | ``` 22 | railway up 23 | ``` 24 | 25 | ## Environment Variables 26 | 27 | Make sure these environment variables are set in your Railway project: 28 | 29 | - `ServerPort`: 4000 30 | - `ServerAddr`: 0.0.0.0 31 | - `DBdir`: /app/storge/ 32 | - `ServerDbFile`: server/db.sqlite3 33 | - `ENCRYPTION_KEY`: A15pG0m3hwf0tfpVW6m92eZ6vRmAQA3C 34 | 35 | ## Database 36 | 37 | The SQLite database will be automatically created at `/app/storge/server/db.sqlite3` during the container build and initialized with the schema defined in `scheme.sql`. 38 | 39 | The initialization creates the following tables: 40 | - `user` - User accounts and authentication 41 | - `DB` - Database definitions 42 | - `tables` - Table definitions 43 | - `dbprivilege` - Database access privileges 44 | - `tableprivilege` - Table-specific access privileges 45 | - `postgres` - PostgreSQL connection configurations 46 | 47 | **Note**: Since Railway containers are ephemeral, any data stored in the SQLite database will be lost when the container restarts. For production use, consider using a persistent database service. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TCP-DuckDB Documentation 2 | 3 | ## Table of Contents 4 | 5 | - [Introduction](#introduction) 6 | - [Key Features](#key-features) 7 | - [Setup Guide](#setup-guide) 8 | - [Prerequisites](#prerequisites) 9 | - [Installation](#installation) 10 | - [Tools and Technologies](#tools-and-technologies) 11 | - [Programming Languages](#programming-languages) 12 | - [Database Technologies](#database-technologies) 13 | - [Libraries and Frameworks](#libraries-and-frameworks) 14 | - [Development Tools](#development-tools) 15 | - [Command Reference](#command-reference) 16 | - [1. Login Command](#1-login-command) 17 | - [2. Create Command](#2-create-command) 18 | - [Create Database](#create-database) 19 | - [Create User](#create-user) 20 | - [3. Connect Command](#3-connect-command) 21 | - [4. Grant Command](#4-grant-command) 22 | - [Grant Database Access](#grant-database-access) 23 | - [Grant Table Access](#grant-table-access) 24 | - [5. Link Command](#5-link-command) 25 | - [6. Migrate Command](#6-migrate-command) 26 | - [7. Update Command](#7-update-command) 27 | - [Update Database Name](#update-database-name) 28 | - [Update Username](#update-username) 29 | - [Update User Password](#update-user-password) 30 | - [Transaction Management](#transaction-management) 31 | - [Error Handling](#error-handling) 32 | - [Environment Configuration](#environment-configuration) 33 | - [Deployment](#deployment) 34 | - [Internal Architecture](#internal-architecture) 35 | - [Core Components and Modules](#core-components-and-modules) 36 | - [1. Server Module](#1-server-module-server) 37 | - [2. Main Module](#2-main-module-main) 38 | - [3. Request Handler Module](#3-request-handler-module-request_handler) 39 | - [4. Connection Pool Module](#4-connection-pool-module-pool) 40 | - [5. Login Module](#5-login-module-login) 41 | - [6. Create Module](#6-create-module-create) 42 | - [7. Connect Module](#7-connect-module-connect) 43 | - [8. Grant Module](#8-grant-module-grant) 44 | - [9. Link Module](#9-link-module-link) 45 | - [10. Migrate Module](#10-migrate-module-migrate) 46 | - [11. Update Module](#11-update-module-update) 47 | - [12. Utils Module](#12-utils-module-utils) 48 | - [Request Processing Flow](#request-processing-flow) 49 | - [Todo](#todo) 50 | - [License](#license) 51 | 52 | ## Introduction 53 | 54 | TCP-DuckDB is a TCP server implementation that provides networked access to DuckDB databases. The server enables remote database management with features like user authentication, access control, and PostgreSQL integration. Written in Go, it leverages the power of DuckDB, a lightweight analytical database engine. 55 | 56 | ### Key Features 57 | 58 | - **TCP Interface**: Network-accessible database service 59 | - **User Authentication**: Multi-user support with authentication 60 | - **Database Management**: Create and manage DuckDB databases 61 | - **Permission Control**: Fine-grained access permissions at database and table levels 62 | - **PostgreSQL Integration**: Link with PostgreSQL databases 63 | - **Transaction Support**: Full transaction support for data operations 64 | - **Connection Pooling**: Efficient database connection management 65 | ![alt text](image/Drawing%202025-04-04%2012.25.23.excalidraw.png) 66 | ## Setup Guide 67 | 68 | ### Installation 69 | 70 | ### Prerequisites 71 | 72 | - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on your system 73 | - Git to clone this repository 74 | 75 | ### Quick Start 76 | 77 | 1. Clone the repository: 78 | ```bash 79 | git clone https://github.com/rag-nar1/tcp-duckdb.git 80 | cd TCP-Duckdb 81 | ``` 82 | 83 | 2. Build and start the server using Docker Compose: 84 | ```bash 85 | docker compose up --build 86 | ``` 87 | 88 | 3. Check logs to verify the server is running: 89 | ```bash 90 | docker logs tcp-duckdb-tcp-duckdb-1 91 | ``` 92 | 93 | You should see output like: 94 | ``` 95 | INFO YYYY/MM/DD HH:MM:SS Super user created 96 | INFO YYYY/MM/DD HH:MM:SS listening to 0.0.0.0:4000 97 | ``` 98 | 99 | 4. Stop the server when finished: 100 | ```bash 101 | docker compose down 102 | ``` 103 | 104 | ### Manual Build 105 | 106 | If you prefer to build and run manually: 107 | 108 | 1. Build the Docker image: 109 | ```bash 110 | docker build -t tcp-duckdb . 111 | ``` 112 | 113 | 2. Run the container: 114 | ```bash 115 | docker run -d -p 4000:4000 \ 116 | -v $(pwd)/storge:/app/storge \ 117 | -v $(pwd)/server:/app/server \ 118 | -e ServerPort=4000 \ 119 | -e ServerAddr=0.0.0.0 \ 120 | -e DBdir=/app/storge/server/ \ 121 | -e ServerDbFile=db.sqlite3 \ 122 | -e ENCRYPTION_KEY=A15pG0m3hwf0tfpVW6m92eZ6vRmAQA3C \ 123 | --name tcp-duckdb-container \ 124 | tcp-duckdb 125 | ``` 126 | 127 | ### Configuration 128 | 129 | The server can be configured using environment variables: 130 | 131 | | Variable | Description | Default | 132 | |----------|-------------|---------| 133 | | ServerPort | Port the server listens on | 4000 | 134 | | ServerAddr | Address the server binds to | 0.0.0.0 | 135 | | DBdir | Directory for the SQLite database | /app/storge/server/ | 136 | | ServerDbFile | SQLite database filename | db.sqlite3 | 137 | | ENCRYPTION_KEY | Key used for encryption | `ENCRYPTION_KEY` | 138 | 139 | You can modify these values in the `docker-compose.yml` file or pass them directly when running the container. 140 | 141 | ### Development 142 | 143 | To build and run the application locally: 144 | 145 | 1. Install Go 1.24 or later 146 | 2. Install SQLite development libraries 147 | 3. Clone the repository 148 | 4. Run: 149 | ```bash 150 | go mod download 151 | go build -o ./build/server main/* 152 | ./build/server 153 | ``` 154 | 155 | ### Troubleshooting 156 | 157 | **Database initialization errors** 158 | 159 | If you see errors related to database tables, ensure the schema is correctly applied: 160 | 161 | ```bash 162 | # Connect to the running container 163 | docker exec -it tcp-duckdb-tcp-duckdb-1 bash 164 | 165 | # Verify the database exists 166 | ls -la /app/storge/server/ 167 | 168 | # Manually apply the schema if needed 169 | sqlite3 /app/storge/server/db.sqlite3 < /app/storge/server/scheme.sql 170 | ``` 171 | 172 | **Connection issues** 173 | 174 | The server listens on TCP port 4000. Verify the port is correctly mapped and not blocked by a firewall. 175 | 176 | 1. Clone the repository: 177 | ```bash 178 | git clone https://github.com/rag-nar1/tcp-duckdb.git 179 | cd TCP-Duckdb 180 | ``` 181 | 182 | 2. Install dependencies: 183 | ```bash 184 | go mod download 185 | ``` 186 | 187 | 3. Configure environment variables: 188 | Create or modify the `.env` file in the project root: 189 | ```env 190 | ServerPort=4000 191 | ServerAddr=localhost 192 | DBdir=/path/to/storage/ 193 | ServerDbFile=server/db.sqlite3 194 | ENCRYPTION_KEY="YourEncryptionKey" 195 | ``` 196 | 197 | 4. Build the project: 198 | ```bash 199 | make build 200 | ``` 201 | 202 | 5. Run the server: 203 | ```bash 204 | make run 205 | ``` 206 | 207 | ## Tools and Technologies 208 | 209 | ### Programming Languages 210 | 211 | - **Go (Golang)**: The primary programming language used for the entire codebase. Go was chosen for its efficiency in building networked services, excellent concurrency support through goroutines, and strong standard library. 212 | 213 | ### Database Technologies 214 | 215 | - **DuckDB**: A lightweight, in-process analytical database management system. It serves as the primary storage engine for the application, providing fast analytical query capabilities. 216 | 217 | - **SQLite**: Used for storing user authentication and permission data. SQLite was chosen for its simplicity, reliability, and zero-configuration nature. 218 | 219 | - **PostgreSQL**: Supported as an optional integration, allowing linking and synchronization with PostgreSQL databases. The system can replicate schema and data from PostgreSQL into DuckDB. 220 | 221 | ### Libraries and Frameworks 222 | 223 | - **go-duckdb**: The Go driver for DuckDB that enables interaction with DuckDB databases from Go code. 224 | 225 | - **go-sqlite3**: The Go interface to the SQLite3 database, used for user management. 226 | 227 | - **lib/pq**: PostgreSQL driver for Go, used for connecting to PostgreSQL databases when using the link functionality. 228 | 229 | - **godotenv**: Used for loading environment variables from .env files. 230 | 231 | - **Standard Library Packages**: 232 | - `net`: Core networking functionality for TCP server implementation 233 | - `database/sql`: Database interaction 234 | - `sync`: Synchronization primitives for concurrent operations 235 | - `bufio`: Buffered I/O operations 236 | - `crypto`: Cryptographic functions for secure password hashing 237 | 238 | ### Development Tools 239 | 240 | - **Makefile**: Used for build automation, with predefined tasks for building, running, and code formatting. 241 | 242 | - **Git**: Version control system with custom pre-commit hooks for code formatting. 243 | 244 | - **Environment Configuration**: Uses .env files for configuration management. 245 | 246 | - **Connection Pool Implementation**: Custom LRU (Least Recently Used) cache implementation for efficient database connection management. 247 | 248 | ## Command Reference 249 | 250 | ### 1. Login Command 251 | 252 | **[All Users]** 253 | 254 | Authenticates a user to access the server. This is the first command that must be executed before any other operation can be performed. 255 | 256 | ``` 257 | login [username] [password] 258 | ``` 259 | 260 | **Authentication Process:** 261 | 1. The client sends the login command with username and password 262 | 2. The server validates the credentials against the SQLite user database 263 | 3. If successful, a user session is established with appropriate privileges 264 | 4. All subsequent commands will operate under this authenticated user context 265 | 266 | **Super User Information:** 267 | - The default super user is `duck` with initial password `duck` 268 | - The super user has full administrative privileges including: 269 | - Creating and managing databases 270 | - Creating and managing users 271 | - Granting permissions 272 | - Linking with PostgreSQL databases 273 | - Performing update operations 274 | - For security reasons, it is strongly recommended to change the super user password after initial setup using the update command 275 | 276 | **Example:** 277 | ``` 278 | login duck duck 279 | ``` 280 | Response on success: 281 | ``` 282 | success 283 | ``` 284 | 285 | ### 2. Create Command 286 | 287 | **[Super User Only]** 288 | 289 | #### Create Database 290 | Creates a new DuckDB database (requires super user privileges). 291 | 292 | ``` 293 | create database [database_name] 294 | ``` 295 | 296 | **Example:** 297 | ``` 298 | create database analytics 299 | ``` 300 | Response on success: 301 | ``` 302 | success 303 | ``` 304 | 305 | #### Create User 306 | Creates a new user (requires super user privileges). 307 | 308 | ``` 309 | create user [username] [password] 310 | ``` 311 | 312 | **Example:** 313 | ``` 314 | create user analyst securepassword 315 | ``` 316 | Response on success: 317 | ``` 318 | success 319 | ``` 320 | 321 | ### 3. Connect Command 322 | 323 | **[All Users]** 324 | 325 | Connects to an existing database to execute queries. 326 | 327 | ``` 328 | connect [database_name] 329 | ``` 330 | 331 | After connecting, the system: 332 | - Verifies database existence 333 | - Checks user permissions 334 | - Acquires database connection from pool 335 | - Allows executing queries or starting transactions 336 | 337 | **Example:** 338 | ``` 339 | connect analytics 340 | ``` 341 | Response on success: 342 | ``` 343 | success 344 | ``` 345 | 346 | Once connected, you can execute SQL queries directly: 347 | ``` 348 | SELECT * FROM users; 349 | ``` 350 | 351 | ### 4. Grant Command 352 | 353 | **[Super User Only]** 354 | 355 | #### Grant Database Access 356 | Grants database access to a user (requires super user privileges). 357 | 358 | ``` 359 | grant database [database_name] [username] [access_type] 360 | ``` 361 | 362 | Access types: 363 | - `read`: Read-only access 364 | - `write`: Read and write access 365 | 366 | **Example:** 367 | ``` 368 | grant database analytics analyst read 369 | ``` 370 | Response on success: 371 | ``` 372 | success 373 | ``` 374 | 375 | #### Grant Table Access 376 | Grants table-level permissions to a user (requires super user privileges). 377 | 378 | ``` 379 | grant table [database_name] [table_name] [username] [access_type...] 380 | ``` 381 | 382 | Access types: 383 | - `select`: Permission to query the table 384 | - `insert`: Permission to insert data 385 | - `update`: Permission to update data 386 | - `delete`: Permission to delete data 387 | 388 | **Example:** 389 | ``` 390 | grant table analytics users analyst select insert 391 | ``` 392 | Response on success: 393 | ``` 394 | success 395 | ``` 396 | 397 | ### 5. Link Command 398 | 399 | **[Super User Only]** 400 | 401 | Links a DuckDB database with a PostgreSQL database (requires super user privileges). 402 | 403 | ``` 404 | link [database_name] [postgresql_connection_string] 405 | ``` 406 | 407 | Implementation: 408 | - Connects to the PostgreSQL database 409 | - Retrieves schema information 410 | - Creates corresponding tables in DuckDB 411 | - Copies data from PostgreSQL to DuckDB 412 | - Sets up audit triggers for change tracking 413 | 414 | **Example:** 415 | ``` 416 | link analytics "postgresql://user:password@localhost:5432/analytics_pg" 417 | ``` 418 | Response on success: 419 | ``` 420 | success 421 | ``` 422 | 423 | ### 6. Migrate Command 424 | 425 | **[Super User Only]** 426 | 427 | Synchronizes changes from a linked PostgreSQL database to DuckDB (requires super user privileges). 428 | 429 | ``` 430 | migrate [database_name] 431 | ``` 432 | 433 | Implementation: 434 | - Reads audit logs from PostgreSQL 435 | - Applies changes to the DuckDB database 436 | - Updates tracking information 437 | 438 | **Example:** 439 | ``` 440 | migrate analytics 441 | ``` 442 | Response on success: 443 | ``` 444 | success 445 | ``` 446 | 447 | ### 7. Update Command 448 | 449 | **[Super User Only]** 450 | 451 | Updates database names or user information (requires super user privileges). 452 | 453 | The update command has three variations: 454 | 455 | #### Update Database Name 456 | ``` 457 | update database [old_database_name] [new_database_name] 458 | ``` 459 | 460 | Implementation: 461 | - Verifies database existence 462 | - Renames the database file 463 | - Updates database name in server records 464 | 465 | **Example:** 466 | ``` 467 | update database analytics analytics_prod 468 | ``` 469 | Response on success: 470 | ``` 471 | success 472 | ``` 473 | 474 | #### Update Username 475 | ``` 476 | update user username [old_username] [new_username] 477 | ``` 478 | 479 | Implementation: 480 | - Verifies user existence 481 | - Updates username in user database 482 | 483 | **Example:** 484 | ``` 485 | update user username analyst data_scientist 486 | ``` 487 | Response on success: 488 | ``` 489 | success 490 | ``` 491 | 492 | #### Update User Password 493 | ``` 494 | update user password [username] [new_password] 495 | ``` 496 | 497 | Implementation: 498 | - Verifies user existence 499 | - Hashes the new password 500 | - Updates password in user database 501 | 502 | **Example:** 503 | ``` 504 | update user password analyst new_secure_password 505 | ``` 506 | Response on success: 507 | ``` 508 | success 509 | ``` 510 | 511 | ## Transaction Management 512 | 513 | **[All Users with Database Access]** 514 | 515 | After connecting to a database, you can manage transactions: 516 | 517 | #### Start Transaction 518 | ``` 519 | start transaction 520 | ``` 521 | 522 | #### Execute Queries in Transaction 523 | ``` 524 | INSERT INTO users VALUES (1, 'John'); 525 | UPDATE users SET name = 'Johnny' WHERE id = 1; 526 | ``` 527 | 528 | #### Commit Transaction 529 | ``` 530 | commit 531 | ``` 532 | 533 | #### Rollback Transaction 534 | ``` 535 | rollback 536 | ``` 537 | 538 | **Example:** 539 | ``` 540 | connect analytics 541 | start transaction 542 | INSERT INTO users VALUES (1, 'John'); 543 | UPDATE users SET name = 'Johnny' WHERE id = 1; 544 | commit 545 | ``` 546 | 547 | ## Error Handling 548 | 549 | The server implements structured error responses: 550 | - `response.BadRequest(writer)` 551 | - `response.InternalError(writer)` 552 | - `response.UnauthorizedError(writer)` 553 | - `response.DoesNotExistDatabse(writer, dbname)` 554 | - `response.AccesDeniedOverDatabase(writer, UserName, dbname)` 555 | 556 | ## Environment Configuration 557 | 558 | The server uses the following environment variables: 559 | - `ServerPort`: TCP port for the server (default: 4000) 560 | - `ServerAddr`: Server address (default: localhost) 561 | - `DBdir`: Directory for storing databases 562 | - `ServerDbFile`: Path to the server's SQLite database 563 | - `ENCRYPTION_KEY`: Key for encrypting/decrypting PostgreSQL connection strings 564 | 565 | ## Deployment 566 | 567 | The server can be deployed on any system with Go and DuckDB installed: 568 | 569 | 1. Build the server: 570 | ```bash 571 | make build 572 | ``` 573 | 574 | 2. Configure environment in `.env` 575 | 576 | 3. Run the server: 577 | ```bash 578 | make run 579 | ``` 580 | 581 | ## Internal Architecture 582 | 583 | ### Core Components and Modules 584 | 585 | #### 1. Server Module (`server/`) 586 | 587 | The main server component is responsible for the core server functionality: 588 | 589 | - **Configuration Management**: Loads environment variables and configures the server 590 | - **Database Connection**: Maintains connection to the SQLite user database 591 | - **Statement Preparation**: Prepares SQL statements for efficient execution 592 | - **Super User Management**: Creates and manages the super user account 593 | - **Logging**: Implements structured logging for errors and information 594 | 595 | **Key Files**: 596 | - `config.go`: Server configuration and initialization 597 | - `server.go`: Core server functionality implementation 598 | 599 | **Key Functions**: 600 | - `NewServer()`: Initializes server with configurations 601 | - `CreateSuper()`: Creates the initial super user if it doesn't exist 602 | - `PrepareStmt()`: Prepares SQL statements for later use 603 | 604 | The Server struct centralizes server state: 605 | ```go 606 | type Server struct { 607 | Sqlitedb *sql.DB // SQLite database connection 608 | Dbstmt map[string]*sql.Stmt // Prepared statements 609 | Pool *request_handler.RequestHandler // Connection pool 610 | Port string // Server port 611 | Address string // Full server address 612 | InfoLog *log.Logger // Information logger 613 | ErrorLog *log.Logger // Error logger 614 | } 615 | ``` 616 | 617 | #### 2. Main Module (`main/`) 618 | 619 | The entry point for the TCP server: 620 | 621 | - **Server Initialization**: Initializes the server components 622 | - **TCP Listener**: Sets up TCP socket and listens for connections 623 | - **Connection Handling**: Accepts connections and spawns goroutines 624 | - **Command Routing**: Routes incoming commands to appropriate handlers 625 | 626 | **Key Files**: 627 | - `main.go`: Entry point with TCP listener 628 | - `router.go`: Command routing implementation 629 | 630 | **Key Functions**: 631 | - `main()`: Starts the TCP server listening on configured port 632 | - `HandleConnection()`: Processes each client connection 633 | - `Router()`: Routes requests to appropriate command handlers 634 | 635 | #### 3. Request Handler Module (`request_handler/`) 636 | 637 | Manages the lifecycle of database requests: 638 | 639 | - **Request Queue**: Maintains queue of database connection requests 640 | - **Connection Pooling Integration**: Works with pool module 641 | - **Concurrency Management**: Handles simultaneous requests safely 642 | 643 | **Key Files**: 644 | - `request_handler.go`: Core request handling logic 645 | 646 | **Key Functions**: 647 | - `HandleRequest()`: Processes each database request 648 | - `Spin()`: Starts the request handling background process 649 | - `Push()`: Adds new requests to the queue 650 | 651 | #### 4. Connection Pool Module (`pool/`) 652 | 653 | Implements efficient connection management for DuckDB databases: 654 | 655 | - **LRU Cache**: Implements Least Recently Used replacement policy 656 | - **Connection Limits**: Manages maximum number of open connections 657 | - **Pin Counting**: Tracks active database usage 658 | - **Resource Management**: Efficiently manages database handles 659 | 660 | **Key Files**: 661 | - `pool.go`: Connection pool implementation 662 | - `lru.go`: LRU cache implementation for connection eviction 663 | 664 | **Key Functions**: 665 | - `Get()`: Retrieves a database connection from the pool 666 | - `NewPool()`: Creates a new connection pool 667 | - `RecordAccess()`: Updates access time for LRU tracking 668 | 669 | #### 5. Login Module (`login/`) 670 | 671 | Handles user authentication: 672 | 673 | - **Credential Verification**: Validates username and password 674 | - **Password Hashing**: Securely stores and validates passwords 675 | - **Session Establishment**: Sets up user session after authentication 676 | 677 | **Key Files**: 678 | - `handler.go`: Authentication request handling 679 | - `service.go`: Authentication logic implementation 680 | 681 | **Key Functions**: 682 | - `Handler()`: Processes login requests 683 | - `Login()`: Validates credentials against database 684 | 685 | #### 6. Create Module (`create/`) 686 | 687 | Manages creation of databases and users: 688 | 689 | - **Database Creation**: Creates new DuckDB databases 690 | - **User Creation**: Creates new user accounts 691 | - **Permission Initialization**: Sets up initial permissions 692 | 693 | **Key Files**: 694 | - `handler.go`: Request handling for creation operations 695 | - `service.go`: Implementation of creation operations 696 | 697 | **Key Functions**: 698 | - `CreateDatabase()`: Creates a new database 699 | - `CreateUser()`: Creates a new user 700 | 701 | #### 7. Connect Module (`connect/`) 702 | 703 | Manages database connections and query execution: 704 | 705 | - **Connection Establishment**: Connects to specified database 706 | - **Permission Checking**: Verifies user has access to database 707 | - **Query Execution**: Executes SQL queries on connected database 708 | - **Transaction Management**: Handles SQL transactions 709 | 710 | **Key Files**: 711 | - `handler.go`: Connection request handling 712 | - `service.go`: Query execution implementation 713 | - `transaction.go`: Transaction management 714 | 715 | **Key Functions**: 716 | - `Handler()`: Processes connection requests 717 | - `QueryService()`: Executes individual queries 718 | - `Transaction()`: Manages database transactions 719 | 720 | #### 8. Grant Module (`grant/`) 721 | 722 | Manages access permissions: 723 | 724 | - **Database Permissions**: Controls database access rights 725 | - **Table Permissions**: Controls table-level access rights 726 | - **Permission Checking**: Verifies permissions before granting 727 | 728 | **Key Files**: 729 | - `handler.go`: Grant request handling 730 | - `service.go`: Permission management implementation 731 | 732 | **Key Functions**: 733 | - `GrantDatabaseAccess()`: Grants database-level access 734 | - `GrantTableAccess()`: Grants table-level access 735 | 736 | #### 9. Link Module (`link/`) 737 | 738 | Facilitates PostgreSQL database integration: 739 | 740 | - **Connection Management**: Establishes connections to PostgreSQL 741 | - **Schema Transfer**: Replicates PostgreSQL schema to DuckDB 742 | - **Data Migration**: Copies data from PostgreSQL to DuckDB 743 | - **Connection String Encryption**: Securely stores PostgreSQL credentials 744 | 745 | **Key Files**: 746 | - `handler.go`: Link request handling 747 | - `service.go`: Link implementation 748 | 749 | **Key Functions**: 750 | - `Link()`: Establishes connection and migrates schema/data 751 | 752 | #### 10. Migrate Module (`migrate/`) 753 | 754 | Handles data synchronization between PostgreSQL and DuckDB: 755 | 756 | - **Change Detection**: Identifies changes in PostgreSQL 757 | - **Synchronization**: Applies changes to DuckDB 758 | - **Audit Log**: Processes audit logs for changes 759 | 760 | **Key Files**: 761 | - `handler.go`: Migrate request handling 762 | - `service.go`: Migration implementation 763 | 764 | **Key Functions**: 765 | - `Migrate()`: Synchronizes changes from PostgreSQL to DuckDB 766 | 767 | #### 11. Update Module (`update/`) 768 | 769 | Manages updates to database and user information: 770 | 771 | - **Database Name Updates**: Renames databases 772 | - **User Information Updates**: Updates usernames and passwords 773 | - **Validation**: Verifies existence before updates 774 | 775 | **Key Files**: 776 | - `handler.go`: Update request handling 777 | - `service.go`: Update implementation 778 | 779 | **Key Functions**: 780 | - `UpdateDatabase()`: Renames a database 781 | - `UpdateUserUsername()`: Updates a user's username 782 | - `UpdateUserPassword()`: Updates a user's password 783 | 784 | #### 12. Utils Module (`utils/`) 785 | 786 | Provides utility functions used throughout the application: 787 | 788 | - **Password Hashing**: Secures user passwords 789 | - **Encryption**: Handles AES encryption for sensitive data 790 | - **Path Management**: Manages file paths for databases 791 | - **String Handling**: Provides string manipulation utilities 792 | 793 | **Key Files**: 794 | - `utils.go`: General utility functions 795 | - `crypto.go`: Cryptographic functions 796 | 797 | **Key Functions**: 798 | - `Hash()`: Hashes passwords 799 | - `Encrypt()/Decrypt()`: Encrypts/decrypts data with AES 800 | - `UserDbPath()`: Resolves database file paths 801 | 802 | ### Request Processing Flow 803 | 804 | 1. **Connection Establishment** 805 | - Client connects to TCP server via `main.go` 806 | - Server spawns a goroutine for the connection via `HandleConnection()` 807 | - Client must authenticate via `login.Handler()` 808 | 809 | 2. **Command Processing** 810 | - After authentication, `Router()` in `main/router.go` processes requests 811 | - Commands are parsed and validated 812 | - Requests are routed to appropriate module handlers 813 | - Responses are sent back to client 814 | 815 | 3. **Database Operations** 816 | - Database connections are managed by the pool module 817 | - Operations are checked against user permissions 818 | - Transactions are handled with ACID guarantees 819 | - Results are returned to client 820 | 821 | 4. **PostgreSQL Integration Process** 822 | - Link operations copy schema and data from PostgreSQL 823 | - Migrate operations synchronize changes from PostgreSQL 824 | - Audit tables track changes for synchronization 825 | 826 | ## Todo 827 | 828 | This section outlines planned enhancements and improvements for the TCP-DuckDB project: 829 | 830 | | Task | Description | Priority | 831 | |------|-------------|----------| 832 | | Client Library | Develop client libraries in multiple languages (Python, JavaScript, Java), go is in progress | High | 833 | | Connection Encryption | Implement TLS/SSL for secure client-server communication | High | 834 | | Change Data Capture | Swap the Audit table with CDC using [Debezium](https://debezium.io/) and [kafka](https://kafka.apache.org/) | Medium | 835 | | Backup & Restore | Add automated backup and point-in-time recovery functionality | Medium | 836 | | Query Caching | Add intelligent query result caching | Low | 837 | | Web Admin Interface | Create a web-based administration interface | Low | 838 | 839 | ## License 840 | 841 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -------------------------------------------------------------------------------- /commands/create/create_test.go: -------------------------------------------------------------------------------- 1 | package commands_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/utils" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestCreateDb tests the creation of a new database. 11 | func TestCreateDb(t *testing.T) { 12 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 13 | conn := utils.StartUp() // Start up the connection 14 | err := utils.LoginAsAdmin(conn) // Log in as admin 15 | assert.Nil(t, err) // Ensure no error during login 16 | err = utils.CreateDB(conn, "db1") // Create a new database named "db1" 17 | assert.Nil(t, err) // Ensure no error during database creation 18 | } 19 | 20 | // TestCreateDbAlreadyExists tests the behavior when trying to create a database that already exists. 21 | func TestCreateDbAlreadyExists(t *testing.T) { 22 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 23 | conn := utils.StartUp() // Start up the connection 24 | err := utils.LoginAsAdmin(conn) // Log in as admin 25 | assert.Nil(t, err) // Ensure no error during login 26 | err = utils.CreateDB(conn, "db1") // Create a new database named "db1" 27 | assert.Nil(t, err) // Ensure no error during first database creation 28 | err = utils.CreateDB(conn, "db1") // Try to create the same database again 29 | assert.NotNil(t, err) // Ensure an error is returned when creating a database that already exists 30 | } 31 | 32 | // TestCreateDbAndConnect tests the creation of a new database and connecting to it. 33 | func TestCreateDbAndConnect(t *testing.T) { 34 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 35 | conn := utils.StartUp() // Start up the connection 36 | err := utils.LoginAsAdmin(conn) // Log in as admin 37 | assert.Nil(t, err) // Ensure no error during login 38 | err = utils.CreateDB(conn, "db1") // Create a new database named "db1" 39 | assert.Nil(t, err) // Ensure no error during database creation 40 | err = utils.ConnectDb(conn, "db1") // Connect to the newly created database 41 | assert.Nil(t, err) // Ensure no error during database connection 42 | } 43 | 44 | // TestCreateUser tests the creation of a new user. 45 | func TestCreateUser(t *testing.T) { 46 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 47 | conn := utils.StartUp() // Start up the connection 48 | err := utils.LoginAsAdmin(conn) // Log in as admin 49 | assert.Nil(t, err) // Ensure no error during login 50 | err = utils.CreateUser(conn, "ragnar", "ragnar") // Create a new user with username and password "ragnar" 51 | assert.Nil(t, err) // Ensure no error during user creation 52 | } 53 | 54 | // TestCreateUserAlreadyExists tests the behavior when trying to create a user that already exists. 55 | func TestCreateUserAlreadyExists(t *testing.T) { 56 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 57 | conn := utils.StartUp() // Start up the connection 58 | err := utils.LoginAsAdmin(conn) // Log in as admin 59 | assert.Nil(t, err) // Ensure no error during login 60 | err = utils.CreateUser(conn, "ragnar", "ragnar") // Create a new user with username and password "ragnar" 61 | assert.Nil(t, err) // Ensure no error during first user creation 62 | err = utils.CreateUser(conn, "ragnar", "ragnar") // Try to create the same user again 63 | assert.NotNil(t, err) // Ensure an error is returned when creating a user that already exists 64 | } 65 | -------------------------------------------------------------------------------- /commands/create/handler.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/response" 7 | 8 | _ "github.com/lib/pq" 9 | _ "github.com/marcboeker/go-duckdb" 10 | ) 11 | 12 | // database [dbname], 13 | // user [username] [password] 14 | func Handler(privilege string, req []string, writer *bufio.Writer) { 15 | if privilege != "super" { 16 | response.UnauthorizedError(writer) 17 | return 18 | } 19 | 20 | if (req[0] != "database" && req[0] != "user") || (req[0] == "database" && len(req) != 2) || (req[0] == "user" && len(req) != 3) { 21 | response.BadRequest(writer) 22 | return 23 | } 24 | 25 | if req[0] == "database" { 26 | CreateDB(req[1], writer) 27 | return 28 | } 29 | CreateUser(req[1], req[2], writer) 30 | } 31 | -------------------------------------------------------------------------------- /commands/create/sercive.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | 5 | "bufio" 6 | "database/sql" 7 | "os" 8 | 9 | "github.com/rag-nar1/tcp-duckdb/response" 10 | "github.com/rag-nar1/tcp-duckdb/server" 11 | "github.com/rag-nar1/tcp-duckdb/utils" 12 | 13 | _ "github.com/lib/pq" 14 | _ "github.com/marcboeker/go-duckdb" 15 | ) 16 | 17 | func CreateDB(dbname string, writer *bufio.Writer) { 18 | var DBID int 19 | err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID) 20 | if err == nil && DBID != 0 { 21 | response.Error(writer, []byte("database: "+dbname+" already exists")) 22 | return 23 | } 24 | 25 | // create file 26 | _, err = sql.Open("duckdb", os.Getenv("DBdir")+"users/"+dbname+".db") 27 | if err != nil { 28 | server.Serv.ErrorLog.Fatal(err) 29 | } 30 | 31 | _, err = server.Serv.Dbstmt["CreateDB"].Exec(dbname) 32 | if err != nil { 33 | response.Error(writer, []byte("could not create databse")) 34 | 35 | err = os.Remove(os.Getenv("DBdir") + "users/" + dbname + ".db") 36 | if err != nil { 37 | server.Serv.ErrorLog.Fatal(err) 38 | } 39 | return 40 | } 41 | response.Success(writer) 42 | } 43 | 44 | func CreateUser(NewUser, password string, writer *bufio.Writer) { 45 | 46 | // create user 47 | _, err := server.Serv.Dbstmt["CreateUser"].Exec(NewUser, utils.Hash(password), "norm") 48 | 49 | if err != nil { 50 | response.Error(writer, []byte("user already exists\n")) 51 | server.Serv.ErrorLog.Println(err) 52 | return 53 | } 54 | 55 | response.Success(writer) 56 | } 57 | -------------------------------------------------------------------------------- /commands/grant/grant_test.go: -------------------------------------------------------------------------------- 1 | package commands_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/utils" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestBasicGrantOverDb(t *testing.T) { 11 | dbname := "db1" 12 | username := "ragnar" 13 | password := "ragnar" 14 | t.Cleanup(utils.CleanUp) // Clean up resources after test 15 | 16 | conn := utils.StartUp() // Start up the database connection 17 | err := utils.LoginAsAdmin(conn) 18 | assert.Nil(t, err) // Ensure no error during admin login 19 | 20 | err = utils.CreateDB(conn, dbname) 21 | assert.Nil(t, err) // Ensure no error during database creation 22 | 23 | err = utils.CreateUser(conn, username, password) 24 | assert.Nil(t, err) // Ensure no error during user creation 25 | 26 | err = utils.GrantDb(conn, username, dbname, "read") 27 | assert.Nil(t, err) // Ensure no error during granting read access to the user 28 | 29 | conn2 := utils.Connection() // Create a new connection 30 | err = utils.Login(conn2, username, password) 31 | assert.Nil(t, err) // Ensure no error during user login 32 | 33 | err = utils.ConnectDb(conn2, dbname) 34 | assert.Nil(t, err) // Ensure no error during database connection 35 | } 36 | 37 | func TestGrantInvalidPermission(t *testing.T) { 38 | dbname := "db3" 39 | username := "doe" 40 | password := "doe" 41 | t.Cleanup(utils.CleanUp) // Clean up resources after test 42 | 43 | conn := utils.StartUp() // Start up the database connection 44 | err := utils.LoginAsAdmin(conn) 45 | assert.Nil(t, err) // Ensure no error during admin login 46 | 47 | err = utils.CreateDB(conn, dbname) 48 | assert.Nil(t, err) // Ensure no error during database creation 49 | 50 | err = utils.CreateUser(conn, username, password) 51 | assert.Nil(t, err) // Ensure no error during user creation 52 | 53 | err = utils.GrantDb(conn, username, dbname, "invalid_permission") 54 | assert.NotNil(t, err) // Ensure error during granting invalid permission 55 | } 56 | 57 | func TestGrantWithoutDb(t *testing.T) { 58 | username := "alice" 59 | password := "alice" 60 | t.Cleanup(utils.CleanUp) // Clean up resources after test 61 | 62 | conn := utils.StartUp() // Start up the database connection 63 | err := utils.LoginAsAdmin(conn) 64 | assert.Nil(t, err) // Ensure no error during admin login 65 | 66 | err = utils.CreateUser(conn, username, password) 67 | assert.Nil(t, err) // Ensure no error during user creation 68 | 69 | err = utils.GrantDb(conn, username, "non_existent_db", "read") 70 | assert.NotNil(t, err) // Ensure error during granting access to non-existent database 71 | } 72 | 73 | func TestGrantMultiplePermissions(t *testing.T) { 74 | dbname := "db4" 75 | username := "bob" 76 | password := "bob" 77 | t.Cleanup(utils.CleanUp) // Clean up resources after test 78 | 79 | conn := utils.StartUp() // Start up the database connection 80 | err := utils.LoginAsAdmin(conn) 81 | assert.Nil(t, err) // Ensure no error during admin login 82 | 83 | err = utils.CreateDB(conn, dbname) 84 | assert.Nil(t, err) // Ensure no error during database creation 85 | 86 | err = utils.CreateUser(conn, username, password) 87 | assert.Nil(t, err) // Ensure no error during user creation 88 | 89 | err = utils.GrantDb(conn, username, dbname, "read") 90 | assert.Nil(t, err) // Ensure no error during granting read access 91 | 92 | err = utils.GrantDb(conn, username, dbname, "write") 93 | assert.Nil(t, err) // Ensure no error during granting write access 94 | 95 | conn2 := utils.Connection() // Create a new connection 96 | err = utils.Login(conn2, username, password) 97 | assert.Nil(t, err) // Ensure no error during user login 98 | 99 | err = utils.ConnectDb(conn2, dbname) 100 | assert.Nil(t, err) // Ensure no error during database connection 101 | } 102 | 103 | func TestGrantOverTable(t *testing.T) { 104 | dbname := "db4" 105 | username := "bob" 106 | password := "bob" 107 | tablename := "t1" 108 | t.Cleanup(utils.CleanUp) // Clean up resources after test 109 | 110 | conn := utils.StartUp() // Start up the database connection 111 | err := utils.LoginAsAdmin(conn) 112 | assert.Nil(t, err) // Ensure no error during admin login 113 | 114 | err = utils.CreateDB(conn, dbname) 115 | assert.Nil(t, err) // Ensure no error during database creation 116 | 117 | err = utils.CreateUser(conn, username, password) 118 | assert.Nil(t, err) // Ensure no error during user creation 119 | 120 | err = utils.GrantDb(conn, username, dbname, "read") 121 | assert.Nil(t, err) // Ensure no error during granting database read access 122 | 123 | err = utils.ConnectDb(conn, dbname) 124 | assert.Nil(t, err) // Ensure no error during database connection 125 | 126 | err = utils.CreateTable(conn, tablename) 127 | assert.Nil(t, err) // Ensure no error during table creation 128 | 129 | conn2 := utils.Connection() // Create a new connection 130 | err = utils.Login(conn2, username, password) 131 | assert.Nil(t, err) // Ensure no error during user login 132 | 133 | err = utils.ConnectDb(conn2, dbname) 134 | assert.Nil(t, err) // Ensure no error during database connection 135 | 136 | err = utils.Query(conn2, "select * from t1;") 137 | assert.NotNil(t, err) // Ensure error during query without table access 138 | 139 | err = utils.Query(conn2, "insert into t1(id, name) values(1, 'ragnar');") 140 | assert.NotNil(t, err) // Ensure error during insert without table access 141 | 142 | conn = utils.Connection() // Create a new connection 143 | err = utils.LoginAsAdmin(conn) 144 | assert.Nil(t, err) // Ensure no error during admin login 145 | 146 | err = utils.GrantTable(conn, username, dbname, tablename, "select") 147 | assert.Nil(t, err) // Ensure no error during granting table select access 148 | 149 | err = utils.Query(conn2, "select * from t1;") 150 | assert.Nil(t, err) // Ensure no error during query after granting select access 151 | 152 | err = utils.Query(conn2, "insert into t1(id, name) values(1, 'ragnar');") 153 | assert.NotNil(t, err) // Ensure error during insert without insert access 154 | 155 | err = utils.GrantTable(conn, username, dbname, tablename, "insert") 156 | assert.NotNil(t, err) // Ensure error during granting insert access without write access 157 | 158 | err = utils.GrantDb(conn, username, dbname, "write") 159 | assert.Nil(t, err) // Ensure no error during granting database write access 160 | 161 | err = utils.GrantTable(conn, username, dbname, tablename, "insert") 162 | assert.Nil(t, err) // Ensure no error during granting table insert access 163 | 164 | err = utils.Query(conn2, "insert into t1(id, name) values(1, 'ragnar');") 165 | assert.Nil(t, err) // Ensure no error during insert after granting insert access 166 | } 167 | -------------------------------------------------------------------------------- /commands/grant/handler.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | 6 | response "github.com/rag-nar1/tcp-duckdb/response" 7 | ) 8 | 9 | // database [dbname] [username] [accesstype] , 10 | // table [dbname] [tablename] [username] [accesstype] 11 | func Handler(privilege string, req []string, writer *bufio.Writer) { 12 | if privilege != "super" { 13 | response.UnauthorizedError(writer) 14 | return 15 | } 16 | if (req[0] != "database" && req[0] != "table") || (req[0] == "database" && len(req) != 4) || (req[0] == "table" && len(req) < 5) { 17 | response.BadRequest(writer) 18 | return 19 | } 20 | 21 | if req[0] == "database" { 22 | GrantDB(writer, req[1], req[2], req[3]) 23 | return 24 | } 25 | 26 | GrantTable(writer, req[1], req[2], req[3], req[4:]...) 27 | } 28 | -------------------------------------------------------------------------------- /commands/grant/service.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | "strings" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/response" 8 | "github.com/rag-nar1/tcp-duckdb/server" 9 | "github.com/rag-nar1/tcp-duckdb/utils" 10 | ) 11 | 12 | func GrantDB(writer *bufio.Writer, dbname, username, accesstype string) { 13 | accesstype = strings.ToLower(accesstype) 14 | // check for DB access 15 | if accesstype != "read" && accesstype != "write" { 16 | response.BadRequest(writer) 17 | return 18 | } 19 | // get DBID , UID 20 | var DBID, UID int 21 | err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID) 22 | if err != nil { 23 | response.DoesNotExistDatabse(writer, dbname) 24 | return 25 | } 26 | 27 | err = server.Serv.Dbstmt["SelectUser"].QueryRow(username).Scan(&UID) 28 | if err != nil { 29 | response.DoesNotExistUser(writer, username) 30 | return 31 | } 32 | 33 | // grant access 34 | transaction, err := server.Serv.Sqlitedb.Begin() 35 | if err != nil { 36 | response.InternalError(writer) 37 | server.Serv.ErrorLog.Println(err) 38 | return 39 | } 40 | defer transaction.Rollback() 41 | 42 | if _, err := transaction.Stmt(server.Serv.Dbstmt["DeleteDbAccess"]).Exec(UID, DBID); err != nil { 43 | response.InternalError(writer) 44 | server.Serv.ErrorLog.Println(err) 45 | return 46 | } 47 | 48 | if _, err := transaction.Stmt(server.Serv.Dbstmt["GrantDB"]).Exec(DBID, UID, accesstype); err != nil { 49 | response.InternalError(writer) 50 | server.Serv.ErrorLog.Println(err) 51 | return 52 | } 53 | 54 | if err := transaction.Commit(); err != nil { 55 | response.InternalError(writer) 56 | server.Serv.ErrorLog.Println(err) 57 | return 58 | } 59 | response.Success(writer) 60 | } 61 | 62 | func GrantTable(writer *bufio.Writer, dbname, tablename, username string, accesstypes ...string) { 63 | accesstypes = utils.ToLower(accesstypes...) 64 | // check for DB access 65 | for _, accesstype := range accesstypes { 66 | if accesstype != "select" && accesstype != "insert" && accesstype != "update" && accesstype != "delete" { 67 | response.BadRequest(writer) 68 | return 69 | } 70 | } 71 | 72 | var DBID, UID, TID int 73 | err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID) 74 | if err != nil { 75 | response.DoesNotExistDatabse(writer, dbname) 76 | return 77 | } 78 | 79 | err = server.Serv.Dbstmt["SelectUser"].QueryRow(username).Scan(&UID) 80 | if err != nil { 81 | response.DoesNotExistUser(writer, username) 82 | return 83 | } 84 | 85 | err = server.Serv.Dbstmt["SelectTable"].QueryRow(tablename, DBID).Scan(&TID) 86 | if err != nil { 87 | response.DoesNotExistTables(writer, tablename) 88 | return 89 | } 90 | 91 | var DbAccessType string 92 | err = server.Serv.Dbstmt["DbAccessType"].QueryRow(UID, DBID).Scan(&DbAccessType) 93 | if err != nil { 94 | response.InternalError(writer) 95 | return 96 | } 97 | 98 | if DbAccessType == "read" { 99 | for _, accesstype := range accesstypes { 100 | if accesstype != "select" { 101 | response.UnauthorizedError(writer) 102 | return 103 | } 104 | } 105 | } 106 | 107 | transaction, err := server.Serv.Sqlitedb.Begin() 108 | if err != nil { 109 | response.InternalError(writer) 110 | server.Serv.ErrorLog.Println(err) 111 | return 112 | } 113 | defer transaction.Rollback() 114 | 115 | for _, accesstype := range accesstypes { 116 | _, err := transaction.Stmt(server.Serv.Dbstmt["GrantTable"]).Exec(TID, UID, accesstype) 117 | if err != nil { 118 | response.InternalError(writer) 119 | server.Serv.ErrorLog.Println(err) 120 | return 121 | } 122 | } 123 | 124 | if err := transaction.Commit(); err != nil { 125 | response.InternalError(writer) 126 | server.Serv.ErrorLog.Println(err) 127 | return 128 | } 129 | 130 | response.Success(writer) 131 | } 132 | -------------------------------------------------------------------------------- /commands/link/handler.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/response" 8 | "github.com/rag-nar1/tcp-duckdb/server" 9 | "github.com/rag-nar1/tcp-duckdb/utils" 10 | 11 | _ "github.com/lib/pq" 12 | ) 13 | 14 | // link dbname connStr 15 | func Handler(privilege string, req []string, writer *bufio.Writer) { 16 | if privilege != "super" { 17 | response.UnauthorizedError(writer) 18 | return 19 | } 20 | 21 | dbname, connStr := req[0], req[1] 22 | // check the existince of the database 23 | var DBID int 24 | err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID) 25 | if err != nil { 26 | response.DoesNotExistDatabse(writer, dbname) 27 | server.Serv.ErrorLog.Println(err) 28 | return 29 | } 30 | 31 | var hasLink int 32 | err = server.Serv.Dbstmt["CheckLink"].QueryRow(DBID).Scan(&hasLink) 33 | if err != nil || hasLink > 0 { 34 | response.Error(writer, []byte("database: "+dbname+" already linked")) 35 | server.Serv.ErrorLog.Println(err) 36 | return 37 | } 38 | 39 | DbConn, err := utils.OpenDb(server.Serv.Pool, dbname) 40 | if err != nil { 41 | response.InternalError(writer) 42 | server.Serv.ErrorLog.Println(err) 43 | return 44 | } 45 | defer DbConn.Destroy() 46 | 47 | // check the connStr 48 | postgres, err := sql.Open("postgres", connStr) 49 | if err != nil { 50 | response.InternalError(writer) 51 | server.Serv.ErrorLog.Println(err) 52 | return 53 | } 54 | defer postgres.Close() 55 | 56 | Link(writer, DbConn.DB(), postgres, connStr, DBID) 57 | } 58 | -------------------------------------------------------------------------------- /commands/link/link_test.go: -------------------------------------------------------------------------------- 1 | package commands_test 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/utils" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // TestBasicLink tests the basic linking of a database 13 | func TestBasicLink(t *testing.T) { 14 | dbname := "db2" 15 | connStr := "postgresql://postgres:1242003@localhost:5432/testdb" 16 | t.Cleanup(utils.CleanUp) 17 | 18 | conn := utils.StartUp() 19 | defer conn.Close() 20 | err := utils.LoginAsAdmin(conn) 21 | assert.Nil(t, err) // Ensure no error during login as admin 22 | 23 | err = utils.CreateDB(conn, dbname) 24 | assert.Nil(t, err) // Ensure no error during database creation 25 | 26 | err = utils.Link(conn, dbname, connStr) 27 | assert.Nil(t, err) // Ensure no error during linking database 28 | 29 | err = utils.ConnectDb(conn, dbname) 30 | assert.Nil(t, err) // Ensure no error during connecting to the database 31 | 32 | err = utils.Query(conn, "select * from t1;") 33 | assert.Nil(t, err) // Ensure no error during querying the database 34 | } 35 | 36 | // TestDoubleLink tests linking the same database to two different connections 37 | func TestDoubleLink(t *testing.T) { 38 | dbname := "db1" 39 | connStr := "postgresql://postgres:1242003@localhost:5432/testdb" 40 | connStr2 := "postgresql://postgres:1242003@localhost:5432/testdb2" 41 | t.Cleanup(utils.CleanUp) 42 | 43 | conn := utils.StartUp() 44 | defer conn.Close() 45 | err := utils.LoginAsAdmin(conn) 46 | assert.Nil(t, err) // Ensure no error during login as admin 47 | 48 | err = utils.CreateDB(conn, dbname) 49 | assert.Nil(t, err) // Ensure no error during database creation 50 | 51 | err = utils.Link(conn, dbname, connStr) 52 | assert.Nil(t, err) // Ensure no error during linking first connection 53 | 54 | err = utils.Link(conn, dbname, connStr2) 55 | log.Println(err) 56 | assert.NotNil(t, err) // Ensure error during linking second connection 57 | } 58 | 59 | // TestNormLink tests linking a database with a normal user 60 | func TestNormLink(t *testing.T) { 61 | dbname := "db3" 62 | connStr := "postgresql://postgres:1242003@localhost:5432/testdb" 63 | t.Cleanup(utils.CleanUp) 64 | 65 | conn := utils.StartUp() 66 | defer conn.Close() 67 | err := utils.LoginAsAdmin(conn) 68 | assert.Nil(t, err) // Ensure no error during login as admin 69 | 70 | err = utils.CreateDB(conn, dbname) 71 | assert.Nil(t, err) // Ensure no error during database creation 72 | 73 | err = utils.CreateUser(conn, "ragnar", "ragnar") 74 | assert.Nil(t, err) // Ensure no error during user creation 75 | 76 | conn = utils.StartUp() 77 | defer conn.Close() 78 | err = utils.Login(conn, "ragnar", "ragnar") 79 | assert.Nil(t, err) // Ensure no error during login as normal user 80 | 81 | err = utils.Link(conn, dbname, connStr) 82 | assert.NotNil(t, err) // Ensure error during linking database with normal user 83 | } 84 | 85 | // TestClean tests the cleanup utility 86 | func TestClean(t *testing.T) { 87 | utils.CleanUp() 88 | } 89 | -------------------------------------------------------------------------------- /commands/link/service.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "os" 7 | 8 | "github.com/rag-nar1/tcp-duckdb/internal" 9 | "github.com/rag-nar1/tcp-duckdb/response" 10 | "github.com/rag-nar1/tcp-duckdb/server" 11 | "github.com/rag-nar1/tcp-duckdb/utils" 12 | ) 13 | 14 | func Link(writer *bufio.Writer, duck, postgres *sql.DB, connStr string, DBID int) { 15 | 16 | encryptedConnStr, err := utils.Encrypt(connStr, []byte(os.Getenv("ENCRYPTION_KEY"))) 17 | if err != nil { 18 | response.InternalError(writer) 19 | server.Serv.ErrorLog.Println(err) 20 | return 21 | } 22 | 23 | // start a transaction to insert the connstr 24 | txServer, err := server.Serv.Sqlitedb.Begin() 25 | if err != nil { 26 | response.InternalError(writer) 27 | server.Serv.ErrorLog.Println(err) 28 | return 29 | } 30 | defer txServer.Rollback() 31 | 32 | txDuck, err := duck.Begin() 33 | if err != nil { 34 | response.InternalError(writer) 35 | server.Serv.ErrorLog.Println(err) 36 | return 37 | } 38 | defer txDuck.Rollback() 39 | 40 | txPg, err := postgres.Begin() 41 | if err != nil { 42 | response.InternalError(writer) 43 | server.Serv.ErrorLog.Println(err) 44 | return 45 | } 46 | defer txPg.Rollback() 47 | 48 | // insert the connstr 49 | _, err = txServer.Stmt(server.Serv.Dbstmt["CreateLink"]).Exec(DBID, encryptedConnStr) 50 | if err != nil { 51 | response.InternalError(writer) 52 | server.Serv.ErrorLog.Println(err) 53 | return 54 | } 55 | 56 | // migrate schema 57 | err = internal.Migrate(DBID, connStr, server.Serv.Dbstmt["CreateTable"], txPg, txDuck, txServer) 58 | if err != nil { 59 | server.Serv.ErrorLog.Println(err) 60 | response.InternalError(writer) 61 | return 62 | } 63 | 64 | err = internal.Audit(txPg) 65 | if err != nil { 66 | server.Serv.ErrorLog.Println(err) 67 | response.InternalError(writer) 68 | return 69 | } 70 | 71 | err = txPg.Commit() 72 | if err != nil { 73 | server.Serv.ErrorLog.Println(err) 74 | response.InternalError(writer) 75 | return 76 | } 77 | err = txDuck.Commit() 78 | if err != nil { 79 | server.Serv.ErrorLog.Println(err) 80 | response.InternalError(writer) 81 | return 82 | } 83 | err = txServer.Commit() 84 | if err != nil { 85 | server.Serv.ErrorLog.Println(err) 86 | response.InternalError(writer) 87 | return 88 | } 89 | 90 | response.Success(writer) 91 | } 92 | -------------------------------------------------------------------------------- /commands/migrate/handler.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/rag-nar1/tcp-duckdb/internal" 5 | "github.com/rag-nar1/tcp-duckdb/response" 6 | "github.com/rag-nar1/tcp-duckdb/server" 7 | "github.com/rag-nar1/tcp-duckdb/utils" 8 | 9 | "bufio" 10 | "os" 11 | 12 | "github.com/jmoiron/sqlx" 13 | _ "github.com/lib/pq" 14 | _ "github.com/marcboeker/go-duckdb" 15 | ) 16 | 17 | func Handler(privilege string, req []string, writer *bufio.Writer) { 18 | if privilege != "super" { 19 | response.UnauthorizedError(writer) 20 | return 21 | } 22 | 23 | dbname := req[0] 24 | // check the existince of the database 25 | var DBID int 26 | 27 | if err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID); err != nil { 28 | response.DoesNotExistDatabse(writer, dbname) 29 | server.Serv.ErrorLog.Println(err) 30 | return 31 | } 32 | 33 | var connStrEncrypted string 34 | 35 | if err := server.Serv.Dbstmt["SelectLink"].QueryRow(DBID).Scan(&connStrEncrypted); err != nil { 36 | response.Error(writer, []byte("database: "+dbname+" is not linked to any postgreSQL database\n")) 37 | server.Serv.ErrorLog.Println(err) 38 | return 39 | } 40 | 41 | connStr, err := utils.Decrypt(connStrEncrypted, []byte(os.Getenv("ENCRYPTION_KEY"))) 42 | if err != nil { 43 | response.InternalError(writer) 44 | server.Serv.ErrorLog.Println(err) 45 | return 46 | } 47 | 48 | // open duckdb 49 | Dbconn, err := utils.OpenDb(server.Serv.Pool, dbname) 50 | if err != nil { 51 | response.InternalError(writer) 52 | server.Serv.ErrorLog.Println(err) 53 | return 54 | } 55 | defer Dbconn.Destroy() 56 | 57 | duck := sqlx.NewDb(Dbconn.DB(), "duckdb") 58 | postgres, err := sqlx.Open("postgres", connStr) 59 | if err != nil { 60 | response.InternalError(writer) 61 | server.Serv.ErrorLog.Println(err) 62 | return 63 | } 64 | defer postgres.Close() 65 | 66 | if err := internal.ReadAudit(duck, postgres); err != nil { 67 | response.InternalError(writer) 68 | server.Serv.ErrorLog.Println(err) 69 | return 70 | } 71 | 72 | utils.Write(writer, []byte("migration is successful")) 73 | } 74 | -------------------------------------------------------------------------------- /commands/migrate/migrate_test.go: -------------------------------------------------------------------------------- 1 | package commands_test 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/utils" 8 | 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | 13 | _ "github.com/lib/pq" 14 | ) 15 | 16 | func Insert(connStr string) error { 17 | pq, err := sql.Open("postgres", connStr) 18 | if err != nil { 19 | return err 20 | } 21 | defer pq.Close() 22 | 23 | _, err = pq.Exec("insert into t1(id) values(777);") 24 | if err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | 30 | func Insert100(connStr string) error { 31 | pq, err := sql.Open("postgres", connStr) 32 | if err != nil { 33 | return err 34 | } 35 | defer pq.Close() 36 | for i := 4; i <= 100; i++ { 37 | _, err = pq.Exec("insert into t1(id) values($1);", i) 38 | if err != nil { 39 | return err 40 | } 41 | } 42 | 43 | return nil 44 | } 45 | 46 | type TestRow struct { 47 | Id int `json:"id"` 48 | } 49 | 50 | func TestBasicMigrate(t *testing.T) { 51 | dbname := "db2" 52 | connStr := "postgresql://postgres:1242003@localhost:5432/testdb" 53 | t.Cleanup(utils.CleanUp) 54 | 55 | conn := utils.StartUp() 56 | defer conn.Close() 57 | 58 | err := utils.LoginAsAdmin(conn) 59 | assert.Nil(t, err) // Ensure no error during login as admin 60 | 61 | err = utils.CreateDB(conn, dbname) 62 | assert.Nil(t, err) // Ensure no error during database creation 63 | 64 | err = utils.Link(conn, dbname, connStr) 65 | assert.Nil(t, err) // Ensure no error during linking database 66 | 67 | err = Insert(connStr) 68 | assert.Nil(t, err) // Ensure no error during linking database 69 | 70 | err = utils.Migrate(conn, dbname) 71 | assert.Nil(t, err) 72 | 73 | err = utils.ConnectDb(conn, dbname) 74 | assert.Nil(t, err) // Ensure no error during connecting to the database 75 | 76 | data, err := utils.QueryData(conn, "select * from t1 where id == 777;") 77 | assert.Nil(t, err) 78 | 79 | res := make([]TestRow, 1) 80 | if err := json.Unmarshal([]byte(data), &res); err != nil { 81 | assert.Fail(t, err.Error()) 82 | } 83 | 84 | assert.Equal(t, 777, res[0].Id) 85 | } 86 | 87 | func TestBigMigration(t *testing.T) { 88 | dbname := "db1" 89 | connStr := "postgresql://postgres:1242003@localhost:5432/testdb" 90 | t.Cleanup(utils.CleanUp) 91 | 92 | conn := utils.StartUp() 93 | defer conn.Close() 94 | 95 | err := utils.LoginAsAdmin(conn) 96 | assert.Nil(t, err) // Ensure no error during login as admin 97 | 98 | err = utils.CreateDB(conn, dbname) 99 | assert.Nil(t, err) // Ensure no error during database creation 100 | 101 | err = utils.Link(conn, dbname, connStr) 102 | assert.Nil(t, err) // Ensure no error during linking database 103 | 104 | err = Insert100(connStr) 105 | assert.Nil(t, err) // Ensure no error during linking database 106 | 107 | err = utils.ConnectDb(conn, dbname) 108 | assert.Nil(t, err) // Ensure no error during connecting to the database 109 | 110 | data, err := utils.QueryData(conn, "select max(id) as id from t1;") 111 | assert.Nil(t, err) 112 | 113 | res := make([]TestRow, 1) 114 | if err := json.Unmarshal([]byte(data), &res); err != nil { 115 | assert.Fail(t, err.Error()) 116 | } 117 | assert.Equal(t, 3, res[0].Id) 118 | conn.Close() 119 | 120 | conn = utils.Connection() 121 | err = utils.LoginAsAdmin(conn) 122 | assert.Nil(t, err) // Ensure no error during login as admin 123 | 124 | err = utils.Migrate(conn, dbname) 125 | assert.Nil(t, err) 126 | 127 | err = utils.ConnectDb(conn, dbname) 128 | assert.Nil(t, err) // Ensure no error during connecting to the database 129 | 130 | data, err = utils.QueryData(conn, "select max(id) as id from t1;") 131 | assert.Nil(t, err) 132 | 133 | if err := json.Unmarshal([]byte(data), &res); err != nil { 134 | assert.Fail(t, err.Error()) 135 | } 136 | assert.Equal(t, 100, res[0].Id) 137 | } 138 | -------------------------------------------------------------------------------- /commands/update/handler.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/response" 7 | ) 8 | 9 | // update databse [dbname] newdbname 10 | // update user username [username] [newusername] 11 | // update user password [username] [password] 12 | func Handler(privilege string, req []string, writer *bufio.Writer) { 13 | if privilege != "super" { 14 | response.UnauthorizedError(writer) 15 | return 16 | } 17 | 18 | if req[0] != "database" && req[0] != "user" { 19 | response.BadRequest(writer) 20 | return 21 | } 22 | 23 | if req[0] == "database" && len(req) != 3{ 24 | response.BadRequest(writer) 25 | return 26 | } 27 | 28 | if req[0] == "user" && len(req) != 4 { 29 | response.BadRequest(writer) 30 | return 31 | } 32 | 33 | if req[0] == "database" { 34 | UpdateDatabase(writer, req[1], req[2]) 35 | return 36 | } 37 | 38 | if req[1] != "username" && req[1] != "password" { 39 | response.BadRequest(writer) 40 | return 41 | } 42 | 43 | if req[1] == "username" { 44 | UpdateUserUsername(writer, req[2], req[3]) 45 | return 46 | } 47 | 48 | UpdateUserPassword(writer, req[2], req[3]) 49 | } -------------------------------------------------------------------------------- /commands/update/service.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/response" 8 | "github.com/rag-nar1/tcp-duckdb/server" 9 | "github.com/rag-nar1/tcp-duckdb/utils" 10 | ) 11 | 12 | 13 | 14 | func UpdateDatabase(writer *bufio.Writer, oldDbname, newDbName string) { 15 | var DBID int 16 | if err := server.Serv.Dbstmt["SelectDB"].QueryRow(oldDbname).Scan(&DBID); err != nil { 17 | response.DoesNotExistDatabse(writer, oldDbname) 18 | server.Serv.ErrorLog.Println(err) 19 | return 20 | } 21 | 22 | if err := os.Rename(utils.UserDbPath(oldDbname), utils.UserDbPath(newDbName)); err != nil { 23 | response.InternalError(writer) 24 | server.Serv.ErrorLog.Println(err) 25 | return 26 | } 27 | 28 | if _, err := server.Serv.Dbstmt["UpdateDB"].Exec(newDbName, DBID); err != nil { 29 | if err := os.Rename(utils.UserDbPath(newDbName), utils.UserDbPath(oldDbname)); err != nil { 30 | response.InternalError(writer) 31 | server.Serv.ErrorLog.Println(err) 32 | } 33 | response.InternalError(writer) 34 | server.Serv.ErrorLog.Println(err) 35 | return 36 | } 37 | 38 | response.Success(writer) 39 | } 40 | 41 | func UpdateUserUsername(writer *bufio.Writer, oldUserName, NewUserName string) { 42 | var UID int 43 | if err := server.Serv.Dbstmt["SelectUser"].QueryRow(oldUserName).Scan(&UID); err != nil { 44 | response.DoesNotExistUser(writer, oldUserName) 45 | server.Serv.ErrorLog.Println(err) 46 | return 47 | } 48 | 49 | if _, err := server.Serv.Dbstmt["UpdateUsername"].Exec(NewUserName, UID); err != nil { 50 | response.InternalError(writer) 51 | server.Serv.ErrorLog.Println(err) 52 | return 53 | } 54 | response.Success(writer) 55 | } 56 | 57 | func UpdateUserPassword(writer *bufio.Writer, UserName, password string) { 58 | var UID int 59 | if err := server.Serv.Dbstmt["SelectUser"].QueryRow(UserName).Scan(&UID); err != nil { 60 | response.DoesNotExistUser(writer, UserName) 61 | server.Serv.ErrorLog.Println(err) 62 | return 63 | } 64 | 65 | password = utils.Hash(password) 66 | if _, err := server.Serv.Dbstmt["UpdateUserPassword"].Exec(password, UID); err != nil { 67 | response.InternalError(writer) 68 | server.Serv.ErrorLog.Println(err) 69 | return 70 | } 71 | response.Success(writer) 72 | } -------------------------------------------------------------------------------- /commands/update/update_test.go: -------------------------------------------------------------------------------- 1 | package commands_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/utils" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type TestRow struct { 12 | Id int `json:"id"` 13 | } 14 | func TestUpdateUser(t *testing.T) { 15 | t.Cleanup(utils.CleanUp) 16 | 17 | username := "mohamed" 18 | password := "mohamed" 19 | newusername := "fathy" 20 | newpassword := "fathy" 21 | 22 | conn := utils.StartUp() 23 | defer conn.Close() 24 | 25 | err := utils.LoginAsAdmin(conn) 26 | assert.Nil(t, err) // Ensure no error during login as admin 27 | 28 | err = utils.CreateUser(conn, username, password) 29 | assert.Nil(t, err) 30 | 31 | conn2 := utils.Connection() 32 | defer conn2.Close() 33 | 34 | err = utils.Login(conn2, username, password) 35 | assert.Nil(t, err) 36 | conn2.Close() 37 | 38 | err = utils.UpdateUserName(conn, username, newusername) 39 | assert.Nil(t, err) 40 | 41 | conn2 = utils.Connection() 42 | err = utils.Login(conn2, newusername, password) 43 | assert.Nil(t, err) 44 | conn2.Close() 45 | 46 | conn2 = utils.Connection() 47 | err = utils.Login(conn2, username, password) 48 | assert.NotNil(t, err) 49 | conn2.Close() 50 | 51 | err = utils.UpdateUserName(conn, username, newusername) 52 | assert.NotNil(t, err) 53 | 54 | err = utils.UpdateUserPassword(conn, newusername, newpassword) 55 | assert.Nil(t, err) 56 | 57 | conn2 = utils.Connection() 58 | err = utils.Login(conn2, newusername, password) 59 | assert.NotNil(t, err) 60 | conn2.Close() 61 | 62 | conn2 = utils.Connection() 63 | err = utils.Login(conn2, newusername, newpassword) 64 | assert.Nil(t, err) 65 | conn2.Close() 66 | } 67 | 68 | 69 | func TestUpdateDatabase(t *testing.T) { 70 | t.Cleanup(utils.CleanUp) 71 | 72 | dbname := "db1" 73 | newdbname := "db2" 74 | 75 | conn := utils.StartUp() 76 | defer conn.Close() 77 | 78 | err := utils.LoginAsAdmin(conn) 79 | assert.Nil(t, err) // Ensure no error during login as admin 80 | 81 | err = utils.CreateDB(conn,dbname) 82 | assert.Nil(t, err) 83 | 84 | err = utils.UpdateDatabase(conn, dbname, newdbname) 85 | assert.Nil(t, err) 86 | 87 | err = utils.ConnectDb(conn, newdbname) 88 | assert.Nil(t, err) 89 | conn.Close() 90 | 91 | conn = utils.Connection() 92 | err = utils.LoginAsAdmin(conn) 93 | assert.Nil(t, err) // Ensure no error during login as admin 94 | 95 | err = utils.ConnectDb(conn, dbname) 96 | assert.NotNil(t, err) 97 | } 98 | 99 | func TestUpdateDatabase2(t *testing.T) { 100 | t.Cleanup(utils.CleanUp) 101 | 102 | dbname := "db1" 103 | newdbname := "db2" 104 | 105 | conn := utils.StartUp() 106 | defer conn.Close() 107 | 108 | err := utils.LoginAsAdmin(conn) 109 | assert.Nil(t, err) // Ensure no error during login as admin 110 | 111 | err = utils.CreateDB(conn,dbname) 112 | assert.Nil(t, err) 113 | 114 | err = utils.UpdateDatabase(conn, dbname, newdbname) 115 | assert.Nil(t, err) 116 | 117 | err = utils.ConnectDb(conn, newdbname) 118 | assert.Nil(t, err) 119 | 120 | err = utils.Query(conn, "create table t1(id int);") 121 | assert.Nil(t, err) 122 | 123 | err = utils.Query(conn, "insert into t1(id) values(1);") 124 | assert.Nil(t, err) 125 | conn.Close() 126 | 127 | conn = utils.Connection() 128 | err = utils.LoginAsAdmin(conn) 129 | assert.Nil(t, err) // Ensure no error during login as admin 130 | 131 | err = utils.ConnectDb(conn, newdbname) 132 | assert.Nil(t, err) 133 | 134 | data, err := utils.QueryData(conn, "select * from t1;") 135 | assert.Nil(t, err) 136 | 137 | res := make([]TestRow, 1) 138 | if err := json.Unmarshal([]byte(data), &res); err != nil { 139 | assert.Fail(t, err.Error()) 140 | } 141 | 142 | assert.Equal(t, 1, res[0].Id) 143 | } 144 | 145 | -------------------------------------------------------------------------------- /connect/connect_test.go: -------------------------------------------------------------------------------- 1 | package connect_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/utils" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | // TestConnectBasic tests the basic connection flow 12 | func TestConnectBasic(t *testing.T) { 13 | t.Cleanup(utils.CleanUp) 14 | conn := utils.Connection() // Establish a new connection 15 | err := utils.LoginAsAdmin(conn) // Login as admin 16 | assert.Nil(t, err) // Ensure no error occurred during login 17 | err = utils.CreateDB(conn, "db1") // Create a new database named "db1" 18 | assert.Nil(t, err) // Ensure no error occurred during database creation 19 | err = utils.ConnectDb(conn, "db1") // Connect to the newly created database 20 | assert.Nil(t, err) // Ensure no error occurred during database connection 21 | } 22 | 23 | // TestConnectFail tests the connection to a non-existent database 24 | func TestConnectFail(t *testing.T) { 25 | t.Cleanup(utils.CleanUp) 26 | conn := utils.Connection() // Establish a new connection 27 | err := utils.LoginAsAdmin(conn) // Login as admin 28 | assert.Nil(t, err) // Ensure no error occurred during login 29 | err = utils.ConnectDb(conn, "doesn't_exist") // Attempt to connect to a non-existent database 30 | assert.NotNil(t, err) // Ensure an error occurred during database connection 31 | } 32 | -------------------------------------------------------------------------------- /connect/handler.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "bufio" 5 | 6 | "strings" 7 | 8 | "github.com/rag-nar1/tcp-duckdb/request_handler" 9 | "github.com/rag-nar1/tcp-duckdb/response" 10 | "github.com/rag-nar1/tcp-duckdb/server" 11 | "github.com/rag-nar1/tcp-duckdb/utils" 12 | ) 13 | 14 | // connect dbname 15 | func Handler(UID int, UserName, privilege, dbname string, reader *bufio.Reader, writer *bufio.Writer) { 16 | // check for db existense 17 | var DBID int 18 | if err := server.Serv.Dbstmt["SelectDB"].QueryRow(dbname).Scan(&DBID); err != nil { 19 | response.DoesNotExistDatabse(writer, dbname) 20 | return 21 | } 22 | 23 | // check for authrization 24 | var access int 25 | if err := server.Serv.Dbstmt["CheckDbAccess"].QueryRow(UID, DBID).Scan(&access); err != nil { 26 | response.InternalError(writer) 27 | server.Serv.ErrorLog.Println(err) 28 | return 29 | } 30 | 31 | if access == 0 && privilege != "super" { 32 | response.AccesDeniedOverDatabase(writer, UserName, dbname) 33 | return 34 | } 35 | 36 | req := request_handler.NewRequest(dbname) 37 | server.Serv.Pool.Push(req) 38 | 39 | DbConn, err := utils.OpenDb(server.Serv.Pool, dbname) 40 | if err != nil { 41 | response.InternalError(writer) 42 | server.Serv.ErrorLog.Println(err) 43 | return 44 | } 45 | defer DbConn.Destroy() 46 | 47 | response.Success(writer) 48 | 49 | buffer := make([]byte, 4096) 50 | for { 51 | n, err := reader.Read(buffer) 52 | if err != nil { 53 | response.InternalError(writer) 54 | server.Serv.ErrorLog.Println(err) 55 | return 56 | } 57 | 58 | query := strings.ToLower(strings.Split(string(buffer[0:n]), " ")[0]) 59 | 60 | if query == "start" { 61 | if strings.ToLower(utils.Trim(string(buffer[0:n]))) != "start transaction" { 62 | response.BadRequest(writer) 63 | continue 64 | } 65 | Transaction(DbConn, UID, DBID, UserName, dbname, privilege, reader, writer) 66 | continue 67 | } 68 | 69 | // single query 70 | QueryService(DbConn, utils.Trim(string(buffer[0:n])), UserName, dbname, privilege, UID, DBID, writer) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /connect/service.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/rag-nar1/tcp-duckdb/internal" 10 | "github.com/rag-nar1/tcp-duckdb/response" 11 | "github.com/rag-nar1/tcp-duckdb/utils" 12 | "github.com/rag-nar1/tcp-duckdb/pool" 13 | "github.com/rag-nar1/tcp-duckdb/server" 14 | ) 15 | 16 | func QueryService(Dbconn pool.Connection,query, username, dbname, privilege string, UID, DBID int, writer *bufio.Writer) { 17 | 18 | data, err := Query(query, privilege, UID, DBID, Dbconn.DB()) 19 | if err != nil { 20 | server.Serv.ErrorLog.Println(err) 21 | if err.Error() == fmt.Errorf(response.UnauthorizedMSG).Error() { 22 | response.UnauthorizedError(writer) 23 | return 24 | } 25 | if err.Error() == fmt.Errorf(response.AccessDeniedMsg).Error() { 26 | response.AccesDeniedOverTables(writer, username, data) 27 | return 28 | } 29 | response.Error(writer, []byte(err.Error())) 30 | return 31 | } 32 | 33 | response.WriteData(writer, data) 34 | } 35 | 36 | func Transaction(Dbconn pool.Connection, UID, DBID int, username, dbname, privilege string, reader *bufio.Reader, writer *bufio.Writer) { 37 | buffer := make([]byte, 4096) 38 | transaction, err := Dbconn.DB().Begin() 39 | if err != nil { 40 | response.InternalError(writer) 41 | server.Serv.ErrorLog.Println(err) 42 | return 43 | } 44 | defer transaction.Rollback() 45 | 46 | for { 47 | n, err := reader.Read(buffer) 48 | if err != nil { 49 | response.InternalError(writer) 50 | server.Serv.ErrorLog.Println("ERROR", err) 51 | return 52 | } 53 | 54 | query := strings.ToLower(utils.Trim(string(buffer[0:n]))) 55 | if strings.HasPrefix(query, "rollback") { 56 | return 57 | } 58 | if strings.HasPrefix(query, "commit") { 59 | err = transaction.Commit() 60 | if err != nil { 61 | server.Serv.ErrorLog.Println(err) 62 | response.Error(writer, []byte(err.Error())) 63 | } 64 | return 65 | } 66 | 67 | data, err := Query(query, privilege, UID, DBID, transaction) 68 | if err != nil { 69 | server.Serv.ErrorLog.Println(err) 70 | if err.Error() == fmt.Errorf(response.UnauthorizedMSG).Error() { 71 | response.UnauthorizedError(writer) 72 | return 73 | } 74 | if err.Error() == fmt.Errorf(response.AccessDeniedMsg).Error() { 75 | response.AccesDeniedOverTables(writer, username, data) 76 | return 77 | } 78 | response.Error(writer, []byte(err.Error())) 79 | return 80 | } 81 | response.WriteData(writer, data) 82 | } 83 | 84 | } 85 | 86 | func Query(query, privilege string, UID, DBID int, executer internal.SQLExecutor) ([]byte, error) { 87 | query = strings.ToLower(query) 88 | authraized, err := Access(query, privilege, UID, DBID) 89 | if err != nil { 90 | return nil, err 91 | } 92 | if !authraized { 93 | tables, _ := internal.ExtractTableNames(query) 94 | 95 | return []byte(strings.Join(tables, ",")), fmt.Errorf(response.AccessDeniedMsg) 96 | } 97 | if strings.HasPrefix(query, "select") { 98 | data, err := internal.SELECT(executer, query) 99 | if err != nil { 100 | return nil, err 101 | } 102 | data = append(data, '\n') 103 | return data, nil 104 | } 105 | 106 | if strings.HasPrefix(query, "create") { 107 | err := internal.CREATE(executer, server.Serv.Sqlitedb, server.Serv.Dbstmt["CreateTable"], query, DBID) 108 | if err != nil { 109 | return nil, err 110 | } 111 | return []byte("Created"), nil 112 | } 113 | 114 | // other statements 115 | LastInsertedID, RowsAffected, err := internal.EXEC(executer, query) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | data := []byte(strconv.Itoa(int(LastInsertedID)) + " " + strconv.Itoa(int(RowsAffected)) + "\n") 121 | return data, nil 122 | } 123 | 124 | func Access(query, privilege string, UID, DBID int) (bool, error) { 125 | query = strings.ToLower(query) 126 | 127 | if privilege != "super" { 128 | hasDDL, err := internal.CheckDDLActions(query) 129 | if err != nil { 130 | return false, fmt.Errorf("%s", response.UnauthorizedMSG) 131 | } 132 | hasaccess, err := internal.CheckAccesOverTable(server.Serv.Sqlitedb, server.Serv.Dbstmt["CheckTableAccess"], query, UID, DBID) 133 | if err != nil { 134 | return false, fmt.Errorf("%s", response.UnauthorizedMSG) 135 | } 136 | return (hasaccess && !hasDDL), nil 137 | } 138 | return true, nil 139 | } 140 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | tcp-duckdb: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | ports: 9 | - "4000:4000" 10 | volumes: 11 | - ./storge:/app/storge 12 | - ./server:/app/server 13 | - ./main:/app/main 14 | environment: 15 | - ServerPort=4000 16 | - ServerAddr=0.0.0.0 17 | - DBdir=/app/storge/ 18 | - ServerDbFile=db.sqlite3 19 | - ENCRYPTION_KEY=A15pG0m3hwf0tfpVW6m92eZ6vRmAQA3C 20 | command: > 21 | sh -c "mkdir -p /app/storge/users && 22 | /app/build/server" 23 | restart: unless-stopped -------------------------------------------------------------------------------- /globals/global.go: -------------------------------------------------------------------------------- 1 | package globals 2 | 3 | 4 | var PreparedStmtStrings = [][]string{ 5 | {"login", "SELECT userid , usertype FROM user WHERE username LIKE ? AND password LIKE ? ;"}, 6 | {"SelectUser", "SELECT userid FROM user WHERE username LIKE ? ;"}, 7 | {"CreateUser", "INSERT INTO user(username, password, usertype) VALUES(?, ?, ?);"}, 8 | {"UpdateUsername", "UPDATE user SET username = ? WHERE userid == ?;"}, 9 | {"UpdateUserPassword", "UPDATE user SET password = ? WHERE userid == ?;"}, 10 | {"SelectDB", "SELECT dbid FROM DB WHERE dbname LIKE ? ;"}, 11 | {"CreateDB", "INSERT INTO DB(dbname) VALUES(?);"}, 12 | {"UpdateDB", "UPDATE DB SET dbname = ? WHERE dbid == ?;"}, 13 | {"GrantDB", "INSERT OR IGNORE INTO dbprivilege(dbid, userid, privilegetype) VALUES(?, ?, ?);"}, 14 | {"CheckDbAccess", "SELECT COUNT(*) FROM dbprivilege WHERE userid == ? AND dbid == ?"}, 15 | {"DbAccessType", "SELECT privilegetype FROM dbprivilege WHERE userid == ? AND dbid == ?;"}, 16 | {"DeleteDbAccess", "DELETE FROM dbprivilege Where userid == ? AND dbid == ?;"}, 17 | {"SelectTable", "SELECT tableid FROM tables WHERE tablename LIKE ? AND dbid == ?;"}, 18 | {"CheckTableAccess", "SELECT COUNT(*) FROM tableprivilege WHERE userid == ? AND tableid == ? AND tableprivilege LIKE ?;"}, 19 | {"GrantTable", "INSERT OR IGNORE INTO tableprivilege(tableid, userid, tableprivilege) VALUES(?, ?, ?);"}, 20 | {"CreateTable", "INSERT OR IGNORE INTO tables(tablename, dbid) VALUES(?, ?);"}, 21 | {"CreateLink", "INSERT OR IGNORE INTO postgres(dbid, connstr) VALUES(?, ?);"}, 22 | {"CheckLink", "SELECT COUNT(*) FROM postgres WHERE dbid == ?;"}, 23 | {"SelectLink", "SELECT connstr FROM postgres WHERE dbid == ?;"}, 24 | } 25 | 26 | var DbPoolSize uint = 1 << 2 // 4 db connections for testing will be 16 for production 27 | var ReplacerK uint = 1 << 2 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rag-nar1/tcp-duckdb 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.0 6 | 7 | require ( 8 | github.com/jmoiron/sqlx v1.4.0 9 | github.com/joho/godotenv v1.5.1 10 | github.com/lib/pq v1.10.9 11 | github.com/marcboeker/go-duckdb v1.8.3 12 | github.com/mattn/go-sqlite3 v1.14.24 13 | github.com/stretchr/testify v1.10.0 14 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 15 | ) 16 | 17 | require ( 18 | github.com/apache/arrow-go/v18 v18.0.0 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/goccy/go-json v0.10.3 // indirect 21 | github.com/google/flatbuffers v24.3.25+incompatible // indirect 22 | github.com/klauspost/compress v1.17.11 // indirect 23 | github.com/klauspost/cpuid/v2 v2.2.8 // indirect 24 | github.com/mitchellh/mapstructure v1.5.0 // indirect 25 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 26 | github.com/pmezard/go-difflib v1.0.0 // indirect 27 | github.com/zeebo/xxh3 v1.0.2 // indirect 28 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect 29 | golang.org/x/mod v0.21.0 // indirect 30 | golang.org/x/sync v0.8.0 // indirect 31 | golang.org/x/sys v0.26.0 // indirect 32 | golang.org/x/tools v0.26.0 // indirect 33 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 34 | gopkg.in/yaml.v3 v3.0.1 // indirect 35 | ) 36 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= 2 | filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= 3 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 4 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 5 | github.com/apache/arrow-go/v18 v18.0.0 h1:1dBDaSbH3LtulTyOVYaBCHO3yVRwjV+TZaqn3g6V7ZM= 6 | github.com/apache/arrow-go/v18 v18.0.0/go.mod h1:t6+cWRSmKgdQ6HsxisQjok+jBpKGhRDiqcf3p0p/F+A= 7 | github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= 8 | github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= 12 | github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= 13 | github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= 14 | github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 15 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 16 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 17 | github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= 18 | github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= 19 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 20 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 21 | github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= 22 | github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= 23 | github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 24 | github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 25 | github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= 26 | github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= 27 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 28 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 29 | github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= 30 | github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 31 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 32 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 33 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 34 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 35 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 36 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 37 | github.com/marcboeker/go-duckdb v1.8.3 h1:ZkYwiIZhbYsT6MmJsZ3UPTHrTZccDdM4ztoqSlEMXiQ= 38 | github.com/marcboeker/go-duckdb v1.8.3/go.mod h1:C9bYRE1dPYb1hhfu/SSomm78B0FXmNgRvv6YBW/Hooc= 39 | github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 40 | github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= 41 | github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 42 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= 43 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= 44 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= 45 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= 46 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 47 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 48 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 49 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 50 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 51 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 52 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 53 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 54 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= 55 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= 56 | github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= 57 | github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= 58 | github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= 59 | github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= 60 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= 61 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= 62 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 63 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 64 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 65 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 66 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 67 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= 68 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 69 | golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= 70 | golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= 71 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= 72 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 73 | gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= 74 | gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= 75 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 76 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 77 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /image/Drawing 2025-04-04 12.25.23.excalidraw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rag-nar1/tcp-duckdb/9bf5fb96416506c84f23f3a3fd8e3343bf3c8657/image/Drawing 2025-04-04 12.25.23.excalidraw.png -------------------------------------------------------------------------------- /image/connect.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Connect 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | database-name 22 | -------------------------------------------------------------------------------- /image/create.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | CREATE 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | DATABASE 25 | 26 | 27 | 28 | 29 | 30 | database-name 31 | 32 | 33 | 34 | USER 35 | 36 | 37 | 38 | 39 | 40 | username 41 | 42 | 43 | 44 | 45 | 46 | password 47 | -------------------------------------------------------------------------------- /internal/migration.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "github.com/rag-nar1/tcp-duckdb/utils" 5 | 6 | "database/sql" 7 | "database/sql/driver" 8 | "encoding/json" 9 | "fmt" 10 | "net" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | "time" 15 | 16 | "github.com/jmoiron/sqlx" 17 | _ "github.com/lib/pq" 18 | _ "github.com/marcboeker/go-duckdb" 19 | ) 20 | 21 | type JSONB map[string]interface{} 22 | 23 | // Value implements the driver.Valuer interface 24 | func (j JSONB) Value() (driver.Value, error) { 25 | if j == nil { 26 | return nil, nil 27 | } 28 | return json.Marshal(j) 29 | } 30 | 31 | // Scan implements the sql.Scanner interface 32 | func (j *JSONB) Scan(src interface{}) error { 33 | if src == nil { 34 | *j = nil 35 | return nil 36 | } 37 | 38 | // Handle different possible source types 39 | switch v := src.(type) { 40 | case []byte: 41 | if len(v) == 0 { 42 | *j = nil 43 | return nil 44 | } 45 | return json.Unmarshal(v, j) 46 | case string: 47 | if v == "" { 48 | *j = nil 49 | return nil 50 | } 51 | return json.Unmarshal([]byte(v), j) 52 | default: 53 | return fmt.Errorf("invalid type for JSONB") 54 | } 55 | } 56 | 57 | func (j JSONB) Get() ([]string, []interface{}) { 58 | columns := make([]string, len(j)) 59 | values := make([]interface{}, len(j)) 60 | i := 0 61 | for k, v := range j { 62 | columns[i] = k 63 | values[i] = v 64 | i++ 65 | } 66 | return columns, values 67 | } 68 | 69 | type AuditRecord struct { 70 | EventID int64 `json:"event_id" db:"event_id"` // BIGSERIAL PRIMARY KEY 71 | SchemaName string `json:"schema_name" db:"schema_name"` // TEXT NOT NULL 72 | TableName string `json:"table_name" db:"table_name"` // TEXT NOT NULL 73 | TablePK string `json:"table_pk" db:"table_pk"` // TEXT 74 | TablePKColumn string `json:"table_pk_column" db:"table_pk_column"` // TEXT 75 | ActionTimestamp time.Time `json:"action_tstamp" db:"action_tstamp"` // TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP 76 | Action string `json:"action" db:"action"` // TEXT NOT NULL CHECK (action IN ('I','D','U','T')) 77 | OriginalData JSONB `json:"original_data" db:"original_data"` // JSONB 78 | NewData JSONB `json:"new_data" db:"new_data"` // JSONB 79 | ChangedFields JSONB `json:"changed_fields" db:"changed_fields"` // JSONB 80 | TransactionID *int64 `json:"transaction_id" db:"transaction_id"` // BIGINT (nullable) 81 | ApplicationName *string `json:"application_name" db:"application_name"` // TEXT (nullable) 82 | ClientAddr *net.IP `json:"client_addr" db:"client_addr"` // INET (nullable) 83 | ClientPort *int32 `json:"client_port" db:"client_port"` // INTEGER (nullable) 84 | SessionUserName *string `json:"session_user_name" db:"session_user_name"` // TEXT (nullable) 85 | CurrentUserName *string `json:"current_user_name" db:"current_user_name"` // TEXT (nullable) 86 | } 87 | 88 | type column struct { 89 | name string 90 | dataType string 91 | } 92 | 93 | type table struct { 94 | name string 95 | columns []column 96 | } 97 | 98 | func (t *table) Add(name, dataType string) { 99 | t.columns = append(t.columns, column{name: name, dataType: dataType}) 100 | } 101 | 102 | func (t table) GenereteSql() string { 103 | query := "CREATE TABLE IF NOT EXISTS %s (%s);" 104 | var columns string = "" 105 | for _, col := range t.columns { 106 | columns += col.name + " " + utils.DbTypeMap(strings.ToUpper(col.dataType)) + "," 107 | } 108 | columns = columns[:len(columns)-1] 109 | return fmt.Sprintf(query, t.name, columns) 110 | } 111 | 112 | func Migrate(DBID int, connStr string, stmt *sql.Stmt, postgres, duck, server *sql.Tx) error { 113 | var tables map[string]*table = make(map[string]*table) 114 | 115 | rows, err := postgres.Query("SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' and table_name not like 'pg%';") 116 | if err != nil { 117 | return err 118 | } 119 | defer rows.Close() 120 | 121 | data := make([]string, 3) 122 | dataptr := make([]interface{}, 3) 123 | for i := range data { 124 | dataptr[i] = &data[i] 125 | } 126 | 127 | for rows.Next() { 128 | rows.Scan(dataptr...) 129 | _, ok := tables[data[0]] 130 | if !ok { 131 | tables[data[0]] = &table{name: data[0]} 132 | } 133 | tables[data[0]].Add(data[1], data[2]) 134 | } 135 | 136 | // create tables in our server 137 | for _, table := range tables { 138 | _, err = server.Stmt(stmt).Exec(table.name, DBID) 139 | if err != nil { 140 | return err 141 | } 142 | } 143 | 144 | // connect to postgresql database to get all data 145 | _, err = duck.Exec(fmt.Sprintf("ATTACH '%s' AS postgres_db (TYPE postgres);", connStr)) 146 | if err != nil { 147 | return err 148 | } 149 | 150 | // Make sure to detach the database at the end of the function to close connections 151 | defer func() { 152 | // We ignore any errors from detach since we're in cleanup mode 153 | _, _ = duck.Exec("DETACH postgres_db;") 154 | }() 155 | 156 | for _, table := range tables { 157 | postgrestable := "postgres_db." + table.name 158 | stmt := fmt.Sprintf("CREATE TABLE %s AS FROM %s;", table.name, postgrestable) 159 | _, err := duck.Exec(stmt) 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | return nil 165 | } 166 | 167 | func Audit(postgres *sql.Tx) error { 168 | sqlfile, err := os.ReadFile(filepath.Join(os.Getenv("DBdir"), "server/audit.sql")) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | _, err = postgres.Exec(string(sqlfile)) 174 | if err != nil { 175 | return err 176 | } 177 | return nil 178 | } 179 | 180 | func ReadAudit(duck, postgres *sqlx.DB) error { 181 | var records []AuditRecord 182 | err := postgres.Select(&records, "select * from audit.logged_actions;") 183 | if err != nil { 184 | return err 185 | } 186 | 187 | transaction, err := duck.Beginx() 188 | if err != nil { 189 | return err 190 | } 191 | defer transaction.Rollback() 192 | 193 | var maxTimeStamp time.Time 194 | 195 | for i := range records { 196 | switch records[i].Action { 197 | case "I": 198 | ApplyInsert(transaction, &records[i]) 199 | case "U": 200 | ApplyUpdate(transaction, &records[i]) 201 | case "D": 202 | ApplyDelete(transaction, &records[i]) 203 | case "T": 204 | ApplyTrancate(transaction, &records[i]) 205 | default: 206 | return fmt.Errorf("Unsupported Action") 207 | } 208 | if maxTimeStamp.Before(records[i].ActionTimestamp) { 209 | maxTimeStamp = records[i].ActionTimestamp 210 | } 211 | } 212 | 213 | // Delete all tuples where time stamp <= max 214 | if _, err = postgres.Exec("DELETE FROM audit.logged_actions WHERE action_tstamp <= $1;", maxTimeStamp); err != nil { 215 | return err 216 | } 217 | 218 | if err := transaction.Commit(); err != nil { 219 | return err 220 | } 221 | 222 | return nil 223 | } 224 | 225 | func GenPlaceHoldersForDuck(num int) string { 226 | placeholders := make([]string, num) 227 | for i := range placeholders { 228 | placeholders[i] = "?" 229 | } 230 | 231 | res := strings.Join(placeholders, ",") 232 | return res 233 | } 234 | 235 | func GenSetForDuck(keys []string) string { 236 | placeholders := make([]string, len(keys)) 237 | for i := range placeholders { 238 | placeholders[i] = keys[i] + " = ?" 239 | } 240 | res := strings.Join(placeholders, ",") 241 | return res 242 | } 243 | 244 | func ApplyInsert(db *sqlx.Tx, record *AuditRecord) error { 245 | query := "INSERT INTO %s(%s) VALUES(%s);" 246 | keys, valuesInterfaces := record.NewData.Get() 247 | columns := strings.Join(keys, ",") 248 | query = fmt.Sprintf(query, record.TableName, columns, GenPlaceHoldersForDuck(len(keys))) 249 | _, err := db.Exec(query, valuesInterfaces...) 250 | return err 251 | } 252 | 253 | func ApplyUpdate(db *sqlx.Tx, record *AuditRecord) error { 254 | query := "UPDATE %s SET %s WHERE %s;" 255 | keys, valuesInterfaces := record.ChangedFields.Get() 256 | predicate := fmt.Sprintf("%s = %s", record.TablePKColumn, record.TablePK) 257 | columns := GenSetForDuck(keys) 258 | query = fmt.Sprintf(query, record.TableName, columns, predicate) 259 | _, err := db.Exec(query, valuesInterfaces...) 260 | return err 261 | } 262 | 263 | func ApplyDelete(db *sqlx.Tx, record *AuditRecord) error { 264 | query := "DELETE FROM %s WHERE %s;" 265 | predicate := fmt.Sprintf("%s = %s", record.TablePKColumn, record.TablePK) 266 | query = fmt.Sprintf(query, record.TableName, predicate) 267 | _, err := db.Exec(query) 268 | return err 269 | } 270 | 271 | func ApplyTrancate(db *sqlx.Tx, record *AuditRecord) error { 272 | query := "TRUNCATE %s;" 273 | query = fmt.Sprintf(query, record.TableName) 274 | _, err := db.Exec(query) 275 | return err 276 | } 277 | -------------------------------------------------------------------------------- /internal/query.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "strings" 7 | 8 | "github.com/xwb1989/sqlparser" 9 | ) 10 | 11 | type SQLExecutor interface { 12 | Query(query string, args ...interface{}) (*sql.Rows, error) 13 | QueryRow(query string, args ...interface{}) *sql.Row 14 | Exec(query string, args ...interface{}) (sql.Result, error) 15 | } 16 | 17 | 18 | func SELECT(db SQLExecutor, query string) ([]byte, error) { 19 | rows, err := db.Query(query) 20 | if err != nil { 21 | return nil ,err 22 | } 23 | defer rows.Close() 24 | 25 | result, err := ReadRows(rows) 26 | if err != nil { 27 | return nil ,err 28 | } 29 | 30 | dataJson, err := json.Marshal(result) 31 | if err != nil { 32 | return nil ,err 33 | } 34 | 35 | return dataJson, nil 36 | } 37 | 38 | func EXEC(db SQLExecutor, query string) (int64, int64, error) { 39 | result, err := db.Exec(query) 40 | if err != nil { 41 | return 0, 0, err 42 | } 43 | 44 | LastInsertedID, err := result.LastInsertId() 45 | if err != nil { 46 | return 0, 0, err 47 | } 48 | 49 | RowsAffected,err := result.RowsAffected() 50 | if err != nil { 51 | return 0, 0, err 52 | } 53 | 54 | return LastInsertedID, RowsAffected, nil 55 | } 56 | 57 | 58 | func CREATE(db SQLExecutor, server *sql.DB, stmt *sql.Stmt, query string, DBID int) (error){ 59 | tables, err := ExtractTableNames(query) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | servertx, err := server.Begin(); 65 | if err != nil { 66 | return err 67 | } 68 | defer servertx.Rollback() 69 | 70 | for _, table := range tables { 71 | _, err := servertx.Stmt(stmt).Exec(table, DBID) 72 | if err != nil { 73 | return err 74 | } 75 | } 76 | _, err = db.Exec(query) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | err = servertx.Commit() 82 | if err != nil { 83 | return err 84 | } 85 | 86 | return nil 87 | } 88 | 89 | 90 | func ReadRows(rows *sql.Rows) ([]map[string]interface{}, error) { 91 | var result []map[string]interface{} 92 | columns, err := rows.Columns() 93 | if err != nil { 94 | return nil ,err 95 | } 96 | 97 | columnsSz := len(columns) 98 | for rows.Next() { 99 | row := make([]interface{}, columnsSz) 100 | rowPtr := make([]interface{}, columnsSz) 101 | for i := range row { 102 | rowPtr[i] = &row[i] 103 | } 104 | 105 | rows.Scan(rowPtr...) 106 | object := make(map[string]interface{}) 107 | for i , columnName := range columns { 108 | object[columnName] = row[i] 109 | } 110 | 111 | result = append(result, object) 112 | } 113 | 114 | return result, nil 115 | } 116 | 117 | func CheckDDLActions(query string) (bool, error) { 118 | DDL := make([]string, 0) 119 | // Parse the query into an AST. 120 | stmt, err := sqlparser.Parse(query) 121 | if err != nil { 122 | return false, err 123 | } 124 | // Walk the AST to catch both top-level and nested statements. 125 | err = sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { 126 | var action string 127 | switch n := node.(type) { 128 | // Data Definition Language: 129 | case *sqlparser.DDL: 130 | // n.Action might be "create", "alter", "drop", etc. 131 | action = strings.ToUpper(n.Action) 132 | DDL = append(DDL, action) 133 | default: 134 | return true, nil 135 | } 136 | return true, nil 137 | }, stmt) 138 | if err != nil { 139 | return false, err 140 | } 141 | 142 | return (len(DDL) > 0), nil 143 | } 144 | 145 | func CheckAccesOverTable(db *sql.DB, stmt *sql.Stmt, query string, UID, DBID int) (bool, error){ 146 | // parse the table and map each to type of acction used to access it 147 | tables , err := classifySQLTables(query) 148 | if err != nil { 149 | return false, err 150 | } 151 | 152 | 153 | var TID int 154 | var cnt int 155 | for table , actions := range tables { 156 | err := db.QueryRow("SELECT tableid FROM tables WHERE tablename LIKE ? AND dbid == ?", table, DBID).Scan(&TID) 157 | if err != nil { 158 | return false, err 159 | } 160 | 161 | for _ , action := range actions { 162 | err = stmt.QueryRow(UID, TID, strings.ToLower(action)).Scan(&cnt) 163 | if err != nil { 164 | return false, err 165 | } 166 | if cnt == 0 { 167 | return false, nil 168 | } 169 | } 170 | } 171 | 172 | return true, nil 173 | } 174 | 175 | func ExtractTableNames(query string) ([]string, error) { 176 | stmt, err := sqlparser.Parse(query) 177 | if err != nil { 178 | return nil, err 179 | } 180 | 181 | tableNames := make(map[string]bool) 182 | 183 | // Walk through the AST and collect table names 184 | sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { 185 | switch n := node.(type) { 186 | case *sqlparser.TableName: 187 | if !n.IsEmpty() { 188 | tableName := n.Name.String() 189 | tableNames[tableName] = true 190 | } 191 | case *sqlparser.DDL: 192 | // Handle CREATE, DROP, ALTER table statements 193 | if !n.NewName.IsEmpty() { 194 | tableName := n.NewName.Name.String() 195 | tableNames[tableName] = true 196 | } 197 | if !n.Table.IsEmpty() { 198 | tableName := n.Table.Name.String() 199 | tableNames[tableName] = true 200 | } 201 | } 202 | return true, nil 203 | }, stmt) 204 | 205 | // Convert map to slice 206 | result := make([]string, 0, len(tableNames)) 207 | for tableName := range tableNames { 208 | result = append(result, tableName) 209 | } 210 | 211 | return result, nil 212 | } 213 | 214 | // extractTableNamesFromTableExprs extracts table names from a list of TableExprs. 215 | // It handles AliasedTableExpr, JoinTableExpr, and ParenTableExpr. 216 | func extractTableNamesFromTableExprs(exprs sqlparser.TableExprs) []string { 217 | var tables []string 218 | for _, expr := range exprs { 219 | switch tbl := expr.(type) { 220 | case *sqlparser.AliasedTableExpr: 221 | // If the expression is a simple table name. 222 | switch expr := tbl.Expr.(type) { 223 | case sqlparser.TableName: 224 | // Use sqlparser.String to get the fully qualified name. 225 | tables = append(tables, sqlparser.String(expr)) 226 | } 227 | case *sqlparser.JoinTableExpr: 228 | tables = append(tables, extractTableNamesFromTableExprs(sqlparser.TableExprs{tbl.LeftExpr})...) 229 | tables = append(tables, extractTableNamesFromTableExprs(sqlparser.TableExprs{tbl.RightExpr})...) 230 | case *sqlparser.ParenTableExpr: 231 | tables = append(tables, extractTableNamesFromTableExprs(tbl.Exprs)...) 232 | } 233 | } 234 | return tables 235 | } 236 | 237 | // classifySQLTables parses the given SQL query and returns a map that maps table names 238 | // to a list of actions (like SELECT, INSERT, CREATE, etc.) that are performed on them. 239 | func classifySQLTables(query string) (map[string][]string, error) { 240 | // Map of table name to a slice of actions. 241 | result := make(map[string][]string) 242 | 243 | // Parse the query into an AST. 244 | stmt, err := sqlparser.Parse(query) 245 | if err != nil { 246 | return nil, err 247 | } 248 | 249 | // Walk the AST to catch both top-level and nested statements. 250 | err = sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { 251 | var action string 252 | var tables []string 253 | 254 | switch n := node.(type) { 255 | // Data Query Language: SELECT statements. 256 | case *sqlparser.Select: 257 | action = "SELECT" 258 | tables = extractTableNamesFromTableExprs(n.From) 259 | // Data Manipulation Language: 260 | case *sqlparser.Insert: 261 | action = "INSERT" 262 | tables = []string{sqlparser.String(n.Table)} 263 | case *sqlparser.Update: 264 | action = "UPDATE" 265 | tables = extractTableNamesFromTableExprs(n.TableExprs) 266 | case *sqlparser.Delete: 267 | action = "DELETE" 268 | tables = extractTableNamesFromTableExprs(n.TableExprs) 269 | default: 270 | return true, nil 271 | } 272 | 273 | // Append the action to each table found. 274 | for _, t := range tables { 275 | t = strings.TrimSpace(t) 276 | if t == "" { 277 | continue 278 | } 279 | // Append action to the list for the table. 280 | result[t] = append(result[t], action) 281 | } 282 | 283 | return true, nil 284 | }, stmt) 285 | if err != nil { 286 | return nil, err 287 | } 288 | 289 | return result, nil 290 | } -------------------------------------------------------------------------------- /login/handler.go: -------------------------------------------------------------------------------- 1 | package login 2 | 3 | import ( 4 | "errors" 5 | 6 | response "github.com/rag-nar1/tcp-duckdb/response" 7 | global "github.com/rag-nar1/tcp-duckdb/server" 8 | utils "github.com/rag-nar1/tcp-duckdb/utils" 9 | 10 | "bufio" 11 | "strings" 12 | ) 13 | 14 | func Handler(reader *bufio.Reader, writer *bufio.Writer, UID *int, userName, privilege *string) error { 15 | // read login request 16 | route := make([]byte, 1024) 17 | n, err := reader.Read(route) 18 | if err != nil { 19 | response.InternalError(writer) 20 | global.Serv.ErrorLog.Println(err) 21 | return err 22 | } 23 | // check for a valid request 24 | request := strings.Split(string(route[0:n]), " ") 25 | if request[0] != "login" || len(request) != 3 { 26 | response.BadRequest(writer) 27 | return errors.New(response.BadRequestMsg) 28 | } 29 | // validate the userName and password 30 | var password string 31 | *userName, password = utils.Trim(request[1]), utils.Trim(request[2]) 32 | *UID, *privilege, err = Login(*userName, password, global.Serv.Dbstmt["login"]) 33 | 34 | if err != nil { 35 | response.UnauthorizedError(writer) 36 | global.Serv.ErrorLog.Println(err) 37 | return err 38 | } 39 | response.Success(writer) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /login/login_test.go: -------------------------------------------------------------------------------- 1 | package login_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/utils" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestLoginAsDuck tests the login as admin functionality 11 | func TestLoginAsDuck(t *testing.T) { 12 | conn := utils.StartUp() // Start up the connection 13 | err := utils.LoginAsAdmin(conn) // Attempt to login as admin 14 | assert.Nil(t, err) // Assert that there is no error 15 | } 16 | 17 | // TestLogin tests the login functionality with valid credentials 18 | func TestLogin(t *testing.T) { 19 | t.Cleanup(utils.CleanUp) // Ensure cleanup after test 20 | conn := utils.StartUp() // Start up the connection 21 | err := utils.LoginAsAdmin(conn) // Attempt to login as admin 22 | assert.Nil(t, err) // Assert that there is no error 23 | 24 | username := "ragnar" 25 | password := "ragnar" 26 | err = utils.CreateUser(conn, username, password) // Create a new user 27 | assert.Nil(t, err) // Assert that there is no error 28 | 29 | conn2 := utils.Connection() // Create a new connection 30 | err = utils.Login(conn2, username, password) // Attempt to login with the new user 31 | assert.Nil(t, err) // Assert that there is no error 32 | } 33 | 34 | // TestLoginWithInvalidCredentials tests the login functionality with invalid credentials 35 | func TestLoginWithInvalidCredentials(t *testing.T) { 36 | conn := utils.StartUp() // Start up the connection 37 | err := utils.LoginAsAdmin(conn) // Attempt to login as admin 38 | assert.Nil(t, err) // Assert that there is no error 39 | 40 | username := "invalid_user" 41 | password := "invalid_pass" 42 | conn2 := utils.Connection() // Create a new connection 43 | err = utils.Login(conn2, username, password) // Attempt to login with invalid credentials 44 | assert.NotNil(t, err) // Assert that there is an error 45 | } 46 | 47 | // TestLoginWithEmptyCredentials tests the login functionality with empty credentials 48 | func TestLoginWithEmptyCredentials(t *testing.T) { 49 | conn := utils.StartUp() // Start up the connection 50 | err := utils.LoginAsAdmin(conn) // Attempt to login as admin 51 | assert.Nil(t, err) // Assert that there is no error 52 | 53 | username := "" 54 | password := "" 55 | conn2 := utils.Connection() // Create a new connection 56 | err = utils.Login(conn2, username, password) // Attempt to login with empty credentials 57 | assert.NotNil(t, err) // Assert that there is an error 58 | } 59 | -------------------------------------------------------------------------------- /login/service.go: -------------------------------------------------------------------------------- 1 | package login 2 | 3 | import ( 4 | utils "github.com/rag-nar1/tcp-duckdb/utils" 5 | 6 | "database/sql" 7 | ) 8 | 9 | func Login(UserName, password string, Dbstmt *sql.Stmt) (int, string, error) { 10 | 11 | var UID int 12 | var privilige string 13 | err := Dbstmt.QueryRow(UserName, utils.Hash(password)).Scan(&UID, &privilige) 14 | if err != nil { 15 | return -1, "", err 16 | } 17 | 18 | return UID, privilige, nil 19 | } 20 | -------------------------------------------------------------------------------- /main/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net" 5 | 6 | _ "github.com/marcboeker/go-duckdb" 7 | _ "github.com/mattn/go-sqlite3" 8 | "github.com/rag-nar1/tcp-duckdb/server" 9 | ) 10 | 11 | func init() { 12 | server.Init() 13 | } 14 | 15 | func main() { 16 | // start listing to tcp connections 17 | listener, err := net.Listen("tcp", server.Serv.Address) 18 | if err != nil { 19 | server.Serv.ErrorLog.Fatal(err) 20 | } 21 | defer listener.Close() 22 | 23 | server.Serv.InfoLog.Println("listening to " + server.Serv.Address) 24 | for { 25 | conn, err := listener.Accept() 26 | if err != nil { 27 | server.Serv.ErrorLog.Fatal(err) 28 | } 29 | // starts a go routin to handle every connection 30 | go HandleConnection(conn) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /main/router.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "net" 6 | "strings" 7 | 8 | create "github.com/rag-nar1/tcp-duckdb/commands/create" 9 | grant "github.com/rag-nar1/tcp-duckdb/commands/grant" 10 | link "github.com/rag-nar1/tcp-duckdb/commands/link" 11 | migrate "github.com/rag-nar1/tcp-duckdb/commands/migrate" 12 | update "github.com/rag-nar1/tcp-duckdb/commands/update" 13 | "github.com/rag-nar1/tcp-duckdb/connect" 14 | "github.com/rag-nar1/tcp-duckdb/login" 15 | "github.com/rag-nar1/tcp-duckdb/response" 16 | "github.com/rag-nar1/tcp-duckdb/utils" 17 | "github.com/rag-nar1/tcp-duckdb/server" 18 | 19 | _ "github.com/lib/pq" 20 | _ "github.com/marcboeker/go-duckdb" 21 | ) 22 | 23 | type Handler interface { 24 | Handler(privilege string, req []string, writer *bufio.Writer) 25 | } 26 | 27 | type HandlerFunc func(privilege string, req []string, writer *bufio.Writer) 28 | 29 | func (f HandlerFunc) Handler(privilege string, req []string, writer *bufio.Writer) { 30 | f(privilege, req, writer) 31 | } 32 | 33 | var Handlers = map[string]Handler{ 34 | "create": HandlerFunc(create.Handler), 35 | "grant": HandlerFunc(grant.Handler), 36 | "link": HandlerFunc(link.Handler), 37 | "migrate": HandlerFunc(migrate.Handler), 38 | "update": HandlerFunc(update.Handler), 39 | } 40 | 41 | func HandleConnection(conn net.Conn) { 42 | defer conn.Close() 43 | server.Serv.InfoLog.Println("Serving " + conn.RemoteAddr().String()) 44 | reader := bufio.NewReader(conn) 45 | writer := bufio.NewWriter(conn) 46 | var ( 47 | UID int 48 | userName string 49 | privilege string 50 | ) 51 | 52 | if err := login.Handler(reader, writer, &UID, &userName, &privilege); err != nil { 53 | return 54 | } 55 | 56 | Router(UID, userName, privilege, reader, writer) 57 | } 58 | 59 | func Router(UID int, UserName, privilege string, reader *bufio.Reader, writer *bufio.Writer) { 60 | server.Serv.InfoLog.Println("Serving: " + UserName) 61 | rawreq := make([]byte, 1024) 62 | for { 63 | n, err := reader.Read(rawreq) 64 | if err != nil { 65 | response.InternalError(writer) 66 | server.Serv.ErrorLog.Println(err) 67 | server.Serv.InfoLog.Println("Connection closed") 68 | return 69 | } 70 | 71 | req := strings.Split(string(rawreq[0:n]), " ") 72 | utils.TrimList(req) 73 | 74 | if _,ok := Handlers[req[0]]; !ok && req[0] != "connect"{ 75 | response.BadRequest(writer) 76 | continue 77 | } 78 | 79 | if req[0] == "connect" { 80 | if len(req) != 2 { 81 | response.BadRequest(writer) 82 | continue 83 | } 84 | connect.Handler(UID, UserName, privilege, req[1], reader, writer) 85 | continue 86 | } 87 | 88 | Handlers[req[0]].Handler(privilege, req[1:], writer) 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /pool/pool.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "container/list" 5 | "database/sql" 6 | "errors" 7 | 8 | "os" 9 | "sync" 10 | "sync/atomic" 11 | 12 | "github.com/rag-nar1/tcp-duckdb/globals" 13 | _ "github.com/marcboeker/go-duckdb" 14 | ) 15 | 16 | type DBPool struct { 17 | ConnPool *sql.DB 18 | Dbname string 19 | Dbid uint 20 | PinCount atomic.Int32 21 | Replacer *LruReplacer 22 | PoolLatch *sync.Mutex 23 | } 24 | 25 | type Connection interface { 26 | DB() *sql.DB 27 | Destroy() 28 | 29 | GetPinCount() int // for testing only 30 | } 31 | 32 | func (d *DBPool) DB() *sql.DB { 33 | return d.ConnPool 34 | } 35 | 36 | func (d *DBPool) GetPinCount() int { 37 | return int(d.PinCount.Load()) 38 | } 39 | 40 | func (d *DBPool) Destroy() { 41 | if d.ConnPool == nil { 42 | return 43 | } 44 | 45 | d.PoolLatch.Lock() 46 | 47 | d.PinCount.Add(-1) 48 | if d.PinCount.Load() == 0 { 49 | d.Replacer.SetEvictable(d.Dbid, true) 50 | } 51 | d.PoolLatch.Unlock() 52 | 53 | d = nil 54 | } 55 | 56 | func (d *DBPool) Delete() { 57 | if d == nil || d.ConnPool == nil { 58 | return 59 | } 60 | d.ConnPool.Close() 61 | } 62 | 63 | 64 | 65 | type Pool struct { 66 | DB []*DBPool 67 | Ids map[string]uint 68 | Free *list.List 69 | Size uint 70 | Replacer *LruReplacer 71 | Latch sync.Mutex 72 | } 73 | 74 | func NewPool() *Pool { 75 | pool := &Pool{ 76 | DB: make([]*DBPool, globals.DbPoolSize + 1), 77 | Ids: make(map[string]uint), 78 | Free: list.New(), 79 | Size: 0, 80 | Replacer: NewLruReplacer(globals.ReplacerK), 81 | Latch: sync.Mutex{}, 82 | } 83 | 84 | for i := 1; i <= int(globals.DbPoolSize); i ++ { 85 | pool.Free.PushBack(uint(i)) 86 | } 87 | 88 | return pool 89 | } 90 | 91 | 92 | func (p *Pool) Get(dbname string) (Connection, error) { 93 | p.Latch.Lock() 94 | defer p.Latch.Unlock() 95 | 96 | dbid, ok := p.Ids[dbname] 97 | if ok { 98 | p.DB[dbid].PinCount.Add(1) 99 | return p.DB[dbid], nil 100 | } 101 | 102 | var connPool *sql.DB 103 | 104 | if p.Size == globals.DbPoolSize { // no evaction needed 105 | // try to evict 106 | dbid = p.Replacer.Evict() 107 | if dbid == InvalidDbId { 108 | return nil, errors.New(LruReplacerFullErrorStmt) 109 | } 110 | p.Size -- 111 | p.Free.PushFront(dbid) 112 | } 113 | 114 | dbid = p.Free.Front().Value.(uint) 115 | p.Free.Remove(p.Free.Front()) 116 | connPool, err := sql.Open("duckdb", os.Getenv("DBdir") + "users/"+ dbname + ".db") 117 | if err != nil { 118 | return nil, err 119 | } 120 | p.DB[dbid].Delete() 121 | 122 | dbPool := DBPool{ 123 | ConnPool: connPool, 124 | Dbname: dbname, 125 | Dbid: dbid, 126 | PinCount: atomic.Int32{}, 127 | Replacer: p.Replacer, 128 | PoolLatch: &p.Latch, 129 | } 130 | 131 | if err := p.Replacer.RecordAccess(dbPool.Dbid); err != nil { 132 | dbPool.Delete() 133 | return nil, err 134 | } 135 | 136 | dbPool.PinCount.Add(1) 137 | p.DB[dbid] = &dbPool 138 | p.Ids[dbname] = dbid 139 | p.Size ++ 140 | 141 | return &dbPool, nil 142 | } -------------------------------------------------------------------------------- /pool/pool_test.go: -------------------------------------------------------------------------------- 1 | package pool_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "testing" 9 | 10 | "github.com/joho/godotenv" 11 | "github.com/rag-nar1/tcp-duckdb/globals" 12 | "github.com/rag-nar1/tcp-duckdb/pool" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | func CleanUp() { 16 | files, err := filepath.Glob("../storge/users/*") 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | for _, file := range files { 22 | err := os.Remove(file) 23 | if err != nil { 24 | panic(err) 25 | } 26 | } 27 | } 28 | 29 | func TestCleanUp(t *testing.T) { 30 | files, err := filepath.Glob("../storge/users/*") 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | for _, file := range files { 36 | err := os.Remove(file) 37 | if err != nil { 38 | panic(err) 39 | } 40 | } 41 | } 42 | func TestPoolBasic(t *testing.T) { 43 | t.Cleanup(CleanUp) 44 | if err := godotenv.Load("../.env"); err != nil { 45 | panic(err) 46 | } 47 | 48 | connPool := pool.NewPool() 49 | 50 | db1, err := connPool.Get("db1") 51 | assert.Nil(t, err) 52 | _, err = connPool.Get("db2") 53 | assert.Nil(t, err) 54 | _, err = connPool.Get("db3") 55 | assert.Nil(t, err) 56 | _, err = connPool.Get("db4") 57 | assert.Nil(t, err) 58 | _, err = connPool.Get("db5") 59 | assert.NotNil(t, err) 60 | 61 | db1.Destroy() 62 | _, err = connPool.Get("db5") 63 | assert.Nil(t, err) 64 | } 65 | 66 | func TestPoolMain(t *testing.T) { 67 | t.Cleanup(CleanUp) 68 | if err := godotenv.Load("../.env"); err != nil { 69 | panic(err) 70 | } 71 | 72 | connPool := pool.NewPool() 73 | 74 | db1, err := connPool.Get("db1") 75 | assert.Nil(t, err) 76 | _, err = connPool.Get("db2") 77 | assert.Nil(t, err) 78 | _, err = connPool.Get("db3") 79 | assert.Nil(t, err) 80 | _, err = connPool.Get("db4") 81 | assert.Nil(t, err) 82 | _, err = connPool.Get("db5") 83 | assert.NotNil(t, err) 84 | 85 | db1.Destroy() 86 | _, err = connPool.Get("db5") 87 | assert.Nil(t, err) 88 | db5, err := connPool.Get("db5") 89 | assert.Nil(t, err) 90 | assert.Equal(t, 2, db5.GetPinCount()) 91 | } 92 | 93 | func TestPoolConcurrunct(t *testing.T) { 94 | t.Cleanup(CleanUp) 95 | if err := godotenv.Load("../.env"); err != nil { 96 | panic(err) 97 | } 98 | 99 | connPool := pool.NewPool() 100 | var wg sync.WaitGroup 101 | 102 | for i := 1; i <= int(globals.DbPoolSize); i ++ { 103 | wg.Add(1) 104 | go func (t *testing.T, dbid int, connPool *pool.Pool) { 105 | db, err := connPool.Get(fmt.Sprintf("db%d", dbid)) 106 | assert.Nil(t, err) 107 | assert.Equal(t, 1, db.GetPinCount()) 108 | defer func() { 109 | db.Destroy() 110 | wg.Done() 111 | }() 112 | }(t, i, connPool) 113 | } 114 | 115 | wg.Wait() 116 | } -------------------------------------------------------------------------------- /pool/replacer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Lrureplacer is an LRU-K Lrureplacer 3 | */ 4 | 5 | package pool 6 | 7 | import ( 8 | list "container/list" 9 | atomic "sync/atomic" 10 | 11 | "errors" 12 | "fmt" 13 | "math" 14 | "sync" 15 | 16 | "github.com/rag-nar1/tcp-duckdb/globals" 17 | ) 18 | 19 | const ( 20 | InvalidDbId uint = 0 21 | InvalidDbidStmt = "invalid dbid %d" 22 | LruReplacerFullErrorStmt = "replacer is full use Evict" 23 | ) 24 | 25 | type Node struct { 26 | Dbid uint 27 | Evictable bool 28 | Access *list.List // accesslist for each database hold the time stamp 29 | } 30 | 31 | func NewNode(dbid uint) *Node { 32 | return &Node{ 33 | Dbid: dbid, 34 | Evictable: false, 35 | Access: &list.List{}, 36 | } 37 | } 38 | 39 | func (n *Node) RecordAccess(time *atomic.Uint64) { 40 | n.Access.PushBack(time.Load()) 41 | time.Add(1) 42 | } 43 | 44 | func (n *Node) Len() uint { 45 | return uint(n.Access.Len()) 46 | } 47 | 48 | func (n *Node) Remove() { 49 | n.Access.Remove(n.Access.Front()) 50 | } 51 | 52 | func (n *Node) GetKDistance(time *atomic.Uint64, K uint) uint64 { 53 | if n.Access.Len() < int(K) { 54 | return math.MaxUint64 55 | } 56 | 57 | return time.Load() - n.Access.Front().Value.(uint64) 58 | } 59 | 60 | func (n *Node) GetMostRecentAccess() uint64 { 61 | return n.Access.Back().Value.(uint64) 62 | } 63 | 64 | type LruReplacer struct { 65 | Nodes map[uint]*Node 66 | Size uint // curr size of the Lrureplacer 67 | K uint // the size of the time window which the Lrureplacer will use to evict db 68 | Latch sync.Mutex // latch to control concurrent use 69 | CurrTime atomic.Uint64 // atomic value for shared access to record the time where a database is accessed 70 | } 71 | 72 | 73 | func NewLruReplacer(k uint) *LruReplacer { 74 | r := LruReplacer { 75 | Nodes: make(map[uint]*Node), 76 | Size: 0, 77 | K: k, 78 | Latch: sync.Mutex{}, 79 | CurrTime: atomic.Uint64{}, 80 | } 81 | 82 | return &r 83 | } 84 | 85 | func (r *LruReplacer) RecordAccess(dbid uint) error { 86 | r.Latch.Lock() // lock the latch to record the access 87 | defer r.Latch.Unlock() 88 | 89 | if dbid > globals.DbPoolSize { 90 | return fmt.Errorf(InvalidDbidStmt, dbid) 91 | } 92 | 93 | if r.Size == uint(globals.DbPoolSize) { 94 | return errors.New(LruReplacerFullErrorStmt) 95 | } 96 | 97 | if _,ok := r.Nodes[dbid]; !ok { 98 | r.Nodes[dbid] = NewNode(dbid) 99 | } 100 | 101 | // record access 102 | r.Nodes[dbid].RecordAccess(&r.CurrTime) 103 | if r.Nodes[dbid].Access.Len() > int(r.K) { 104 | r.Nodes[dbid].Remove() 105 | } 106 | 107 | return nil 108 | } 109 | 110 | func (r *LruReplacer) Evict() (uint) { 111 | r.Latch.Lock() // lock the latch to evict 112 | defer r.Latch.Unlock() 113 | 114 | var victim uint = InvalidDbId 115 | var maxKDistans uint64 = 0 116 | var mostRecentAccess uint64 = 0 117 | 118 | for dbid, node := range r.Nodes { 119 | if !node.Evictable { 120 | continue 121 | } 122 | 123 | currKDistance := node.GetKDistance(&r.CurrTime, r.K) 124 | if currKDistance == math.MaxUint64 && mostRecentAccess < node.GetMostRecentAccess() { 125 | victim = uint(dbid) 126 | maxKDistans = currKDistance 127 | mostRecentAccess = node.GetMostRecentAccess() 128 | continue 129 | } 130 | 131 | if currKDistance > maxKDistans { 132 | victim = uint(dbid) 133 | maxKDistans = currKDistance 134 | mostRecentAccess = node.GetMostRecentAccess() 135 | } 136 | } 137 | 138 | if victim != InvalidDbId { 139 | r.Size -- 140 | delete(r.Nodes, victim) 141 | } 142 | 143 | return victim 144 | } 145 | 146 | func (r *LruReplacer) SetEvictable(dbid uint, evictable bool) { 147 | r.Latch.Lock() // lock the latch 148 | defer r.Latch.Unlock() 149 | if _,ok := r.Nodes[dbid]; !ok { 150 | return 151 | } 152 | 153 | if r.Nodes[dbid].Evictable == evictable { 154 | return 155 | } 156 | 157 | r.Nodes[dbid].Evictable = evictable 158 | } -------------------------------------------------------------------------------- /pool/replacer_test.go: -------------------------------------------------------------------------------- 1 | package pool_test 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | 7 | "github.com/rag-nar1/tcp-duckdb/pool" 8 | "github.com/rag-nar1/tcp-duckdb/globals" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | 13 | func TestReaplacerBasic(t *testing.T) { 14 | var dbid1 uint = 1 15 | var dbid2 uint = 2 16 | replacer := pool.NewLruReplacer(2) 17 | replacer.RecordAccess(dbid1) 18 | replacer.RecordAccess(dbid1) 19 | replacer.RecordAccess(dbid2) 20 | replacer.SetEvictable(dbid1, true) 21 | replacer.SetEvictable(dbid2, true) 22 | evicted := replacer.Evict() 23 | assert.Equal(t, dbid2, evicted) 24 | } 25 | 26 | func TestReaplacerMain(t *testing.T) { 27 | var dbid1 uint = 1 28 | var dbid2 uint = 2 29 | replacer := pool.NewLruReplacer(2) 30 | replacer.RecordAccess(dbid1) 31 | replacer.RecordAccess(dbid1) 32 | replacer.RecordAccess(dbid2) 33 | replacer.RecordAccess(dbid2) 34 | replacer.SetEvictable(dbid1, true) 35 | replacer.SetEvictable(dbid2, true) 36 | evicted := replacer.Evict() 37 | assert.Equal(t, dbid1, evicted) 38 | 39 | replacer.SetEvictable(dbid2, false) 40 | replacer.RecordAccess(dbid1) 41 | replacer.RecordAccess(dbid1) 42 | evicted = replacer.Evict() 43 | assert.Equal(t, pool.InvalidDbId, evicted) 44 | 45 | replacer.SetEvictable(dbid1, true) 46 | evicted = replacer.Evict() 47 | assert.Equal(t, dbid1, evicted) 48 | } 49 | 50 | func TestReaplacerConcurruncy(t *testing.T) { 51 | replacer := pool.NewLruReplacer(3) 52 | var wg sync.WaitGroup // WaitGroup to synchronize goroutines 53 | for i := 1; i <= int(globals.DbPoolSize); i ++ { 54 | wg.Add(1) 55 | go func(dbid uint, replacer *pool.LruReplacer) { 56 | for time := 0; time < 3; time ++ { 57 | if err := replacer.RecordAccess(uint(dbid)); err != nil { 58 | panic(err) 59 | } 60 | } 61 | if dbid % 2 == 1 { 62 | replacer.SetEvictable(dbid, true) 63 | } 64 | wg.Done() 65 | }(uint(i), replacer) 66 | } 67 | wg.Wait() 68 | 69 | replacer.SetEvictable(1, false) 70 | err := replacer.RecordAccess(1) 71 | assert.Nil(t, err) 72 | 73 | evicted := replacer.Evict() 74 | assert.Equal(t, uint(1), evicted % 2) 75 | assert.NotEqual(t, uint(1) , evicted) 76 | 77 | for i := 1; i <= int(globals.DbPoolSize); i ++ { 78 | wg.Add(1) 79 | go func(dbid uint, replacer *pool.LruReplacer) { 80 | replacer.SetEvictable(dbid, false) 81 | wg.Done() 82 | }(uint(i), replacer) 83 | } 84 | wg.Wait() 85 | 86 | replacer.SetEvictable(1, true) 87 | evicted = replacer.Evict() 88 | assert.Equal(t, uint(1) , evicted) 89 | } 90 | 91 | -------------------------------------------------------------------------------- /railway.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | builder = "DOCKERFILE" 3 | dockerfilePath = "Dockerfile" 4 | 5 | [deploy] 6 | numReplicas = 1 7 | restartPolicyType = "ON_FAILURE" 8 | restartPolicyMaxRetries = 10 -------------------------------------------------------------------------------- /request_handler/request_handler.go: -------------------------------------------------------------------------------- 1 | package request_handler 2 | 3 | import "github.com/rag-nar1/tcp-duckdb/pool" 4 | 5 | type Request struct { 6 | Dbname string 7 | Response chan pool.Connection 8 | Err chan error 9 | } 10 | 11 | func NewRequest(dbname string) *Request { 12 | return &Request{ 13 | Dbname: dbname, 14 | Response: make(chan pool.Connection, 1), 15 | Err: make(chan error, 1), 16 | } 17 | } 18 | 19 | type RequestHandler struct { 20 | Requests chan *Request 21 | PoolManger *pool.Pool 22 | } 23 | 24 | func NewRequestHandler() *RequestHandler { 25 | return &RequestHandler{ 26 | Requests: make(chan *Request), 27 | PoolManger: pool.NewPool(), 28 | } 29 | } 30 | 31 | func HandleRequest(rh *RequestHandler, curr *Request) { 32 | for { 33 | connection, err := rh.PoolManger.Get(curr.Dbname) 34 | if err != nil && err.Error() == pool.LruReplacerFullErrorStmt { 35 | continue 36 | } 37 | 38 | if err != nil { 39 | close(curr.Response) 40 | curr.Err <- err 41 | break 42 | } 43 | 44 | curr.Response <- connection 45 | close(curr.Err) 46 | break 47 | } 48 | } 49 | 50 | func (rh *RequestHandler) Spin() { 51 | for curr := range rh.Requests { 52 | go HandleRequest(rh, curr) 53 | } 54 | } 55 | 56 | func (rh *RequestHandler) Push(req *Request) { 57 | rh.Requests <- req 58 | } 59 | -------------------------------------------------------------------------------- /request_handler/request_handler_test.go: -------------------------------------------------------------------------------- 1 | package request_handler_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "testing" 9 | 10 | "github.com/joho/godotenv" 11 | "github.com/rag-nar1/tcp-duckdb/request_handler" 12 | 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func CleanUp() { 17 | files, err := filepath.Glob("../storge/users/*") 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | for _, file := range files { 23 | err := os.Remove(file) 24 | if err != nil { 25 | panic(err) 26 | } 27 | } 28 | } 29 | 30 | func TestRHBasic(t *testing.T) { 31 | t.Cleanup(CleanUp) 32 | if err := godotenv.Load("../.env"); err != nil { 33 | panic(err) 34 | } 35 | 36 | rh := request_handler.NewRequestHandler() 37 | go rh.Spin() 38 | 39 | req := request_handler.NewRequest("db1") 40 | rh.Push(req) 41 | 42 | connection := <- req.Response 43 | assert.NotNil(t, connection) 44 | _,err := connection.DB().Exec("create table t1(id int);") 45 | assert.Nil(t, err) 46 | } 47 | 48 | func TestRHConcurruncy(t *testing.T) { 49 | t.Cleanup(CleanUp) 50 | if err := godotenv.Load("../.env"); err != nil { 51 | panic(err) 52 | } 53 | rh := request_handler.NewRequestHandler() 54 | go rh.Spin() 55 | 56 | threads := 10 57 | 58 | var wg sync.WaitGroup 59 | 60 | for i := 1; i <= threads; i ++ { 61 | wg.Add(1) 62 | go func(t *testing.T, rh *request_handler.RequestHandler, dbname string) { 63 | req := request_handler.NewRequest(dbname) 64 | rh.Push(req) 65 | 66 | connection := <- req.Response 67 | assert.NotNil(t, connection) 68 | defer connection.Destroy() 69 | 70 | _,err := connection.DB().Exec("create table t1(id int);") 71 | assert.Nil(t, err) 72 | wg.Done() 73 | }(t, rh, fmt.Sprintf("db%d", i)) 74 | } 75 | wg.Wait() 76 | 77 | for i := 1; i <= threads; i ++ { 78 | wg.Add(1) 79 | go func(t *testing.T, rh *request_handler.RequestHandler, dbname string) { 80 | req := request_handler.NewRequest(dbname) 81 | rh.Push(req) 82 | 83 | connection := <- req.Response 84 | assert.NotNil(t, connection) 85 | defer connection.Destroy() 86 | 87 | _,err := connection.DB().Exec("insert into t1(id) values(1);") 88 | assert.Nil(t, err) 89 | wg.Done() 90 | }(t, rh, fmt.Sprintf("db%d", i)) 91 | } 92 | wg.Wait() 93 | for i := 1; i <= threads; i ++ { 94 | wg.Add(1) 95 | go func(t *testing.T, rh *request_handler.RequestHandler, dbname string) { 96 | req := request_handler.NewRequest(dbname) 97 | rh.Push(req) 98 | 99 | connection := <- req.Response 100 | assert.NotNil(t, connection) 101 | defer connection.Destroy() 102 | 103 | res := connection.DB().QueryRow("select * from t1 limit 1;") 104 | assert.NotNil(t, res) 105 | var id int 106 | err := res.Scan(&id) 107 | assert.Nil(t, err) 108 | assert.Equal(t, 1, id) 109 | wg.Done() 110 | }(t, rh, fmt.Sprintf("db%d", i)) 111 | } 112 | wg.Wait() 113 | } -------------------------------------------------------------------------------- /response/error.go: -------------------------------------------------------------------------------- 1 | package response 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | func Error(w *bufio.Writer, msg []byte) error { 10 | msg = append([]byte("Error: "), msg...) 11 | if _, err := w.Write(msg); err != nil { 12 | return err 13 | } 14 | return w.Flush() 15 | } 16 | 17 | func BadRequest(w *bufio.Writer) error { 18 | return Error(w, []byte(BadRequestMsg)) 19 | } 20 | 21 | func InternalError(w *bufio.Writer) error { 22 | return Error(w, []byte(InternalErrorMSG)) 23 | } 24 | 25 | func UnauthorizedError(w *bufio.Writer) error { 26 | return Error(w, []byte(UnauthorizedMSG)) 27 | } 28 | 29 | func DoesNotExist(w *bufio.Writer, prefix string, objNames ...string) error { 30 | objName := strings.Join(objNames, ",") 31 | return Error(w, []byte(prefix + " " + objName + " " + DoesNotExistMsg)) 32 | } 33 | 34 | func DoesNotExistDatabse(w *bufio.Writer, dbname string) error { 35 | return DoesNotExist(w, "Database", dbname) 36 | } 37 | 38 | func DoesNotExistUser(w *bufio.Writer, username string) error { 39 | return DoesNotExist(w, "User", username) 40 | } 41 | 42 | func DoesNotExistTables(w *bufio.Writer, tables ...string) error { 43 | return DoesNotExist(w, "Tables", tables...) 44 | } 45 | 46 | 47 | func AccessDenied(w *bufio.Writer, prtObj, prtVal, childObj, childVal string) error { 48 | return Error(w, []byte(fmt.Sprintf(AccessDeniedMsg, prtObj, prtVal, childObj, childVal))) 49 | } 50 | 51 | func AccesDeniedOverDatabase(w *bufio.Writer, username, dbname string) error { 52 | return AccessDenied(w, "User", username, "Database", dbname) 53 | } 54 | 55 | func AccesDeniedOverTables(w *bufio.Writer, username string, tables []byte) error { 56 | return AccessDenied(w, "User", username, "Tables", string(tables)) 57 | } -------------------------------------------------------------------------------- /response/statements.go: -------------------------------------------------------------------------------- 1 | package response 2 | 3 | const ( 4 | // error messege 5 | BadRequestMsg = "Bad Request" 6 | InternalErrorMSG = "Internal Error" 7 | UnauthorizedMSG = "Unauthorized" 8 | DoesNotExistMsg = " Does Not Exist" 9 | AccessDeniedMsg = `Access Denied: %s "%s" does not have access over %s "%s"` 10 | 11 | 12 | // success messege 13 | SuccessMsg = "Success" 14 | ) -------------------------------------------------------------------------------- /response/success.go: -------------------------------------------------------------------------------- 1 | package response 2 | 3 | import ( 4 | "bufio" 5 | ) 6 | 7 | func Success(w *bufio.Writer) error { 8 | if _, err := w.Write([]byte(SuccessMsg)); err != nil { 9 | return err 10 | } 11 | return w.Flush() 12 | } 13 | 14 | func WriteData(w *bufio.Writer, data []byte) error { 15 | if _, err := w.Write(data); err != nil { 16 | return err 17 | } 18 | return w.Flush() 19 | } -------------------------------------------------------------------------------- /scripts/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | GO_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.go$' || true) 3 | 4 | if [ -n "$GO_FILES" ]; then 5 | echo -e "\033[32mFormatting code with go fmt...\033[0m" 6 | go fmt $GO_FILES 7 | git add $GO_FILES # Stage formatted files 8 | fi 9 | 10 | -------------------------------------------------------------------------------- /server/config.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "database/sql" 5 | "log" 6 | "os" 7 | 8 | "github.com/rag-nar1/tcp-duckdb/globals" 9 | "github.com/rag-nar1/tcp-duckdb/request_handler" 10 | "github.com/rag-nar1/tcp-duckdb/utils" 11 | 12 | "github.com/joho/godotenv" 13 | _ "github.com/mattn/go-sqlite3" 14 | ) 15 | 16 | type Server struct { 17 | // db connection bool 18 | Sqlitedb *sql.DB 19 | Dbstmt map[string]*sql.Stmt 20 | Pool *request_handler.RequestHandler 21 | Port string 22 | Address string 23 | InfoLog *log.Logger 24 | ErrorLog *log.Logger 25 | } 26 | 27 | var Serv *Server 28 | 29 | // cread prepared statments to use in executing queries 30 | func (s *Server) PrepareStmt() { 31 | var tmpStmt *sql.Stmt 32 | var err error 33 | for _, stmt := range globals.PreparedStmtStrings { 34 | tmpStmt, err = s.Sqlitedb.Prepare(stmt[1]) 35 | if err != nil { 36 | s.ErrorLog.Fatal(err) 37 | } 38 | 39 | s.Dbstmt[stmt[0]] = tmpStmt 40 | } 41 | } 42 | 43 | // create the only superuser user if not already created "duck" with an initial password "duck" 44 | func (s *Server) CreateSuper() { 45 | hashedPassword := utils.Hash("duck") 46 | res, err := s.Sqlitedb.Exec("INSERT OR IGNORE INTO user(username, password, usertype) values('duck', ?, 'super')", hashedPassword) 47 | if err != nil { 48 | s.ErrorLog.Fatal(err) 49 | } 50 | affected, _ := res.RowsAffected() 51 | if affected == 1 { 52 | s.InfoLog.Println("Super user created") 53 | } 54 | } 55 | 56 | func ExecuteScheme(db *sql.DB) error { 57 | scheme, err := os.ReadFile(os.Getenv("DBdir") + "server/scheme.sql") 58 | if err != nil { 59 | return err 60 | } 61 | _, err = db.Exec(string(scheme)) 62 | if err != nil { 63 | return err 64 | } 65 | return nil 66 | } 67 | 68 | func NewServer() error { 69 | // Check if database file exists 70 | log.Println(os.Getenv("DBdir") + "server/" + os.Getenv("ServerDbFile")) 71 | dbPath := os.Getenv("DBdir") + "server/" + os.Getenv("ServerDbFile") 72 | executeScheme := false 73 | if _, err := os.Stat(dbPath); os.IsNotExist(err) { 74 | executeScheme = true 75 | } 76 | 77 | // Open database connection 78 | dbconn, err := sql.Open("sqlite3", dbPath) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | // Test connection 84 | err = dbconn.Ping() 85 | if err != nil { 86 | return err 87 | } 88 | 89 | if executeScheme { 90 | err = ExecuteScheme(dbconn) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | 96 | Serv = &Server{ 97 | Sqlitedb: dbconn, 98 | Port: os.Getenv("ServerPort"), 99 | Dbstmt: make(map[string]*sql.Stmt), 100 | InfoLog: log.New(os.Stdout, "INFO\t", log.Ldate|log.Ltime), 101 | ErrorLog: log.New(os.Stdout, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile), 102 | Pool: request_handler.NewRequestHandler(), 103 | } 104 | Serv.Address = os.Getenv("ServerAddr") + ":" + Serv.Port 105 | return nil 106 | } 107 | 108 | func Init() { 109 | // Create default loggers for initialization errors before Serv is ready 110 | errorLog := log.New(os.Stdout, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile) 111 | 112 | // First try to use existing environment variables 113 | 114 | // Only try to load .env file if ServerAddr is not set 115 | if os.Getenv("ServerAddr") == "" { 116 | // Try to load .env file from different locations 117 | err1 := godotenv.Load(".env") 118 | err2 := godotenv.Load("../.env") 119 | if err1 != nil && err2 != nil { 120 | errorLog.Fatal("Failed to load .env file:", err1, err2) 121 | } 122 | } 123 | 124 | if err := NewServer(); err != nil { 125 | panic(err) 126 | } 127 | 128 | Serv.CreateSuper() 129 | Serv.PrepareStmt() 130 | go Serv.Pool.Spin() 131 | } 132 | -------------------------------------------------------------------------------- /storge/server/audit.sql: -------------------------------------------------------------------------------- 1 | -- Complete PostgreSQL Audit System with Automatic Triggers 2 | -- This script creates a comprehensive audit system that: 3 | -- 1. Records all data changes (inserts, updates, deletes) across all tables 4 | -- 2. Automatically adds audit triggers to newly created tables 5 | -- 3. Provides utility functions for managing the audit system 6 | 7 | -- Step 1: Create the audit schema 8 | CREATE SCHEMA IF NOT EXISTS audit; 9 | 10 | -- Step 2: Create the audit table to store all change records 11 | CREATE TABLE IF NOT EXISTS audit.logged_actions ( 12 | event_id BIGSERIAL PRIMARY KEY, 13 | schema_name TEXT NOT NULL, 14 | table_name TEXT NOT NULL, 15 | table_pk TEXT, -- Primary key column(s) value 16 | table_pk_column TEXT, -- Primary key column(s) name 17 | action_tstamp TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, 18 | action TEXT NOT NULL CHECK (action IN ('I','D','U','T')), -- Insert, Delete, Update, Truncate 19 | original_data JSONB, -- Previous data for updates and deletes 20 | new_data JSONB, -- New data for inserts and updates 21 | changed_fields JSONB, -- Changed fields for updates 22 | transaction_id BIGINT, -- Transaction ID 23 | application_name TEXT, -- Application name from current_setting 24 | client_addr INET NULL, -- Client IP address 25 | client_port INTEGER NULL, -- Client port 26 | session_user_name TEXT, -- Session user 27 | current_user_name TEXT -- Current user performing action 28 | ); 29 | 30 | -- Create comments on the audit table for better documentation 31 | COMMENT ON TABLE audit.logged_actions IS 'History of auditable actions on audited tables'; 32 | COMMENT ON COLUMN audit.logged_actions.event_id IS 'Unique identifier for each auditable event'; 33 | COMMENT ON COLUMN audit.logged_actions.schema_name IS 'Database schema audited table is in'; 34 | COMMENT ON COLUMN audit.logged_actions.table_name IS 'Non-schema-qualified table name of table event occured in'; 35 | COMMENT ON COLUMN audit.logged_actions.table_pk IS 'Primary key value of the affected row'; 36 | COMMENT ON COLUMN audit.logged_actions.table_pk_column IS 'Name of the primary key column'; 37 | COMMENT ON COLUMN audit.logged_actions.action_tstamp IS 'Transaction start timestamp for tx in which the audited event occurred'; 38 | COMMENT ON COLUMN audit.logged_actions.action IS 'Action type; I = insert, D = delete, U = update, T = truncate'; 39 | COMMENT ON COLUMN audit.logged_actions.original_data IS 'Record value before modification (for updates and deletes)'; 40 | COMMENT ON COLUMN audit.logged_actions.new_data IS 'New record value (for inserts and updates)'; 41 | COMMENT ON COLUMN audit.logged_actions.changed_fields IS 'Updated fields (for updates only)'; 42 | COMMENT ON COLUMN audit.logged_actions.client_addr IS 'IP address of client that issued query'; 43 | COMMENT ON COLUMN audit.logged_actions.client_port IS 'Port address of client that issued query'; 44 | COMMENT ON COLUMN audit.logged_actions.session_user_name IS 'Login / session user whose actions are being audited'; 45 | COMMENT ON COLUMN audit.logged_actions.current_user_name IS 'User who actually performed the action'; 46 | 47 | -- Create indexes for better query performance 48 | CREATE INDEX IF NOT EXISTS logged_actions_schema_table_idx 49 | ON audit.logged_actions(schema_name, table_name); 50 | 51 | CREATE INDEX IF NOT EXISTS logged_actions_action_tstamp_idx 52 | ON audit.logged_actions(action_tstamp); 53 | 54 | CREATE INDEX IF NOT EXISTS logged_actions_action_idx 55 | ON audit.logged_actions(action); 56 | 57 | -- Step 3: Create the audit trigger function that handles data changes 58 | CREATE OR REPLACE FUNCTION audit.if_modified_func() 59 | RETURNS TRIGGER AS $body$ 60 | DECLARE 61 | v_old_data JSONB; 62 | v_new_data JSONB; 63 | v_changed_fields JSONB; 64 | v_primary_key_column TEXT; 65 | v_primary_key_value TEXT; 66 | v_action TEXT; 67 | BEGIN 68 | -- Set the action type 69 | IF TG_OP = 'INSERT' THEN 70 | v_action := 'I'; 71 | ELSIF TG_OP = 'UPDATE' THEN 72 | v_action := 'U'; 73 | ELSIF TG_OP = 'DELETE' THEN 74 | v_action := 'D'; 75 | ELSIF TG_OP = 'TRUNCATE' THEN 76 | v_action := 'T'; 77 | ELSE 78 | RAISE EXCEPTION 'Unsupported trigger operation: %', TG_OP; 79 | END IF; 80 | 81 | -- Validate trigger type 82 | IF TG_WHEN <> 'AFTER' THEN 83 | RAISE EXCEPTION 'audit.if_modified_func() may only run as an AFTER trigger'; 84 | END IF; 85 | 86 | -- Determine primary key column if it exists 87 | -- This improved version will not fail if no primary key exists 88 | SELECT a.attname INTO v_primary_key_column 89 | FROM pg_index i 90 | JOIN pg_attribute a ON a.attrelid = i.indrelid 91 | AND a.attnum = ANY(i.indkey) 92 | WHERE i.indrelid = TG_RELID 93 | AND i.indisprimary 94 | LIMIT 1; 95 | 96 | -- Get the primary key value if primary key exists 97 | IF v_primary_key_column IS NOT NULL THEN 98 | IF TG_OP = 'UPDATE' OR TG_OP = 'DELETE' THEN 99 | EXECUTE 'SELECT ($1).' || quote_ident(v_primary_key_column) 100 | INTO v_primary_key_value 101 | USING OLD; 102 | ELSIF TG_OP = 'INSERT' THEN 103 | EXECUTE 'SELECT ($1).' || quote_ident(v_primary_key_column) 104 | INTO v_primary_key_value 105 | USING NEW; 106 | END IF; 107 | END IF; 108 | 109 | -- Convert to JSON data formats based on operation 110 | IF (TG_OP = 'UPDATE' OR TG_OP = 'DELETE') THEN 111 | v_old_data = to_jsonb(OLD); 112 | END IF; 113 | 114 | IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN 115 | v_new_data = to_jsonb(NEW); 116 | END IF; 117 | 118 | -- For UPDATE operations, calculate the changed fields 119 | IF (TG_OP = 'UPDATE') THEN 120 | -- Calculate changed fields by comparing old and new data 121 | SELECT jsonb_object_agg(key, value) INTO v_changed_fields 122 | FROM jsonb_each(v_new_data) 123 | WHERE NOT jsonb_contains(v_old_data, jsonb_build_object(key, value)) 124 | OR NOT v_old_data ? key; 125 | END IF; 126 | 127 | -- Insert audit log entry 128 | INSERT INTO audit.logged_actions ( 129 | schema_name, 130 | table_name, 131 | table_pk, 132 | table_pk_column, 133 | action, 134 | original_data, 135 | new_data, 136 | changed_fields, 137 | transaction_id, 138 | application_name, 139 | client_addr, 140 | client_port, 141 | session_user_name, 142 | current_user_name 143 | ) VALUES ( 144 | TG_TABLE_SCHEMA::TEXT, -- schema_name 145 | TG_TABLE_NAME::TEXT, -- table_name 146 | v_primary_key_value, -- table_pk 147 | v_primary_key_column, -- table_pk_column 148 | v_action, -- action 149 | v_old_data, -- original_data 150 | v_new_data, -- new_data 151 | v_changed_fields, -- changed_fields 152 | txid_current(), -- transaction_id 153 | current_setting('application_name', true), -- application_name 154 | inet_client_addr(), -- client_addr 155 | inet_client_port(), -- client_port 156 | session_user::TEXT, -- session_user_name 157 | current_user::TEXT -- current_user_name 158 | ); 159 | 160 | RETURN NULL; -- For AFTER triggers, return value is ignored 161 | END; 162 | $body$ 163 | LANGUAGE plpgsql 164 | SECURITY DEFINER; 165 | 166 | COMMENT ON FUNCTION audit.if_modified_func() IS 'Trigger function that logs changes to the audit.logged_actions table'; 167 | 168 | -- Step 4: Function to automatically create audit triggers for all tables in a schema 169 | CREATE OR REPLACE FUNCTION audit.create_audit_triggers_for_schema(target_schema TEXT) 170 | RETURNS VOID AS $$ 171 | DECLARE 172 | table_record RECORD; 173 | BEGIN 174 | -- Loop through all tables in the schema and add audit triggers 175 | FOR table_record IN 176 | SELECT tablename 177 | FROM pg_tables 178 | WHERE schemaname = target_schema 179 | AND tablename NOT LIKE 'pg_%' 180 | AND tablename <> 'logged_actions' 181 | LOOP 182 | -- Create the audit trigger for the current table 183 | EXECUTE format(' 184 | DROP TRIGGER IF EXISTS audit_trigger_row ON %I.%I; 185 | CREATE TRIGGER audit_trigger_row 186 | AFTER INSERT OR UPDATE OR DELETE ON %I.%I 187 | FOR EACH ROW EXECUTE FUNCTION audit.if_modified_func(); 188 | 189 | DROP TRIGGER IF EXISTS audit_trigger_stmt ON %I.%I; 190 | CREATE TRIGGER audit_trigger_stmt 191 | AFTER TRUNCATE ON %I.%I 192 | FOR EACH STATEMENT EXECUTE FUNCTION audit.if_modified_func(); 193 | ', 194 | target_schema, table_record.tablename, 195 | target_schema, table_record.tablename, 196 | target_schema, table_record.tablename, 197 | target_schema, table_record.tablename 198 | ); 199 | 200 | RAISE NOTICE 'Added audit triggers to table: %.%', target_schema, table_record.tablename; 201 | END LOOP; 202 | END; 203 | $$ LANGUAGE plpgsql; 204 | 205 | COMMENT ON FUNCTION audit.create_audit_triggers_for_schema(TEXT) IS 'Function to add audit triggers to all tables in a schema'; 206 | 207 | -- Step 5: Function to create audit triggers for all tables in the database (except system schemas) 208 | CREATE OR REPLACE FUNCTION audit.create_audit_triggers_for_all_tables() 209 | RETURNS VOID AS $$ 210 | DECLARE 211 | schema_record RECORD; 212 | BEGIN 213 | -- Loop through all non-system schemas and add audit triggers 214 | FOR schema_record IN 215 | SELECT nspname 216 | FROM pg_namespace 217 | WHERE nspname NOT LIKE 'pg_%' 218 | AND nspname <> 'information_schema' 219 | AND nspname <> 'audit' 220 | LOOP 221 | -- Call the function to create audit triggers for all tables in the current schema 222 | PERFORM audit.create_audit_triggers_for_schema(schema_record.nspname); 223 | END LOOP; 224 | 225 | RAISE NOTICE 'Added audit triggers to all tables in all non-system schemas'; 226 | END; 227 | $$ LANGUAGE plpgsql; 228 | 229 | COMMENT ON FUNCTION audit.create_audit_triggers_for_all_tables() IS 'Function to add audit triggers to all tables in all non-system schemas'; 230 | 231 | -- Step 6: Function to automatically add audit triggers to newly created tables 232 | CREATE OR REPLACE FUNCTION audit.add_audit_trigger_for_new_table() 233 | RETURNS event_trigger AS $$ 234 | DECLARE 235 | obj RECORD; 236 | schema_name TEXT; 237 | table_name TEXT; 238 | BEGIN 239 | -- Loop through objects created in this transaction 240 | FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands() WHERE command_tag = 'CREATE TABLE' 241 | LOOP 242 | -- Extract schema and table name 243 | schema_name := obj.schema_name; 244 | table_name := obj.object_identity; 245 | 246 | -- Skip audit tables to avoid recursion 247 | IF schema_name = 'audit' THEN 248 | CONTINUE; 249 | END IF; 250 | 251 | -- Skip tables with table name that contains the full schema path 252 | IF position('.' in table_name) > 0 THEN 253 | table_name := substring(table_name from position('.' in table_name) + 1); 254 | END IF; 255 | 256 | -- Add audit triggers to the new table 257 | EXECUTE format(' 258 | CREATE TRIGGER audit_trigger_row 259 | AFTER INSERT OR UPDATE OR DELETE ON %I.%I 260 | FOR EACH ROW EXECUTE FUNCTION audit.if_modified_func(); 261 | 262 | CREATE TRIGGER audit_trigger_stmt 263 | AFTER TRUNCATE ON %I.%I 264 | FOR EACH STATEMENT EXECUTE FUNCTION audit.if_modified_func();', 265 | schema_name, table_name, 266 | schema_name, table_name 267 | ); 268 | 269 | RAISE NOTICE 'Added audit triggers to new table: %.%', schema_name, table_name; 270 | END LOOP; 271 | END; 272 | $$ LANGUAGE plpgsql; 273 | 274 | COMMENT ON FUNCTION audit.add_audit_trigger_for_new_table() IS 'Function to automatically add audit triggers to newly created tables'; 275 | 276 | -- Step 7: Create the event trigger that fires when a table is created 277 | DROP EVENT TRIGGER IF EXISTS audit_table_creation_trigger; 278 | CREATE EVENT TRIGGER audit_table_creation_trigger 279 | ON ddl_command_end 280 | WHEN TAG IN ('CREATE TABLE') 281 | EXECUTE FUNCTION audit.add_audit_trigger_for_new_table(); 282 | 283 | COMMENT ON EVENT TRIGGER audit_table_creation_trigger IS 'Event trigger that adds audit triggers to newly created tables'; 284 | 285 | -- Step 8: Function to enable or disable the event trigger (useful for maintenance) 286 | CREATE OR REPLACE FUNCTION audit.toggle_audit_creation_trigger(enable BOOLEAN) 287 | RETURNS VOID AS $$ 288 | BEGIN 289 | IF enable THEN 290 | ALTER EVENT TRIGGER audit_table_creation_trigger ENABLE; 291 | RAISE NOTICE 'Automatic audit triggers for new tables enabled'; 292 | ELSE 293 | ALTER EVENT TRIGGER audit_table_creation_trigger DISABLE; 294 | RAISE NOTICE 'Automatic audit triggers for new tables disabled'; 295 | END IF; 296 | END; 297 | $$ LANGUAGE plpgsql; 298 | 299 | COMMENT ON FUNCTION audit.toggle_audit_creation_trigger(BOOLEAN) IS 'Function to enable or disable automatic audit triggers for new tables'; 300 | 301 | -- Step 9: Function to set up the complete audit system 302 | CREATE OR REPLACE FUNCTION audit.setup_complete_audit_system() 303 | RETURNS VOID AS $$ 304 | BEGIN 305 | -- Add triggers to all existing tables 306 | PERFORM audit.create_audit_triggers_for_all_tables(); 307 | 308 | -- Enable the event trigger for new tables 309 | PERFORM audit.toggle_audit_creation_trigger(true); 310 | 311 | RAISE NOTICE 'Audit system is now fully set up for existing and future tables'; 312 | END; 313 | $$ LANGUAGE plpgsql; 314 | 315 | COMMENT ON FUNCTION audit.setup_complete_audit_system() IS 'Function to set up the complete audit system for existing and future tables'; 316 | 317 | -- Step 10: Function to get basic audit statistics 318 | CREATE OR REPLACE FUNCTION audit.get_audit_statistics() 319 | RETURNS TABLE( 320 | schema_name TEXT, 321 | table_name TEXT, 322 | inserts BIGINT, 323 | updates BIGINT, 324 | deletes BIGINT, 325 | truncates BIGINT, 326 | total_actions BIGINT, 327 | last_action_time TIMESTAMPTZ 328 | ) AS $$ 329 | BEGIN 330 | RETURN QUERY 331 | SELECT 332 | la.schema_name, 333 | la.table_name, 334 | SUM(CASE WHEN la.action = 'I' THEN 1 ELSE 0 END)::BIGINT AS inserts, 335 | SUM(CASE WHEN la.action = 'U' THEN 1 ELSE 0 END)::BIGINT AS updates, 336 | SUM(CASE WHEN la.action = 'D' THEN 1 ELSE 0 END)::BIGINT AS deletes, 337 | SUM(CASE WHEN la.action = 'T' THEN 1 ELSE 0 END)::BIGINT AS truncates, 338 | COUNT(*)::BIGINT AS total_actions, 339 | MAX(la.action_tstamp) AS last_action_time 340 | FROM audit.logged_actions la 341 | GROUP BY la.schema_name, la.table_name 342 | ORDER BY la.schema_name, la.table_name; 343 | END; 344 | $$ LANGUAGE plpgsql; 345 | 346 | COMMENT ON FUNCTION audit.get_audit_statistics() IS 'Function to get basic statistics about audit records'; 347 | 348 | -- Step 11: Function to clean up old audit records based on retention period 349 | CREATE OR REPLACE FUNCTION audit.cleanup_old_audit_records(retention_days INTEGER DEFAULT 90) 350 | RETURNS INTEGER AS $$ 351 | DECLARE 352 | deleted_count INTEGER; 353 | BEGIN 354 | DELETE FROM audit.logged_actions 355 | WHERE action_tstamp < (CURRENT_TIMESTAMP - (retention_days || ' days')::INTERVAL); 356 | 357 | GET DIAGNOSTICS deleted_count = ROW_COUNT; 358 | 359 | RAISE NOTICE 'Deleted % audit records older than % days', deleted_count, retention_days; 360 | 361 | RETURN deleted_count; 362 | END; 363 | $$ LANGUAGE plpgsql; 364 | 365 | COMMENT ON FUNCTION audit.cleanup_old_audit_records(INTEGER) IS 'Function to clean up old audit records based on retention period'; 366 | 367 | -- Step 12: Simple view to see recent audit activity 368 | CREATE OR REPLACE VIEW audit.recent_activity AS 369 | SELECT 370 | event_id, 371 | action_tstamp, 372 | CASE 373 | WHEN action = 'I' THEN 'INSERT' 374 | WHEN action = 'U' THEN 'UPDATE' 375 | WHEN action = 'D' THEN 'DELETE' 376 | WHEN action = 'T' THEN 'TRUNCATE' 377 | END AS action_type, 378 | schema_name || '.' || table_name AS table_path, 379 | table_pk, 380 | current_user_name, 381 | client_addr 382 | FROM 383 | audit.logged_actions 384 | ORDER BY 385 | action_tstamp DESC 386 | LIMIT 100; 387 | 388 | COMMENT ON VIEW audit.recent_activity IS 'View to see recent activity in the audit log'; 389 | SELECT audit.setup_complete_audit_system(); 390 | 391 | -- Final Step: Initialize everything with a single command 392 | -- Uncomment and run this line to set up the entire audit system -------------------------------------------------------------------------------- /storge/server/scheme.sql: -------------------------------------------------------------------------------- 1 | -- Create the 'user' table 2 | CREATE TABLE user ( 3 | userid INTEGER PRIMARY KEY AUTOINCREMENT, 4 | username TEXT NOT NULL UNIQUE, 5 | password TEXT NOT NULL, 6 | usertype TEXT NOT NULL -- [super , norm] 7 | ); 8 | 9 | -- Create the 'DB' table 10 | CREATE TABLE DB ( 11 | dbid INTEGER PRIMARY KEY AUTOINCREMENT, 12 | dbname TEXT NOT NULL UNIQUE 13 | ); 14 | 15 | -- Create the 'tables' table 16 | CREATE TABLE tables ( 17 | tableid INTEGER PRIMARY KEY AUTOINCREMENT, 18 | tablename TEXT NOT NULL, 19 | dbid INTEGER NOT NULL, 20 | FOREIGN KEY (dbid) REFERENCES DB(dbid) ON DELETE CASCADE 21 | ); 22 | 23 | -- Create the 'dbprivilege' table 24 | CREATE TABLE dbprivilege ( 25 | dbid INTEGER NOT NULL, 26 | userid INTEGER NOT NULL, 27 | privilegetype TEXT NOT NULL, -- [read , write] 28 | PRIMARY KEY (dbid, userid, privilegetype), 29 | FOREIGN KEY (dbid) REFERENCES DB(dbid) ON DELETE CASCADE, 30 | FOREIGN KEY (userid) REFERENCES user(userid) ON DELETE CASCADE 31 | ); 32 | 33 | -- Create the 'tableprivilege' table 34 | CREATE TABLE tableprivilege ( 35 | tableid INTEGER NOT NULL, 36 | userid INTEGER NOT NULL, 37 | tableprivilege TEXT NOT NULL, --[select, insert, update, delete] 38 | PRIMARY KEY (tableid, userid, tableprivilege), 39 | FOREIGN KEY (tableid) REFERENCES tables(tableid) ON DELETE CASCADE, 40 | FOREIGN KEY (userid) REFERENCES user(userid) ON DELETE CASCADE 41 | ); 42 | 43 | CREATE TABLE postgres ( 44 | dbid INTEGER NOT NULL UNIQUE, 45 | connstr TEXT NOT NULL, 46 | PRIMARY KEY (dbid), 47 | FOREIGN KEY (dbid) REFERENCES DB(dbid) ON DELETE CASCADE 48 | ); 49 | -------------------------------------------------------------------------------- /utils/connection.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | 6 | "github.com/rag-nar1/tcp-duckdb/pool" 7 | "github.com/rag-nar1/tcp-duckdb/request_handler" 8 | ) 9 | 10 | 11 | func Write(writer *bufio.Writer, data []byte) error { 12 | data = append(data, '\n') 13 | if _, err := writer.Write(data); err != nil { 14 | return err 15 | } 16 | return writer.Flush() 17 | } 18 | 19 | func OpenDb(rh *request_handler.RequestHandler, dbname string) (pool.Connection, error){ 20 | req := request_handler.NewRequest(dbname) 21 | rh.Push(req) 22 | 23 | Dbconn := <- req.Response 24 | err := <- req.Err 25 | return Dbconn, err 26 | } -------------------------------------------------------------------------------- /utils/password.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import( 4 | "crypto/sha256" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "crypto/rand" 8 | "encoding/base64" 9 | "errors" 10 | "fmt" 11 | "io" 12 | ) 13 | 14 | 15 | func Hash(password string) string { 16 | sum := sha256.Sum256([]byte(password)) 17 | return string(sum[:]) 18 | } 19 | 20 | func Encrypt(plaintext string, key []byte) (string, error) { 21 | // Create cipher block 22 | block, err := aes.NewCipher(key) 23 | if err != nil { 24 | return "", fmt.Errorf("error creating cipher block: %v", err) 25 | } 26 | 27 | // Create initialization vector 28 | iv := make([]byte, aes.BlockSize) 29 | if _, err := io.ReadFull(rand.Reader, iv); err != nil { 30 | return "", fmt.Errorf("error creating IV: %v", err) 31 | } 32 | 33 | // Pad the plaintext 34 | padded := pad([]byte(plaintext)) 35 | 36 | // Create ciphertext slice 37 | ciphertext := make([]byte, len(iv)+len(padded)) 38 | copy(ciphertext, iv) 39 | 40 | // Create CBC encrypter 41 | mode := cipher.NewCBCEncrypter(block, iv) 42 | mode.CryptBlocks(ciphertext[aes.BlockSize:], padded) 43 | 44 | // Encode to base64 and return 45 | return base64.StdEncoding.EncodeToString(ciphertext), nil 46 | } 47 | 48 | // Decrypt takes a base64-encoded ciphertext and key, returns the original plaintext 49 | func Decrypt(encodedCiphertext string, key []byte) (string, error) { 50 | // Decode base64 51 | ciphertext, err := base64.StdEncoding.DecodeString(encodedCiphertext) 52 | if err != nil { 53 | return "", fmt.Errorf("error decoding base64: %v", err) 54 | } 55 | 56 | // Check for minimum length 57 | if len(ciphertext) < aes.BlockSize { 58 | return "", errors.New("ciphertext too short") 59 | } 60 | 61 | // Create cipher block 62 | block, err := aes.NewCipher(key) 63 | if err != nil { 64 | return "", fmt.Errorf("error creating cipher block: %v", err) 65 | } 66 | 67 | // Extract IV 68 | iv := ciphertext[:aes.BlockSize] 69 | ciphertext = ciphertext[aes.BlockSize:] 70 | 71 | // Create CBC decrypter 72 | mode := cipher.NewCBCDecrypter(block, iv) 73 | mode.CryptBlocks(ciphertext, ciphertext) 74 | 75 | // Unpad the result 76 | unpadded, err := unpad(ciphertext) 77 | if err != nil { 78 | return "", fmt.Errorf("error unpadding: %v", err) 79 | } 80 | 81 | return string(unpadded), nil 82 | } 83 | 84 | // pad implements PKCS7 padding 85 | func pad(data []byte) []byte { 86 | padLen := aes.BlockSize - (len(data) % aes.BlockSize) 87 | padding := make([]byte, padLen) 88 | for i := range padding { 89 | padding[i] = byte(padLen) 90 | } 91 | return append(data, padding...) 92 | } 93 | 94 | // unpad removes PKCS7 padding 95 | func unpad(data []byte) ([]byte, error) { 96 | if len(data) == 0 { 97 | return nil, errors.New("empty data") 98 | } 99 | 100 | padLen := int(data[len(data)-1]) 101 | if padLen > len(data) || padLen > aes.BlockSize { 102 | return nil, errors.New("invalid padding") 103 | } 104 | 105 | // Verify padding 106 | for i := len(data) - padLen; i < len(data); i++ { 107 | if data[i] != byte(padLen) { 108 | return nil, errors.New("invalid padding") 109 | } 110 | } 111 | 112 | return data[:len(data)-padLen], nil 113 | } -------------------------------------------------------------------------------- /utils/stringsUtils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | 9 | func Trim(s string) string { 10 | for len(s) > 0 && !(s[0] >= 33 && s[0] <= 126) { 11 | s = s[1:] 12 | } 13 | for len(s) > 0 && !(s[len(s) - 1] >= 33 && s[len(s) - 1] <= 126) { 14 | s = s[:len(s) - 1] 15 | } 16 | return s 17 | } 18 | 19 | func ToLower(s ...string) []string { 20 | for i := range s { 21 | s[i] = strings.ToLower(s[i]) 22 | } 23 | 24 | return s 25 | } 26 | 27 | func TrimList(list []string) { 28 | for i := 0; i < len(list); i ++ { 29 | list[i] = Trim(list[i]) 30 | } 31 | } 32 | 33 | func UserDbPath(dbname string) string { 34 | return os.Getenv("DBdir") + "users/" + dbname + ".db" 35 | } -------------------------------------------------------------------------------- /utils/test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | // "bufio" 5 | "database/sql" 6 | "fmt" 7 | "log" 8 | "net" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "time" 13 | 14 | "github.com/rag-nar1/tcp-duckdb/response" 15 | 16 | "github.com/joho/godotenv" 17 | _ "github.com/lib/pq" // Import the PostgreSQL driver 18 | _ "github.com/mattn/go-sqlite3" 19 | ) 20 | 21 | func StartUp() *net.TCPConn { 22 | if err := godotenv.Load("../.env"); err != nil { 23 | panic(err) 24 | } 25 | conn := Connection() 26 | return conn 27 | } 28 | 29 | func Connection() *net.TCPConn { 30 | tcpAddr, err := net.ResolveTCPAddr("tcp", "localhost:4000") 31 | if err != nil { 32 | panic(err) 33 | } 34 | conn, err := net.DialTCP("tcp", nil, tcpAddr) 35 | if err != nil { 36 | panic(err) 37 | } 38 | // Set a deadline for the operation (optional, for timeout) 39 | conn.SetDeadline(time.Now().Add(10 * time.Second)) 40 | return conn 41 | } 42 | 43 | func LoginAsAdmin(conn *net.TCPConn) error { 44 | _, err := conn.Write([]byte("login duck duck")) 45 | if err != nil { 46 | return err 47 | } 48 | res := Read(conn) 49 | if res != response.SuccessMsg { 50 | return fmt.Errorf("unauth: %s", res) 51 | } 52 | return nil 53 | } 54 | 55 | func Login(conn *net.TCPConn, username, password string) error { 56 | _, err := conn.Write([]byte(fmt.Sprintf("login %s %s", username, password))) 57 | if err != nil { 58 | return err 59 | } 60 | res := Read(conn) 61 | if res != response.SuccessMsg { 62 | return fmt.Errorf("unauth: %s", res) 63 | } 64 | return nil 65 | } 66 | 67 | func Read(conn *net.TCPConn) string { 68 | buffer := make([]byte, 4096) 69 | n, err := conn.Read(buffer) 70 | if err != nil { 71 | panic(err) 72 | } 73 | return strings.Trim(string(buffer[:n]), " \n\t") 74 | } 75 | 76 | func CreateDB(conn *net.TCPConn, dbname string) error { 77 | if _, err := conn.Write([]byte("create database " + dbname)); err != nil { 78 | return err 79 | } 80 | 81 | if res := Read(conn); res != response.SuccessMsg { 82 | return fmt.Errorf("%s", res) 83 | } 84 | return nil 85 | } 86 | 87 | func CreateUser(conn *net.TCPConn, userName, password string) error { 88 | if _, err := conn.Write([]byte("create user " + userName + " " + password)); err != nil { 89 | return err 90 | } 91 | 92 | if res := Read(conn); res != response.SuccessMsg { 93 | return fmt.Errorf("%s", res) 94 | } 95 | return nil 96 | } 97 | 98 | func UpdateUserName(conn *net.TCPConn, oldUserName, newUserName string) error { 99 | if _, err := conn.Write([]byte("update user username " + oldUserName + " " + newUserName)); err != nil { 100 | return err 101 | } 102 | 103 | if res := Read(conn); res != response.SuccessMsg { 104 | return fmt.Errorf("%s", res) 105 | } 106 | return nil 107 | } 108 | 109 | func UpdateUserPassword(conn *net.TCPConn, userName, password string) error { 110 | if _, err := conn.Write([]byte("update user password " + userName + " " + password)); err != nil { 111 | return err 112 | } 113 | 114 | if res := Read(conn); res != response.SuccessMsg { 115 | return fmt.Errorf("%s", res) 116 | } 117 | return nil 118 | } 119 | 120 | func UpdateDatabase(conn *net.TCPConn, oldDbname, newDbname string) error { 121 | if _, err := conn.Write([]byte("update database " + oldDbname + " " + newDbname)); err != nil { 122 | return err 123 | } 124 | 125 | if res := Read(conn); res != response.SuccessMsg { 126 | return fmt.Errorf("%s", res) 127 | } 128 | return nil 129 | } 130 | 131 | func CreateTable(conn *net.TCPConn, tablename string) error { 132 | if _, err := conn.Write([]byte(fmt.Sprintf("CREATE TABLE %s(id int, name text);", tablename))); err != nil { 133 | return err 134 | } 135 | 136 | if res := Read(conn); strings.HasPrefix(res, "ERROR") { 137 | return fmt.Errorf("%s", res) 138 | } 139 | return nil 140 | } 141 | 142 | func ConnectDb(conn *net.TCPConn, dbname string) error { 143 | _, err := conn.Write([]byte("connect " + dbname)) 144 | if err != nil { 145 | return err 146 | } 147 | res := Read(conn) 148 | if res != response.SuccessMsg { 149 | return fmt.Errorf("%s", res) 150 | } 151 | return nil 152 | } 153 | 154 | func Query(conn *net.TCPConn, query string) error { 155 | _, err := conn.Write([]byte(query)) 156 | if err != nil { 157 | return err 158 | } 159 | res := Read(conn) 160 | if strings.HasPrefix(res, "Error") { 161 | return fmt.Errorf("%s", res) 162 | } 163 | return nil 164 | } 165 | 166 | func QueryData(conn *net.TCPConn, query string) (string, error) { 167 | _, err := conn.Write([]byte(query)) 168 | if err != nil { 169 | return "", err 170 | } 171 | res := Read(conn) 172 | if strings.HasPrefix(res, "Error") { 173 | return "", fmt.Errorf("%s", res) 174 | } 175 | return res, nil 176 | } 177 | 178 | func GrantDb(conn *net.TCPConn, username, dbname, privilege string) error { 179 | _, err := conn.Write([]byte(fmt.Sprintf("grant database %s %s %s", dbname, username, privilege))) 180 | if err != nil { 181 | return err 182 | } 183 | res := Read(conn) 184 | if res != response.SuccessMsg { 185 | return fmt.Errorf("%s", res) 186 | } 187 | return nil 188 | } 189 | 190 | func GrantTable(conn *net.TCPConn, username, dbname, tablename, privilege string) error { 191 | _, err := conn.Write([]byte(fmt.Sprintf("grant table %s %s %s %s", dbname, tablename, username, privilege))) 192 | if err != nil { 193 | 194 | return err 195 | } 196 | res := Read(conn) 197 | if res != response.SuccessMsg { 198 | return fmt.Errorf("%s", res) 199 | } 200 | return nil 201 | } 202 | 203 | func CleanUpDb(db *sql.DB) error { 204 | files, err := filepath.Glob("../../storge/users/*") 205 | if err != nil { 206 | return err 207 | } 208 | 209 | for _, file := range files { 210 | err := os.Remove(file) 211 | if err != nil { 212 | return err 213 | } 214 | } 215 | 216 | if _, err := db.Exec("DELETE FROM DB;"); err != nil { 217 | return err 218 | } 219 | 220 | if _, err := db.Exec("DELETE FROM dbprivilege;"); err != nil { 221 | return err 222 | } 223 | if _, err := db.Exec("DELETE FROM postgres;"); err != nil { 224 | return err 225 | } 226 | 227 | return nil 228 | } 229 | 230 | func CleanUpUsers(db *sql.DB) error { 231 | 232 | if _, err := db.Exec("DELETE FROM user WHERE usertype not like 'super';"); err != nil { 233 | return err 234 | } 235 | 236 | return nil 237 | } 238 | 239 | func CleanUpTables(db *sql.DB) error { 240 | 241 | if _, err := db.Exec("DELETE FROM tables;"); err != nil { 242 | return err 243 | } 244 | 245 | if _, err := db.Exec("DELETE FROM tableprivilege;"); err != nil { 246 | return err 247 | } 248 | 249 | return nil 250 | } 251 | 252 | func CleanUp() { 253 | db, err := sql.Open("sqlite3", "../../storge/server/db.sqlite3") 254 | if err != nil { 255 | log.Fatal(err) 256 | 257 | } 258 | defer db.Close() 259 | 260 | if err := CleanUpDb(db); err != nil { 261 | log.Fatal(err) 262 | 263 | } 264 | 265 | if err := CleanUpUsers(db); err != nil { 266 | log.Fatal(err) 267 | 268 | } 269 | 270 | if err := CleanUpTables(db); err != nil { 271 | log.Fatal(err) 272 | 273 | } 274 | if err := CleanUpLink(); err != nil { 275 | log.Fatal(err) 276 | 277 | } 278 | } 279 | 280 | func Link(conn *net.TCPConn, dbname, connStr string) error { 281 | if _, err := conn.Write([]byte(fmt.Sprintf("link %s %s", dbname, connStr))); err != nil { 282 | return err 283 | } 284 | res := Read(conn) 285 | if strings.HasPrefix(res, "Error") { 286 | return fmt.Errorf("%s", res) 287 | } 288 | return nil 289 | } 290 | 291 | func CleanUpLink() error { 292 | pq, err := sql.Open("postgres", "postgresql://postgres:1242003@localhost:5432") 293 | if err != nil { 294 | return err 295 | } 296 | defer pq.Close() 297 | 298 | if _, err := pq.Exec("Drop database testdb;"); err != nil { 299 | return err 300 | } 301 | if _, err := pq.Exec("create database testdb;"); err != nil { 302 | return err 303 | } 304 | pq.Close() 305 | pq, err = sql.Open("postgres", "postgresql://postgres:1242003@localhost:5432/testdb") 306 | if err != nil { 307 | return err 308 | } 309 | defer pq.Close() 310 | 311 | for _, t := range []string{"t1", "t2", "t3"} { 312 | if _, err := pq.Exec(fmt.Sprintf("create table %s(id int primary key);", t)); err != nil { 313 | return err 314 | } 315 | for i := 1; i <= 3; i++ { 316 | if _, err := pq.Exec(fmt.Sprintf("insert into %s(id) values(%d);", t, i)); err != nil { 317 | return err 318 | } 319 | } 320 | } 321 | return nil 322 | } 323 | 324 | func Migrate(conn *net.TCPConn, dbname string) error { 325 | if _, err := conn.Write([]byte(fmt.Sprintf("migrate %s", dbname))); err != nil { 326 | return err 327 | } 328 | res := Read(conn) 329 | if strings.HasPrefix(res, "Error") { 330 | return fmt.Errorf("%s", res) 331 | } 332 | return nil 333 | } 334 | -------------------------------------------------------------------------------- /utils/types.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | var typeMapping = map[string]string{ 4 | // Numeric Types 5 | "SMALLINT": "SMALLINT", 6 | "INTEGER": "INTEGER", 7 | "INT": "INTEGER", 8 | "BIGINT": "BIGINT", 9 | "DECIMAL": "DECIMAL", 10 | "NUMERIC": "DECIMAL", 11 | "REAL": "REAL", 12 | "FLOAT4": "REAL", 13 | "DOUBLE PRECISION": "DOUBLE", 14 | "FLOAT8": "DOUBLE", 15 | "SERIAL": "INTEGER AUTOINCREMENT", 16 | "BIGSERIAL": "BIGINT AUTOINCREMENT", 17 | 18 | // Character Types 19 | "VARCHAR": "VARCHAR", 20 | "TEXT": "TEXT", 21 | "CHAR": "VARCHAR", 22 | 23 | // Date & Time Types 24 | "DATE": "DATE", 25 | "TIME": "TIME", 26 | "TIMESTAMP": "TIMESTAMP", 27 | "TIMESTAMPTZ": "TIMESTAMP WITH TIME ZONE", 28 | "TIMESTAMP WITH TIME ZONE": "TIMESTAMP WITH TIME ZONE", 29 | 30 | // Boolean Type 31 | "BOOLEAN": "BOOLEAN", 32 | 33 | // JSON & Array Types 34 | "JSON": "JSON", 35 | "JSONB": "JSON", 36 | "ARRAY": "UNSUPPORTED", // Needs conversion 37 | 38 | // Other Types 39 | "UUID": "VARCHAR(36)", 40 | "BYTEA": "BLOB", 41 | "INET": "VARCHAR", 42 | "CIDR": "VARCHAR", 43 | "ENUM": "VARCHAR", 44 | } 45 | 46 | func DbTypeMap(postgres string) string { 47 | return typeMapping[postgres] 48 | } --------------------------------------------------------------------------------