├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── TODO.md ├── roadmap ├── KV.md └── Vector.md ├── src ├── deprecated │ └── SnapShot.java ├── kv │ ├── GlobSnapShot.java │ ├── InMemoryStorage.java │ ├── KVClient.java │ ├── KVServer.java │ └── PubSub.java ├── tests │ ├── KVTest.java │ └── performance_test_1.py ├── utils │ └── JSONParser.java └── vector │ └── InMemoryVectorStorage.java ├── tangerinekv-cli.sh ├── tangerinekv-compile.sh └── tangerinekv-server.sh /.gitignore: -------------------------------------------------------------------------------- 1 | snapshots/ 2 | 3 | # Java 4 | *.class 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # Directory for compiled binary classes 12 | /bin/ 13 | /build/ 14 | 15 | # IntelliJ 16 | *.iml 17 | *.iws 18 | .idea/ 19 | 20 | # Eclipse 21 | *.project 22 | .classpath 23 | .settings/ 24 | 25 | # Mac OS X 26 | .DS_Store -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11-jdk 2 | 3 | WORKDIR /app 4 | 5 | COPY . . 6 | 7 | RUN chmod +x tangerinekv-server.sh 8 | 9 | CMD ["./tangerinekv-server.sh"] 10 | 11 | EXPOSE 1111 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Anurag Sharma 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🍊 tangerine. 2 | ## in-memory vector database 3 | 4 | tangerine is a in-memory vector database that is designed to have fast read and write speeds. 5 | 6 | tangerine is not limited to vector database, as it has its own key-value store written from scratch in Java. 7 | 8 | ## Key-Value Store 9 | 10 | ### Installation and Running. 11 | 12 | #### Using Docker 13 | 14 | ```bash 15 | docker run -p 1111:1111 anuragdev123/tangerine-kv:latest 16 | ``` 17 | 18 | #### Manual Installation 19 | 20 | Making sh executable 21 | ```bash 22 | chmod +x tangerinekv-server.sh 23 | chmod +x tangerinekv-cli.sh 24 | chmod +x tangerinekv-compile.sh 25 | ``` 26 | 27 | > [!NOTE] 28 | > The .java files are already compiled when running the scripts, but to prevent any issues with the classpath, you should compile them first. 29 | 30 | ```bash 31 | ./tangerinekv-compile.sh 32 | 33 | ``` 34 | 35 | Running the Tangerine-KV Server 36 | 37 | ```bash 38 | ./tangerinekv-server.sh 39 | ``` 40 | 41 | Running the Tangerine-KV Client 42 | ```bash 43 | ./tangerinekv-cli.sh 44 | ``` 45 | Testing 46 | ```bash 47 | $ SET key value 48 | ``` 49 | if server responds with `OK`, then the client & server is working. 50 | 51 | 52 | ### Commands 53 | 54 | #### PING 55 | ```bash 56 | $ PING 57 | ``` 58 | 59 | #### SET 60 | 61 | ```bash 62 | $ SET key value 63 | ``` 64 | 65 | #### GET 66 | 67 | ```bash 68 | $ GET key 69 | ``` 70 | 71 | #### REMOVE 72 | 73 | ```bash 74 | $ REMOVE key 75 | ``` 76 | 77 | #### ALL 78 | 79 | ```bash 80 | $ ALL 81 | ``` 82 | 83 | #### CONTAINS 84 | ```bash 85 | $ CONTAINS key 86 | ``` 87 | 88 | #### CLEAR 89 | 90 | ```bash 91 | $ CLEAR 92 | ``` 93 | 94 | #### EXPIRE 95 | 96 | ```bash 97 | $ EXPIRE key seconds 98 | ``` 99 | 100 | #### TTL 101 | 102 | ```bash 103 | $ TTL key 104 | ``` 105 | 106 | #### HELP 107 | 108 | ```bash 109 | $ HELP 110 | ``` 111 | 112 | #### SUBSCRIBE 113 | ```bash 114 | $ SUBSCRIBE topic 115 | ``` 116 | 117 | #### PUBLISH 118 | ```bash 119 | $ PUBLISH topic message 120 | ``` 121 | 122 | 126 | 127 | 128 | for progress and status check TODO.md 129 | 130 | ## License 131 | 132 | MIT -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | - [X] Basic KV store use hashmap 2 | - [X] Create a KV server to handle the requests. 3 | - [X] Implement KV Client Handing 4 | - [X] Concurrent KV Client Handling 5 | - [X] Add KV Snapshots to disks. 6 | - [X] use glob file to store the snapshots. (replace json with glob) 7 | - [X] Update SET Command to take expire time as a argument. 8 | - [X] Expire Keys 9 | - [X] TTL on Keys 10 | - [X] Change the snapshot to work on after some period of time instead of on every request. 11 | - [X] Add graceful shutdown handling to server. Listen to OS signals. 12 | - Up arrow key in client terminal will load last command. 13 | - Take port no and debug mode as arguments. 14 | - [X] Support Capitalize commands. (made commands capitalized) 15 | - [X] HELP Command 16 | - DBSIZE Command 17 | - Multiple DB Support 18 | - Queue support 19 | - [X] Pub Sub (unsubscribe does not work) 20 | - Check B+ trees video of Arpit Bhayani and implement it. 21 | - Transactions 22 | - [] Vector DB 23 | - Persistence 24 | - Tangerine config file 25 | - Binary Executables 26 | - [X] Dockerise 27 | - [X] Website 28 | - Node SDK 29 | - Cloud Support -------------------------------------------------------------------------------- /roadmap/KV.md: -------------------------------------------------------------------------------- 1 | ### Data Structure 2 | - Hashmap 3 | - ConcurrentHashMap 4 | 5 | ### Commands 6 | - SET - set a key to a value 7 | - GET - get a value for a key 8 | - REMOVE - remove a key 9 | - CONTAINS - check if a key exists 10 | - CLEAR - clear all keys 11 | - SEEALL - see all keys and values 12 | - EXIT - exit the program 13 | 14 | -------------------------------------------------------------------------------- /roadmap/Vector.md: -------------------------------------------------------------------------------- 1 | ### Embeddings -------------------------------------------------------------------------------- /src/deprecated/SnapShot.java: -------------------------------------------------------------------------------- 1 | package deprecated; 2 | 3 | import utils.JSONParser; 4 | 5 | import java.io.FileWriter; 6 | import java.io.IOException; 7 | import java.nio.file.Files; 8 | import java.nio.file.Paths; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.stream.Collectors; 12 | 13 | public class SnapShot { 14 | 15 | // Specify your directory and filename here 16 | private final String directory = "snapshots/"; 17 | private final String filename = "snapshot.json"; 18 | 19 | public void saveToJSON(Map data) { 20 | // Read existing data 21 | Map existingData = readSnapshot(); 22 | 23 | // Merge existing data with new data 24 | existingData.putAll(data); 25 | 26 | // Convert the map to a JSON string 27 | String json = JSONParser.toString(existingData); 28 | 29 | try { 30 | // Create directory if it doesn't exist 31 | Files.createDirectories(Paths.get(directory)); 32 | } catch (IOException e) { 33 | e.printStackTrace(); 34 | } 35 | 36 | // Write JSON string to file 37 | try (FileWriter file = new FileWriter(directory + filename)) { 38 | file.write(json); 39 | System.out.println("Successfully saved data to " + directory + filename); 40 | } catch (IOException e) { 41 | e.printStackTrace(); 42 | } 43 | } 44 | 45 | public Map readSnapshot() { 46 | try { 47 | String content = Files.lines(Paths.get(directory + filename)).collect(Collectors.joining()); 48 | return JSONParser.parseJSON(content); 49 | } catch (IOException e) { 50 | e.printStackTrace(); 51 | } 52 | return new HashMap<>(); 53 | } 54 | 55 | public static void main(String[] args) { 56 | // Create an instance of SnapShot 57 | SnapShot snapShot = new SnapShot(); 58 | 59 | // Create a map with your data 60 | Map data = new HashMap<>(); 61 | data.put("chota", "bheem"); 62 | data.put("key", "value"); 63 | data.put("raj", "shah"); 64 | 65 | // Use the saveToJSON method 66 | snapShot.saveToJSON(data); 67 | System.out.println("read"); 68 | System.out.println(snapShot.readSnapshot()); 69 | } 70 | } -------------------------------------------------------------------------------- /src/kv/GlobSnapShot.java: -------------------------------------------------------------------------------- 1 | package kv; 2 | 3 | import java.io.BufferedWriter; 4 | import java.io.IOException; 5 | import java.nio.file.Files; 6 | import java.nio.file.Path; 7 | import java.nio.file.Paths; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | import java.util.stream.Stream; 11 | import java.nio.file.StandardOpenOption; 12 | 13 | public class GlobSnapShot { 14 | 15 | private final String directory = "snapshots/"; 16 | private final String filename = "snapshot.glob"; 17 | 18 | public void saveToGlob(Map data) { 19 | try { 20 | // Create directory if it doesn't exist 21 | Files.createDirectories(Paths.get(directory)); 22 | 23 | // Read existing data 24 | Map existingData = readSnapshot(); 25 | 26 | // Append new data to file 27 | try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(directory + filename), 28 | StandardOpenOption.APPEND)) { 29 | for (Map.Entry entry : data.entrySet()) { 30 | String key = entry.getKey(); 31 | String value = entry.getValue(); 32 | 33 | // Only append entries that don't already exist in the file 34 | // or have a different value 35 | if (!existingData.containsKey(key) || !existingData.get(key).equals(value)) { 36 | writer.write(key + "=" + value); 37 | writer.newLine(); 38 | } 39 | } 40 | } 41 | 42 | // System.out.println("Successfully saved data to " + directory + filename); 43 | } catch (IOException e) { 44 | e.printStackTrace(); 45 | } 46 | } 47 | 48 | public Map readSnapshot() { 49 | Map data = new HashMap<>(); 50 | Path path = Paths.get(directory + filename); 51 | 52 | try { 53 | // Create file if it doesn't exist 54 | if (!Files.exists(path)) { 55 | Files.createFile(path); 56 | } 57 | 58 | try (Stream lines = Files.lines(path)) { 59 | lines.forEach(line -> { 60 | String[] parts = line.split("=", 2); 61 | if (parts.length >= 2) { 62 | String key = parts[0]; 63 | String value = parts[1]; 64 | data.put(key, value); 65 | } 66 | }); 67 | } 68 | } catch (IOException e) { 69 | e.printStackTrace(); 70 | } 71 | 72 | return data; 73 | } 74 | 75 | public static void main(String[] args) { 76 | // Create an instance of SnapShot 77 | GlobSnapShot globSnapShot = new GlobSnapShot(); 78 | 79 | // Create a map with your data 80 | Map data = new HashMap<>(); 81 | // data.put("openai", "gpt3"); 82 | data.put("anthropic", "claude"); 83 | 84 | // Use the saveToGlob method 85 | globSnapShot.saveToGlob(data); 86 | 87 | // Use the readSnapshot method 88 | System.out.println(globSnapShot.readSnapshot()); 89 | } 90 | } -------------------------------------------------------------------------------- /src/kv/InMemoryStorage.java: -------------------------------------------------------------------------------- 1 | package kv; 2 | 3 | import java.util.concurrent.ConcurrentHashMap; 4 | import java.util.concurrent.Executors; 5 | import java.util.concurrent.ScheduledExecutorService; 6 | import java.util.concurrent.TimeUnit; 7 | 8 | public class InMemoryStorage { 9 | // using ConcurrentHashMap to store the data in memory. 10 | private ConcurrentHashMap storage = new ConcurrentHashMap<>(); 11 | // An hashmap to store the expiration times of the keys. 12 | private ConcurrentHashMap expirationTimes = new ConcurrentHashMap<>(); // this TTL is not saved as snapshot 13 | 14 | // Allocation one thread for the executor service. 15 | private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); 16 | 17 | public void set(String key, String value) { 18 | storage.put(key, value); 19 | } 20 | 21 | // Method overloading for ttl. 22 | 23 | public void set(String key, String value, int ttl) { 24 | long expirationTime = System.currentTimeMillis() + ttl * 1000; 25 | storage.put(key, value); 26 | expirationTimes.put(key, expirationTime); 27 | 28 | executorService.schedule(() -> { 29 | storage.remove(key); 30 | expirationTimes.remove(key); 31 | }, ttl, TimeUnit.SECONDS); 32 | } 33 | 34 | public String get(String key) { 35 | return storage.get(key); 36 | } 37 | 38 | public String PING(){ 39 | return "PONG"; 40 | } 41 | 42 | public void remove(String key) { 43 | storage.remove(key); 44 | } 45 | 46 | public boolean containsKey(String key) { 47 | return storage.containsKey(key); 48 | } 49 | 50 | public void clear() { 51 | storage.clear(); 52 | } 53 | 54 | public String seeAll() { 55 | return storage.toString(); 56 | } 57 | 58 | public String expire(String key, int ttl) { 59 | // tts is int as seconds 60 | long expirationTime = System.currentTimeMillis() + ttl * 1000; 61 | 62 | expirationTimes.put(key, expirationTime); 63 | 64 | executorService.schedule(() -> { 65 | storage.remove(key); 66 | }, ttl, TimeUnit.SECONDS); 67 | return "OK"; 68 | } 69 | 70 | public String TTL(String key) { 71 | if (expirationTimes.containsKey(key)) { 72 | // get expiration time of the key - current time 73 | long remainingTime = expirationTimes.get(key) - System.currentTimeMillis(); 74 | return remainingTime > 0 ? remainingTime / 1000 + " seconds" : "Expired"; 75 | } else { 76 | return "No TTL set"; 77 | } 78 | } 79 | 80 | public String HELP() { 81 | String resetColor = "\u001B[0m"; 82 | String commandColor = "\u001B[34m"; 83 | String descriptionColor = "\u001B[37m"; 84 | 85 | StringBuilder helpMessage = new StringBuilder(); 86 | 87 | helpMessage.append(commandColor + "SET key value [ttl]" + resetColor + descriptionColor + " : Store the value with the specified key. Optional ttl (time to live) in seconds can be provided, after which the key-value pair will be automatically removed.\n" + resetColor); 88 | helpMessage.append(commandColor + "GET key" + resetColor + descriptionColor + " : Retrieve the value of the specified key.\n" + resetColor); 89 | helpMessage.append(commandColor + "REMOVE key" + resetColor + descriptionColor + " : Remove the key-value pair with the specified key.\n" + resetColor); 90 | helpMessage.append(commandColor + "CONTAINS key" + resetColor + descriptionColor + " : Check if the storage contains a value for the specified key.\n" + resetColor); 91 | helpMessage.append(commandColor + "CLEAR" + resetColor + descriptionColor + " : Remove all key-value pairs from the storage.\n" + resetColor); 92 | helpMessage.append(commandColor + "ALL" + resetColor + descriptionColor + " : See all key-value pairs in the storage.\n" + resetColor); 93 | helpMessage.append(commandColor + "EXPIRE key ttl" + resetColor + descriptionColor + " : Set a ttl (time to live) in seconds for the specified key, after which the key-value pair will be automatically removed.\n" + resetColor); 94 | helpMessage.append(commandColor + "TTL key" + resetColor + descriptionColor + " : Get the remaining time to live (in seconds) for the specified key.\n" + resetColor); 95 | 96 | return helpMessage.toString(); 97 | } 98 | } -------------------------------------------------------------------------------- /src/kv/KVClient.java: -------------------------------------------------------------------------------- 1 | package kv; 2 | 3 | import java.io.*; 4 | import java.net.*; 5 | import java.util.concurrent.locks.Condition; 6 | import java.util.concurrent.locks.Lock; 7 | import java.util.concurrent.locks.ReentrantLock; 8 | 9 | public class KVClient { 10 | private Socket socket; 11 | private PrintWriter writer; 12 | private BufferedReader reader; 13 | // Synchronizing both threads so client main thread can wait for response from server. 14 | private Lock lock = new ReentrantLock(); 15 | private Condition receivedResponse = lock.newCondition(); 16 | 17 | public KVClient(String host, int port) throws IOException { 18 | this.socket = new Socket(host, port); 19 | this.writer = new PrintWriter(socket.getOutputStream(), true); 20 | this.reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); 21 | 22 | // thread listens for events from server. Is asynchronous in nature. 23 | new Thread(() -> { 24 | String response; 25 | try { 26 | while ((response = reader.readLine()) != null) { 27 | System.out.println(response); 28 | lock.lock(); 29 | try { 30 | receivedResponse.signalAll(); 31 | } finally { 32 | lock.unlock(); 33 | } 34 | } 35 | } catch (IOException e) { 36 | e.printStackTrace(); 37 | } 38 | }).start(); 39 | } 40 | 41 | public void sendCommand(String command) throws InterruptedException { 42 | // main thread is locked 43 | lock.lock(); 44 | try { 45 | writer.println(command); 46 | // awaiting response from server 47 | receivedResponse.await(); 48 | } finally { 49 | // main thread is unlocked 50 | lock.unlock(); 51 | } 52 | } 53 | 54 | public void close() throws IOException { 55 | socket.close(); 56 | } 57 | 58 | public static void main(String[] args) throws IOException, InterruptedException { 59 | String host = "localhost"; 60 | int port = 1111; 61 | 62 | KVClient client = new KVClient(host, port); 63 | 64 | BufferedReader consoleReader = new BufferedReader(new InputStreamReader(System.in)); 65 | String input; 66 | while (true) { 67 | System.out.print("tangerine-cli> "); 68 | input = consoleReader.readLine(); 69 | if (input.equalsIgnoreCase("quit")) { 70 | client.close(); 71 | break; 72 | } 73 | client.sendCommand(input); 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /src/kv/KVServer.java: -------------------------------------------------------------------------------- 1 | package kv; 2 | 3 | import java.io.*; 4 | import java.net.*; 5 | import java.util.Map; 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | import java.util.concurrent.ScheduledExecutorService; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | import utils.*; 12 | 13 | public class KVServer { 14 | private InMemoryStorage storage; 15 | private ServerSocket serverSocket; 16 | private ExecutorService executor; 17 | private File snapshotFile; 18 | private ScheduledExecutorService snapshotExecutor = Executors.newSingleThreadScheduledExecutor(); 19 | private PubSub pubSub = new PubSub(); // Instance of our PubSub class 20 | 21 | public KVServer(InMemoryStorage storage, int port) throws IOException { 22 | this.storage = storage; 23 | this.serverSocket = new ServerSocket(port); 24 | this.executor = Executors.newFixedThreadPool(3); // fixed no of threads (can be increased or decreased according 25 | // to need) 26 | 27 | GlobSnapShot globSnapShot = new GlobSnapShot(); 28 | 29 | System.out.println("KV Server started on port " + port); 30 | snapshotFile = new File("snapshots"); 31 | snapshotFile.mkdirs(); 32 | 33 | // Schedule the snapshot task to run every 120 seconds 34 | snapshotExecutor.scheduleAtFixedRate(() -> globSnapShot.saveToGlob(JSONParser.parseJSON(storage.seeAll())), 0, 35 | 20, TimeUnit.SECONDS); 36 | // Initialize GlobSnapShot and load the existing data 37 | Map existingData = globSnapShot.readSnapshot(); 38 | for (Map.Entry entry : existingData.entrySet()) { 39 | storage.set(entry.getKey(), entry.getValue()); 40 | } 41 | 42 | // Add graceful shutdown to handle the server shutdown and take snapshot of the 43 | // data. 44 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 45 | System.out.println("Shutting down server..."); 46 | try { 47 | globSnapShot.saveToGlob(JSONParser.parseJSON(storage.seeAll())); 48 | serverSocket.close(); // Close the socket connection 49 | executor.shutdownNow(); // Attempts to stop all actively executing tasks 50 | snapshotExecutor.shutdownNow(); // Attempts to stop all actively executing tasks 51 | } catch (IOException e) { 52 | e.printStackTrace(); 53 | } 54 | })); 55 | 56 | } 57 | 58 | public void run() throws IOException { 59 | while (true) { 60 | // accept a connection from the client and handle the request in a separate 61 | // thread using handleRequest method. 62 | Socket socket = serverSocket.accept(); 63 | executor.submit(() -> { 64 | try { 65 | handleRequest(socket); 66 | } catch (IOException e) { 67 | e.printStackTrace(); 68 | } 69 | }); 70 | // check snapshot files and read the files and set or sync the current in memory 71 | // storage to the snapshot data 72 | File[] snapshotFiles = snapshotFile.listFiles(); 73 | if (snapshotFiles != null) { 74 | for (File file : snapshotFiles) { 75 | try (BufferedReader reader = new BufferedReader(new FileReader(file))) { 76 | String line; 77 | while ((line = reader.readLine()) != null) { 78 | // snapshot is '=' kv pair because that is how hashmap in java stores data 79 | String[] parts = line.split("="); 80 | if (parts.length >= 2) { 81 | storage.set(parts[0], parts[1]); 82 | } else { 83 | System.out.println("Invalid line in snapshot file: " + line); 84 | } 85 | } 86 | } catch (IOException e) { 87 | e.printStackTrace(); 88 | } 89 | } 90 | } 91 | } 92 | } 93 | 94 | private void handleRequest(Socket socket) throws IOException { 95 | // create a new GlobSnapShot object to save the snapshot. 96 | GlobSnapShot globSnapShot = new GlobSnapShot(); 97 | // create a new JSONParser object to parse string to object. This is essential 98 | // because 99 | // getAll returns a string of object. 100 | 101 | // get input stream and output stream from the socket and create a buffered 102 | // reader and a writer 103 | try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); 104 | PrintWriter writer = new PrintWriter(socket.getOutputStream(), true)) { 105 | String input; 106 | while ((input = reader.readLine()) != null) { 107 | String[] parts = input.split(" "); 108 | switch (parts[0]) { 109 | case "PING": 110 | writer.println(storage.PING()); 111 | writer.println(); 112 | break; 113 | case "SET": 114 | // method overloading if ttl (time to live) is provided 115 | // ttl is in seconds. 116 | if (parts.length == 4) { 117 | int ttl = Integer.parseInt(parts[3]); 118 | storage.set(parts[1], parts[2], ttl); 119 | System.out.println("TTL set to " + ttl); 120 | } else { 121 | storage.set(parts[1], parts[2]); 122 | } 123 | writer.println("OK"); 124 | writer.println(); 125 | break; 126 | case "GET": 127 | String value = storage.get(parts[1]); 128 | if (value != null) { 129 | writer.println(value); 130 | writer.println(); 131 | } else { 132 | writer.println("NOT_FOUND"); 133 | writer.println(); 134 | } 135 | break; 136 | case "REMOVE": 137 | storage.remove(parts[1]); 138 | globSnapShot.saveToGlob(JSONParser.parseJSON(storage.seeAll())); 139 | writer.println("OK"); 140 | writer.println(); 141 | break; 142 | case "CONTAINS": 143 | boolean contains = storage.containsKey(parts[1]); 144 | writer.println(contains ? "TRUE" : "FALSE"); 145 | writer.println(); 146 | break; 147 | case "CLEAR": 148 | storage.clear(); 149 | writer.println("OK"); 150 | writer.println(); 151 | break; 152 | case "ALL": 153 | writer.println(storage.seeAll()); 154 | writer.println(); 155 | break; 156 | case "EXPIRE": 157 | String expire = storage.expire(parts[1], Integer.parseInt(parts[2])); 158 | writer.println(expire); 159 | writer.println(); 160 | break; 161 | case "TTL": 162 | String ttl = storage.TTL(parts[1]); 163 | writer.println(ttl); 164 | writer.println(); 165 | break; 166 | case "SUBSCRIBE": 167 | pubSub.subscribe(parts[1], message -> writer.println("Received: " + message)); 168 | writer.println("OK"); 169 | writer.println(); 170 | break; 171 | case "UNSUBSCRIBE": 172 | pubSub.unsubscribe(parts[1], message -> writer.println("Received: " + message)); 173 | writer.println("OK"); 174 | writer.println(); 175 | break; 176 | case "PUBLISH": 177 | pubSub.publish(parts[1], parts[2]); 178 | writer.println("OK"); 179 | writer.println(); 180 | break; 181 | case "HELP": 182 | writer.println(storage.HELP()); 183 | writer.println(); 184 | break; 185 | default: 186 | writer.println("Unknown command"); 187 | writer.println(); 188 | break; 189 | } 190 | } 191 | } 192 | } 193 | 194 | public static void main(String[] args) throws IOException { 195 | InMemoryStorage storage = new InMemoryStorage(); 196 | int port = 1111; 197 | KVServer server = new KVServer(storage, port); 198 | server.run(); 199 | } 200 | } -------------------------------------------------------------------------------- /src/kv/PubSub.java: -------------------------------------------------------------------------------- 1 | package kv; 2 | 3 | import java.util.*; 4 | 5 | public class PubSub { 6 | private Map> subscribers = new HashMap<>(); 7 | 8 | public void subscribe(String topic, Subscriber subscriber) { 9 | if (!subscribers.containsKey(topic)) { 10 | subscribers.put(topic, new ArrayList<>()); 11 | } 12 | subscribers.get(topic).add(subscriber); 13 | } 14 | 15 | public void unsubscribe(String topic, Subscriber subscriber) { 16 | if (subscribers.containsKey(topic)) { 17 | subscribers.get(topic).remove(subscriber); 18 | } 19 | } 20 | 21 | public void publish(String topic, String message) { 22 | if (subscribers.containsKey(topic)) { 23 | for (Subscriber subscriber : subscribers.get(topic)) { 24 | subscriber.receive(message); 25 | } 26 | } 27 | } 28 | 29 | public interface Subscriber { 30 | void receive(String message); 31 | } 32 | 33 | public static void main(String[] args) { 34 | PubSub pubSub = new PubSub(); 35 | 36 | PubSub.Subscriber subscriber1 = System.out::println; 37 | PubSub.Subscriber subscriber2 = System.out::println; 38 | PubSub.Subscriber subscriber3 = System.out::println; 39 | 40 | pubSub.subscribe("topic1", subscriber1); 41 | pubSub.subscribe("topic1", subscriber2); 42 | pubSub.subscribe("topic2", subscriber3); 43 | 44 | pubSub.publish("topic1", "Hello, Topic 1!"); 45 | pubSub.publish("topic2", "Hello, Topic 2!"); 46 | 47 | pubSub.unsubscribe("topic1", subscriber2); 48 | 49 | pubSub.publish("topic1", "Goodbye, Topic 1!"); 50 | } 51 | } -------------------------------------------------------------------------------- /src/tests/KVTest.java: -------------------------------------------------------------------------------- 1 | package tests; 2 | 3 | import kv.InMemoryStorage; 4 | 5 | public class KVTest { 6 | public static void main(String[] args) { 7 | test(); 8 | } 9 | 10 | private static void test() { 11 | InMemoryStorage storage = new InMemoryStorage(); 12 | storage.set("key1", "value1"); 13 | storage.set("key1", "value1"); 14 | storage.set("key1", "value1"); 15 | System.out.println(storage.seeAll()); 16 | System.out.println(storage.get("key1")); 17 | System.out.println(storage.get("key2")); 18 | System.out.println(storage.get("key3")); 19 | storage.remove("key1"); 20 | System.out.println(storage.seeAll()); 21 | storage.clear(); 22 | System.out.println(storage.seeAll()); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/tests/performance_test_1.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import socket 3 | import threading 4 | import time 5 | 6 | class Counter(object): 7 | def __init__(self): 8 | self.val = 0 9 | self._lock = threading.Lock() 10 | 11 | def increment(self): 12 | with self._lock: 13 | self.val += 1 14 | 15 | def value(self): 16 | with self._lock: 17 | return self.val 18 | 19 | command_counter_1 = Counter() 20 | command_counter_2 = Counter() 21 | command_counter_3 = Counter() 22 | 23 | CLIENT_1: redis.Redis = redis.Redis(host="localhost", port=6379, db=0, decode_responses=True) 24 | CLIENT_2: redis.Redis = redis.Redis(host="localhost", port=6378, db=0, decode_responses=True) 25 | 26 | def send_command_to_kvstore(command: str, port: int): 27 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 28 | s.connect(("localhost", port)) 29 | s.sendall(command.encode()) 30 | response = s.recv(1024) 31 | return response.decode() 32 | 33 | def test_set_redis(client: redis.Redis, counter: Counter): 34 | key = "mykey" + str(counter.value()) 35 | value = "Hello, World!" 36 | client.set(key, value) 37 | counter.increment() 38 | 39 | def test_set_kvstore(counter: Counter, port: int): 40 | key = "mykey" + str(counter.value()) 41 | value = "Hello, World!" 42 | send_command_to_kvstore(f"SET {key} {value}\n", port) 43 | counter.increment() 44 | 45 | def test_speed(): 46 | print("Testing speed...") 47 | 48 | start = time.time() 49 | test_set_redis(CLIENT_1, command_counter_1) 50 | test_set_kvstore(command_counter_2, 1111) 51 | test_set_redis(CLIENT_2, command_counter_3) 52 | while time.time() - start < 10: 53 | threading.Thread(target=test_set_redis, args=(CLIENT_1, command_counter_1)).start() 54 | threading.Thread(target=test_set_kvstore, args=(command_counter_2, 1111)).start() 55 | threading.Thread(target=test_set_redis, args=(CLIENT_2, command_counter_3)).start() 56 | 57 | print(f"Number of SET commands sent by Redis client (6378): {command_counter_3.value()}") 58 | print(f"Number of SET commands sent by Radish client (6379): {command_counter_1.value()}") 59 | print(f"Number of SET commands sent by Tangerine-kv client (1111): {command_counter_2.value()}") 60 | 61 | test_speed() -------------------------------------------------------------------------------- /src/utils/JSONParser.java: -------------------------------------------------------------------------------- 1 | package utils; 2 | 3 | import java.util.*; 4 | 5 | public class JSONParser { 6 | public static Map parseJSON(String jsonString) { 7 | Map jsonMap = new HashMap<>(); 8 | jsonString = jsonString.trim(); 9 | if (jsonString.startsWith("{") && jsonString.endsWith("}")) { 10 | jsonString = jsonString.substring(1, jsonString.length() - 1); 11 | String[] keyValuePairs = jsonString.split(","); 12 | for (String pair : keyValuePairs) { 13 | String[] entry = pair.split("[=:]"); 14 | if (entry.length == 2) { 15 | String key = entry[0].trim(); 16 | String value = entry[1].trim(); 17 | if (key.startsWith("\"") && key.endsWith("\"")) { 18 | key = key.substring(1, key.length() - 1); 19 | } 20 | if (value.startsWith("\"") && value.endsWith("\"")) { 21 | value = value.substring(1, value.length() - 1); 22 | } 23 | jsonMap.put(key, value); 24 | } 25 | } 26 | } 27 | return jsonMap; 28 | } 29 | 30 | public static String convertToColonSyntax(String jsonString) { 31 | return jsonString.replace("=", ":"); 32 | } 33 | 34 | public static String convertToEqualSyntax(String jsonString) { 35 | return jsonString.replace(":", "="); 36 | } 37 | 38 | public static String toString(Map map) { 39 | StringBuilder sb = new StringBuilder("{"); 40 | for (String key : map.keySet()) { 41 | if (sb.length() > 1) { 42 | sb.append(", "); 43 | } 44 | sb.append(key).append(":").append(map.get(key)); 45 | } 46 | sb.append("}"); 47 | return sb.toString(); 48 | } 49 | 50 | public static void main(String[] args) { 51 | String jsonString = "{chota:bheem, key=value, raj=shah}"; 52 | Map parsedJSON = parseJSON(jsonString); 53 | System.out.println("Parsed JSON: " + toString(parsedJSON)); 54 | for (String key : parsedJSON.keySet()) { 55 | System.out.println("Key: " + key + ", Value: " + parsedJSON.get(key)); 56 | } 57 | System.out.println(toString(parsedJSON)); 58 | } 59 | } -------------------------------------------------------------------------------- /src/vector/InMemoryVectorStorage.java: -------------------------------------------------------------------------------- 1 | package vector; 2 | 3 | import java.util.concurrent.ConcurrentHashMap; 4 | import java.util.concurrent.Executors; 5 | import java.util.concurrent.ScheduledExecutorService; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.Vector; 8 | 9 | public class InMemoryVectorStorage { 10 | // Using ConcurrentHashMap to store the vectors in memory. 11 | private ConcurrentHashMap> storage = new ConcurrentHashMap<>(); 12 | // An hashmap to store the expiration times of the keys. 13 | private ConcurrentHashMap expirationTimes = new ConcurrentHashMap<>(); 14 | 15 | // Allocation one thread for the executor service. 16 | private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); 17 | 18 | public void set(String key, Vector value) { 19 | storage.put(key, value); 20 | } 21 | 22 | // Method overloading for ttl. 23 | public void set(String key, Vector value, int ttl) { 24 | long expirationTime = System.currentTimeMillis() + ttl * 1000; 25 | storage.put(key, value); 26 | expirationTimes.put(key, expirationTime); 27 | 28 | executorService.schedule(() -> { 29 | storage.remove(key); 30 | expirationTimes.remove(key); 31 | }, ttl, TimeUnit.SECONDS); 32 | } 33 | 34 | public Vector get(String key) { 35 | return storage.get(key); 36 | } 37 | 38 | public String PING() { 39 | return "PONG"; 40 | } 41 | 42 | public void remove(String key) { 43 | storage.remove(key); 44 | } 45 | 46 | public boolean containsKey(String key) { 47 | return storage.containsKey(key); 48 | } 49 | 50 | public void clear() { 51 | storage.clear(); 52 | } 53 | 54 | public String seeAll() { 55 | return storage.toString(); 56 | } 57 | 58 | public String expire(String key, int ttl) { 59 | long expirationTime = System.currentTimeMillis() + ttl * 1000; 60 | 61 | expirationTimes.put(key, expirationTime); 62 | 63 | executorService.schedule(() -> { 64 | storage.remove(key); 65 | }, ttl, TimeUnit.SECONDS); 66 | return "OK"; 67 | } 68 | 69 | public String TTL(String key) { 70 | if (expirationTimes.containsKey(key)) { 71 | long remainingTime = expirationTimes.get(key) - System.currentTimeMillis(); 72 | return remainingTime > 0 ? remainingTime / 1000 + " seconds" : "Expired"; 73 | } else { 74 | return "No TTL set"; 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /tangerinekv-cli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Navigate to the src directory 4 | cd src 5 | 6 | # Compile all the .java files in the kv and utils packages 7 | javac kv/*.java utils/*.java 8 | 9 | # Run the KVServer program 10 | java kv.KVClient 11 | 12 | # Navigate back to the original directory 13 | cd .. 14 | -------------------------------------------------------------------------------- /tangerinekv-compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Navigate to the src directory 3 | cd src 4 | 5 | # Compile all the .java files in the kv and utils packages 6 | javac kv/*.java utils/*.java 7 | 8 | # Navigate back to the original directory 9 | cd .. 10 | -------------------------------------------------------------------------------- /tangerinekv-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Navigate to the src directory 4 | cd src 5 | 6 | # Compile all the .java files in the kv and utils packages 7 | javac kv/*.java utils/*.java 8 | 9 | # Run the KVServer program 10 | java kv.KVServer 11 | 12 | # Navigate back to the original directory 13 | cd .. 14 | --------------------------------------------------------------------------------