├── Dockerfile ├── README.md ├── cmd └── server │ └── main.go ├── docker-compose.yml ├── go.mod ├── go.sum ├── imp.py ├── internal ├── api │ └── handler.go ├── config │ └── config.go ├── graphdb │ ├── node.go │ └── partition.go └── ncs │ ├── cache.go │ └── setcover.go └── pkg └── models └── graph.go /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21-alpine 2 | 3 | WORKDIR /app 4 | 5 | RUN apk add --no-cache gcc musl-dev 6 | COPY go.mod go.sum ./ 7 | RUN go mod download 8 | COPY . . 9 | RUN go build -o graph-server cmd/server/main.go 10 | CMD ["./graph-server"] 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BiDirect - Social Graph Service 2 | 3 | BiDirect is a minimalist implementation of a distributed social graph service, designed to demonstrate the core concepts of building scalable social networking systems like LinkedIn or Facebook's social graph infrastructure. 4 | 5 | ## Overview 6 | 7 | This implementation is a micro-instance of how large-scale social networks manage connection data and calculate relationship distances. While production systems handle billions of connections across thousands of nodes, this implementation uses a smaller scale to demonstrate the key architectural concepts. 8 | 9 | ### Key Features 10 | - Distributed graph storage using Redis 11 | - Partitioned data architecture 12 | - Connection degree calculation (1st, 2nd, 3rd degree connections) 13 | - Shared connection discovery 14 | - Network distance computation 15 | 16 | ## Architecture 17 | 18 | ### Scaled-Down Components 19 | 20 | 1. **Partitioning Strategy** 21 | - Current: Simple modulo-based partitioning across 3 Redis nodes 22 | - Production: Would use consistent hashing or range-based sharding across thousands of nodes 23 | 24 | 2. **Caching Layer** 25 | - Current: Single Redis instance for second-degree connections 26 | - Production: Multi-tiered caching with in-memory, near-memory, and disk-based caches 27 | 28 | 3. **Node Management** 29 | - Current: Static node configuration 30 | - Production: Dynamic node discovery and automated rebalancing 31 | 32 | ### API Endpoints 33 | 34 | ``` 35 | GET /api/connections/{memberID} # Get member's direct connections 36 | GET /api/shared-connections/{id1}/{id2} # Find shared connections 37 | POST /api/distances # Calculate network distances 38 | ``` 39 | 40 | ## Technical Design Decisions 41 | 42 | ### Data Distribution 43 | - Uses partition-based distribution to demonstrate how social graphs can be split across multiple nodes 44 | - Each node manages multiple partitions to show how load can be distributed 45 | - Simplified partitioning function for demonstration purposes 46 | 47 | ### Connection Traversal 48 | - Implements efficient 2nd and 3rd-degree connection discovery 49 | - Uses Redis sorted sets for quick connection lookups 50 | - Demonstrates caching of frequently accessed paths 51 | 52 | ### Set Cover Algorithm 53 | - Implements a greedy approach for finding minimum node sets 54 | - Shows how to optimize multi-node queries in a distributed system 55 | 56 | ## Getting Started 57 | 58 | ### Prerequisites 59 | - Go 1.21+ 60 | - Docker and Docker Compose 61 | 62 | ### Running the Service 63 | 64 | 1. Start the infrastructure: 65 | ```bash 66 | docker-compose up -d 67 | ``` 68 | 69 | 2. The service will be available at `http://localhost:8080` 70 | 71 | 3. Load sample data: 72 | ```bash 73 | python imp.py 74 | ``` 75 | 76 | ## Production Considerations 77 | 78 | This implementation is intentionally simplified. In a production environment, you would need to consider: 79 | 80 | 1. **Scalability** 81 | - Current: 3 Redis nodes, 10 partitions per node 82 | - Production: Thousands of nodes, dynamic partition allocation 83 | 84 | 2. **Reliability** 85 | - Current: Basic error handling 86 | - Production: Circuit breakers, fallbacks, redundancy 87 | 88 | 3. **Performance** 89 | - Current: Simple caching strategy 90 | - Production: Multi-level caching, pre-computation of common paths 91 | 92 | 4. **Monitoring** 93 | - Current: Basic logging 94 | - Production: Comprehensive metrics, tracing, alerting 95 | 96 | 5. **Security** 97 | - Current: No authentication 98 | - Production: OAuth, rate limiting, encryption 99 | 100 | ## Design Choices 101 | 102 | ### Why Redis? 103 | - Demonstrates in-memory graph storage principles 104 | - Sorted sets provide efficient connection lookups 105 | - Easy to understand and set up for demonstration 106 | 107 | ### Why Partition-Based Distribution? 108 | - Shows basic concepts of data sharding 109 | - Demonstrates how to handle cross-partition queries 110 | - Simplified version of production-grade distribution strategies 111 | 112 | ## Contributing 113 | 114 | This is an educational project designed to demonstrate distributed systems concepts. Contributions that help clarify these concepts or add new educational examples are welcome. 115 | 116 | ## License 117 | 118 | MIT License 119 | 120 | ## Acknowledgments 121 | 122 | This implementation draws inspiration from real-world social graph systems while maintaining a focus on educational value and clarity over production-grade features. -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | // "os" 9 | // "os/signal" 10 | // "strings" 11 | // "syscall" 12 | "time" 13 | 14 | "github.com/mrinalxdev/bidirect/internal/api" 15 | "github.com/mrinalxdev/bidirect/internal/config" 16 | "github.com/mrinalxdev/bidirect/internal/graphdb" 17 | "github.com/mrinalxdev/bidirect/internal/ncs" 18 | ) 19 | 20 | func main(){ 21 | redisConfig := config.LoadRedisConfig() 22 | partitionsPerNode := 10 23 | totalPartitions := len(redisConfig.Addresses) * partitionsPerNode 24 | 25 | nodes := make([]*graphdb.Node, len(redisConfig.Addresses)) 26 | logPartitionsInfo := func(nodeID string, partitionID int, addr string){ 27 | log.Printf("Initializing - Noe : %s, Partitions: %d, Redis: %s", nodeID, partitionID, addr) 28 | } 29 | 30 | for i, addr := range redisConfig.Addresses{ 31 | startPartition := i * partitionsPerNode 32 | endPartitions := startPartition + partitionsPerNode 33 | 34 | nodeConfig := graphdb.NodeConfig{ 35 | ID : fmt.Sprintf("node-%d", i), 36 | PartitionIDs: make([]int, 0, partitionsPerNode), 37 | RedisAddr: addr, 38 | ReplicaFactor: len(redisConfig.Addresses), 39 | } 40 | 41 | for pid := startPartition; pid < endPartitions; pid++{ 42 | nodeConfig.PartitionIDs = append(nodeConfig.PartitionIDs, pid) 43 | logPartitionsInfo(nodeConfig.ID, pid, addr) 44 | } 45 | 46 | node := graphdb.NewNode(nodeConfig) 47 | if node == nil { 48 | log.Fatalf("Failed to create node %d", i) 49 | } 50 | 51 | for _, pid := range nodeConfig.PartitionIDs { 52 | if partition, exists := node.Partition[pid]; !exists || partition == nil { 53 | log.Fatalf("Faile to initialize partition %d in node %s", pid, node.ID) 54 | } 55 | } 56 | 57 | nodes[i] = node 58 | } 59 | 60 | log.Printf("Verifuing partition distribution ...") 61 | for i := 0; i < totalPartitions; i++ { 62 | found := false 63 | for _, node := range nodes { 64 | if _, exists := node.Partition[i]; exists { 65 | found = true 66 | log.Printf("Partition %d found in node %s", i, node.ID) 67 | break 68 | } 69 | } 70 | if !found { 71 | log.Printf("Warning : Partition %d is not assigned to any node", i) 72 | } 73 | } 74 | 75 | networkCache := ncs.NewNetworkCache(redisConfig.Addresses[0], 24*time.Hour) 76 | handler := api.NewHandler(nodes, networkCache) 77 | 78 | log.Printf("Starting server with %d nodes and %d partitions per node", 79 | len(nodes), partitionsPerNode) 80 | log.Fatal(http.ListenAndServe(":8080", handler)) 81 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | redis-node-1: 5 | image: redis:7.0 6 | container_name: graph-redis-1 7 | command: redis-server --port 6379 8 | ports: 9 | - "6379:6379" 10 | volumes: 11 | - redis-data-1:/data 12 | networks: 13 | - graph-network 14 | 15 | redis-node-2: 16 | image: redis:7.0 17 | container_name: graph-redis-2 18 | command: redis-server --port 6379 19 | ports: 20 | - "6380:6379" 21 | volumes: 22 | - redis-data-2:/data 23 | networks: 24 | - graph-network 25 | 26 | redis-node-3: 27 | image: redis:7.0 28 | container_name: graph-redis-3 29 | command: redis-server --port 6379 30 | ports: 31 | - "6381:6379" 32 | volumes: 33 | - redis-data-3:/data 34 | networks: 35 | - graph-network 36 | 37 | graph-service: 38 | build: 39 | context: . 40 | dockerfile: Dockerfile 41 | container_name: graph-service 42 | ports: 43 | - "8080:8080" 44 | depends_on: 45 | - redis-node-1 46 | - redis-node-2 47 | - redis-node-3 48 | networks: 49 | - graph-network 50 | environment: 51 | - REDIS_NODES=graph-redis-1:6379,graph-redis-2:6379,graph-redis-3:6379 52 | 53 | networks: 54 | graph-network: 55 | driver: bridge 56 | 57 | volumes: 58 | redis-data-1: 59 | redis-data-2: 60 | redis-data-3: -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mrinalxdev/bidirect 2 | 3 | go 1.21.5 4 | 5 | require ( 6 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 7 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 8 | github.com/go-redis/redis/v8 v8.11.5 // indirect 9 | github.com/gorilla/mux v1.8.1 // indirect 10 | github.com/redis/go-redis/v9 v9.7.0 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 2 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 4 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 5 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 6 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 7 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= 8 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= 9 | github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= 10 | github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= 11 | -------------------------------------------------------------------------------- /imp.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | # Sample connections data 5 | connections = [ 6 | {"sourceId": 1, "destinationIds": [2, 3, 4]}, 7 | {"sourceId": 2, "destinationIds": [1, 3, 5]}, 8 | {"sourceId": 3, "destinationIds": [1, 2, 4, 6]}, 9 | {"sourceId": 4, "destinationIds": [1, 3, 7]}, 10 | {"sourceId": 5, "destinationIds": [2, 8]}, 11 | {"sourceId": 6, "destinationIds": [3, 9]}, 12 | {"sourceId": 7, "destinationIds": [4, 10]} 13 | ] 14 | 15 | for conn in connections: 16 | response = requests.post( 17 | "http://localhost:8080/api/connections", 18 | json=conn 19 | ) 20 | print(f"Stored connections for member {conn['sourceId']}") -------------------------------------------------------------------------------- /internal/api/handler.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/gorilla/mux" 13 | "github.com/mrinalxdev/bidirect/internal/graphdb" 14 | "github.com/mrinalxdev/bidirect/internal/ncs" 15 | "github.com/mrinalxdev/bidirect/pkg/models" 16 | ) 17 | 18 | type Handler struct { 19 | nodes []*graphdb.Node 20 | networkCache *ncs.NetworkCache 21 | router *mux.Router 22 | } 23 | 24 | func NewHandler(nodes []*graphdb.Node, networkCache *ncs.NetworkCache) *Handler { 25 | h := &Handler{ 26 | nodes: nodes, 27 | networkCache: networkCache, 28 | } 29 | 30 | router := mux.NewRouter() 31 | router.HandleFunc("/api/connections/{memberID}", h.GetConnections).Methods("GET") 32 | router.HandleFunc("/api/shared-connections/{memberID1}/{memberID2}", h.GetSharedConnections).Methods("GET") 33 | router.HandleFunc("/api/distances", h.GetDistances).Methods("POST") 34 | 35 | h.router = router 36 | return h 37 | } 38 | 39 | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 40 | h.router.ServeHTTP(w, r) 41 | } 42 | 43 | func parseMemberID(idStr string) (models.MemberID, error) { 44 | id, err := strconv.ParseInt(idStr, 10, 64) 45 | if err != nil { 46 | return 0, fmt.Errorf("invalid member ID format: %v", err) 47 | } 48 | return models.MemberID(id), nil 49 | } 50 | 51 | 52 | func (h *Handler) GetConnections(w http.ResponseWriter, r *http.Request) { 53 | ctx := r.Context() 54 | vars := mux.Vars(r) 55 | memberIDStr, ok := vars["memberID"] 56 | if !ok { 57 | http.Error(w, "memberID is required", http.StatusBadRequest) 58 | return 59 | } 60 | 61 | memberID, err := parseMemberID(memberIDStr) 62 | if err != nil { 63 | http.Error(w, fmt.Sprintf("Invalid member ID: %v", err), http.StatusBadRequest) 64 | return 65 | } 66 | 67 | // validate nodes array 68 | if len(h.nodes) == 0 { 69 | http.Error(w, "No nodes available", http.StatusInternalServerError) 70 | return 71 | } 72 | 73 | // Get partition ID with bounds checking 74 | partitionID := graphdb.GetPartitionID(memberID, len(h.nodes)) 75 | if partitionID < 0 || partitionID >= len(h.nodes) { 76 | http.Error(w, "Invalid partition ID calculated", http.StatusInternalServerError) 77 | return 78 | } 79 | 80 | // Validate node exists 81 | if h.nodes[partitionID] == nil { 82 | http.Error(w, fmt.Sprintf("Node %d not initialized", partitionID), http.StatusInternalServerError) 83 | return 84 | } 85 | 86 | 87 | partition, exists := h.nodes[partitionID].Partition[partitionID] 88 | if !exists || partition == nil { 89 | http.Error(w, fmt.Sprintf("Partition %d not found", partitionID), http.StatusInternalServerError) 90 | return 91 | } 92 | 93 | 94 | connections, err := partition.GetConnections(ctx, memberID) 95 | if err != nil { 96 | log.Printf("Error getting connections for member %d: %v", memberID, err) 97 | http.Error(w, "Failed to retrieve connections", http.StatusInternalServerError) 98 | return 99 | } 100 | 101 | 102 | w.Header().Set("Content-Type", "application/json") 103 | if err := json.NewEncoder(w).Encode(connections); err != nil { 104 | log.Printf("Error encoding response: %v", err) 105 | http.Error(w, "Failed to encode response", http.StatusInternalServerError) 106 | return 107 | } 108 | } 109 | 110 | 111 | func (h *Handler) GetSharedConnections(w http.ResponseWriter, r *http.Request) { 112 | ctx := r.Context() 113 | vars := mux.Vars(r) 114 | 115 | member1ID, err := parseMemberID(vars["memberID1"]) 116 | if err != nil { 117 | http.Error(w, fmt.Sprintf("Invalid member ID 1: %v", err), http.StatusBadRequest) 118 | return 119 | } 120 | 121 | member2ID, err := parseMemberID(vars["memberID2"]) 122 | if err != nil { 123 | http.Error(w, fmt.Sprintf("Invalid member ID 2: %v", err), http.StatusBadRequest) 124 | return 125 | } 126 | 127 | partition1 := graphdb.GetPartitionID(member1ID, len(h.nodes)) 128 | partition2 := graphdb.GetPartitionID(member2ID, len(h.nodes)) 129 | 130 | connections1, err := h.nodes[partition1].Partition[partition1].GetConnections(ctx, member1ID) 131 | if err != nil { 132 | http.Error(w, err.Error(), http.StatusInternalServerError) 133 | return 134 | } 135 | 136 | connections2, err := h.nodes[partition2].Partition[partition2].GetConnections(ctx, member2ID) 137 | if err != nil { 138 | http.Error(w, err.Error(), http.StatusInternalServerError) 139 | return 140 | } 141 | 142 | // Find shared connections 143 | shared := make([]models.MemberID, 0) 144 | connectionMap := make(map[models.MemberID]bool) 145 | 146 | for _, conn := range connections1 { 147 | connectionMap[conn] = true 148 | } 149 | 150 | for _, conn := range connections2 { 151 | if connectionMap[conn] { 152 | shared = append(shared, conn) 153 | } 154 | } 155 | 156 | json.NewEncoder(w).Encode(shared) 157 | } 158 | 159 | // handles the distance calculation between members 160 | func (h *Handler) GetDistances(w http.ResponseWriter, r *http.Request) { 161 | ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) 162 | defer cancel() 163 | 164 | var request struct { 165 | SourceID models.MemberID `json:"sourceId"` 166 | DestinationIDs []models.MemberID `json:"destinationIds"` 167 | } 168 | if err := json.NewDecoder(r.Body).Decode(&request); err != nil { 169 | http.Error(w, err.Error(), http.StatusBadRequest) 170 | return 171 | } 172 | 173 | secondDegree, err := h.networkCache.GetSecondDegree(ctx, request.SourceID) // Updated with context 174 | if err != nil { 175 | secondDegree, err = h.buildSecondDegreeConnections(ctx, request.SourceID) 176 | if err != nil { 177 | http.Error(w, err.Error(), http.StatusInternalServerError) 178 | return 179 | } 180 | } 181 | 182 | distances := make([]models.GraphDistance, 0, len(request.DestinationIDs)) 183 | for _, destID := range request.DestinationIDs { 184 | distance := h.calculateDistance(ctx, request.SourceID, destID, secondDegree) 185 | distances = append(distances, models.GraphDistance{ 186 | SourceID: request.SourceID, 187 | DestID: destID, 188 | Distance: distance, 189 | }) 190 | } 191 | 192 | json.NewEncoder(w).Encode(distances) 193 | } 194 | 195 | func (h *Handler) buildSecondDegreeConnections(ctx context.Context, memberID models.MemberID) ([]models.MemberID, error) { 196 | partitionID := graphdb.GetPartitionID(memberID, len(h.nodes)) 197 | firstDegree, err := h.nodes[partitionID].Partition[partitionID].GetConnections(ctx, memberID) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | requiredPartitions := make(map[int]struct{}) 203 | for _, conn := range firstDegree { 204 | pid := graphdb.GetPartitionID(conn, len(h.nodes)) 205 | requiredPartitions[pid] = struct{}{} 206 | } 207 | 208 | nodesList := make([]ncs.GraphNode, len(h.nodes)) 209 | for i, node := range h.nodes { 210 | nodesList[i] = ncs.GraphNode{ 211 | ID: node.ID, 212 | Partitions: make(map[int]struct{}), 213 | } 214 | for pid := range node.Partition { 215 | nodesList[i].Partitions[pid] = struct{}{} 216 | } 217 | } 218 | 219 | selectedNodes := ncs.FindMinimumNodeSet(requiredPartitions, nodesList) 220 | var secondDegree []models.MemberID 221 | 222 | for _, nodeInfo := range selectedNodes { 223 | for i, node := range h.nodes { 224 | if node.ID == nodeInfo.ID { 225 | connections, err := h.nodes[i].GetSecondDegreeConnections(ctx, firstDegree) 226 | if err != nil { 227 | continue 228 | } 229 | secondDegree = append(secondDegree, connections...) 230 | break 231 | } 232 | } 233 | } 234 | 235 | err = h.networkCache.StoreSecondDegree(ctx, memberID, secondDegree) 236 | if err != nil { 237 | log.Printf("Failed to cache second degree connections: %v", err) 238 | } 239 | 240 | return secondDegree, nil 241 | } 242 | 243 | func (h *Handler) calculateDistance(ctx context.Context, source, dest models.MemberID, secondDegree []models.MemberID) int { 244 | partitionID := graphdb.GetPartitionID(source, len(h.nodes)) 245 | firstDegree, err := h.nodes[partitionID].Partition[partitionID].GetConnections(ctx, source) 246 | if err == nil { 247 | for _, conn := range firstDegree { 248 | if conn == dest { 249 | return 1 250 | } 251 | } 252 | } 253 | 254 | for _, conn := range secondDegree { 255 | if conn == dest { 256 | return 2 257 | } 258 | } 259 | 260 | destPartitionID := graphdb.GetPartitionID(dest, len(h.nodes)) 261 | destConnections, err := h.nodes[destPartitionID].Partition[destPartitionID].GetConnections(ctx, dest) 262 | if err == nil { 263 | for _, conn := range destConnections { 264 | for _, secondConn := range secondDegree { 265 | if conn == secondConn { 266 | return 3 267 | } 268 | } 269 | } 270 | } 271 | 272 | return 4 // More than 3 degrees away 273 | } -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "strings" 5 | "os" 6 | ) 7 | 8 | type RedisConfig struct { 9 | Addresses []string 10 | } 11 | 12 | func LoadRedisConfig() RedisConfig { 13 | // Get Redis nodes from environment variable 14 | redisNodes := os.Getenv("REDIS_NODES") 15 | if redisNodes == "" { 16 | redisNodes = "localhost:6379" 17 | } 18 | 19 | return RedisConfig{ 20 | Addresses: strings.Split(redisNodes, ","), 21 | } 22 | } -------------------------------------------------------------------------------- /internal/graphdb/node.go: -------------------------------------------------------------------------------- 1 | package graphdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | // "github.com/go-redis/redis/v8" 9 | "github.com/mrinalxdev/bidirect/pkg/models" 10 | ) 11 | 12 | type Node struct { 13 | ID string 14 | Partition map[int]*Partition 15 | mu sync.RWMutex 16 | } 17 | 18 | type NodeConfig struct { 19 | ID string 20 | PartitionIDs []int 21 | RedisAddr string 22 | ReplicaFactor int 23 | } 24 | 25 | func NewNode(config NodeConfig) *Node { 26 | node := &Node{ 27 | ID : config.ID, 28 | Partition: make(map[int]*Partition), 29 | } 30 | 31 | for _, pid := range config.PartitionIDs{ 32 | node.Partition[pid] = NewPartition(pid, config.RedisAddr) 33 | } 34 | 35 | return node 36 | } 37 | 38 | // retrives and merges second-degree connections for 39 | // given number Id's from node's particition 40 | func (n *Node) GetSecondDegreeConnections(ctx context.Context, memberIDs []models.MemberID) ([]models.MemberID, error) { 41 | n.mu.RLock() 42 | defer n.mu.RUnlock() 43 | uniqueConnections := make(map[models.MemberID]struct{}) 44 | 45 | for _, memberID := range memberIDs { 46 | partitionID := GetPartitionID(memberID, len(n.Partition)) 47 | partition, exists := n.Partition[partitionID] 48 | if !exists { 49 | continue 50 | } 51 | 52 | connections, err := partition.GetConnections(ctx, memberID) 53 | if err != nil { 54 | return nil, fmt.Errorf("failed to get connections for member %d: %w", memberID, err) 55 | } 56 | 57 | for _, conn := range connections { 58 | uniqueConnections[conn] = struct{}{} 59 | } 60 | } 61 | 62 | result := make([]models.MemberID, 0, len(uniqueConnections)) 63 | for conn := range uniqueConnections { 64 | result = append(result, conn) 65 | } 66 | 67 | return result, nil 68 | } -------------------------------------------------------------------------------- /internal/graphdb/partition.go: -------------------------------------------------------------------------------- 1 | package graphdb 2 | 3 | import ( 4 | "context" 5 | // "crypto/sha256" 6 | // "encoding/binary" 7 | "fmt" 8 | "strconv" 9 | 10 | "github.com/go-redis/redis/v8" 11 | "github.com/mrinalxdev/bidirect/pkg/models" 12 | ) 13 | 14 | type Partition struct { 15 | ID int 16 | RedisClient *redis.Client 17 | } 18 | 19 | func NewPartition(id int, redisAddr string) *Partition { 20 | client := redis.NewClient(&redis.Options{ 21 | Addr: redisAddr, 22 | }) 23 | 24 | return &Partition{ 25 | ID : id, 26 | RedisClient: client, 27 | } 28 | } 29 | 30 | func GetPartitionID(memberID models.MemberID, totalPartitions int) int { 31 | 32 | if totalPartitions <= 0 { 33 | return 0 34 | } 35 | 36 | // hash := sha256.Sum256([]byte(string(memberID))) 37 | // return int(binary.BigEndian.Uint64(hash[:8]) % uint64(totalPartitions)) 38 | return int(uint64(memberID) % uint64(totalPartitions)) 39 | } 40 | 41 | func (p *Partition) StoreConnection(ctx context.Context, conn models.Connection) error { 42 | // Store connections in Redis using sorted sets 43 | key := fmt.Sprintf("connections:%d", conn.SourceID) 44 | return p.RedisClient.ZAdd(ctx, key, &redis.Z{ 45 | Score: float64(conn.DestID), 46 | Member: conn.DestID, 47 | }).Err() 48 | } 49 | 50 | func (p *Partition) GetConnections(ctx context.Context, memberID models.MemberID) ([]models.MemberID, error) { 51 | key := fmt.Sprintf("connections:%d", memberID) 52 | results, err := p.RedisClient.ZRange(ctx, key, 0, -1).Result() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | connections := make([]models.MemberID, len(results)) 58 | for i, result := range results { 59 | memberID, _ := strconv.ParseInt(result, 10, 64) 60 | connections[i] = models.MemberID(memberID) 61 | } 62 | return connections, nil 63 | } -------------------------------------------------------------------------------- /internal/ncs/cache.go: -------------------------------------------------------------------------------- 1 | package ncs 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | "github.com/mrinalxdev/bidirect/pkg/models" 11 | ) 12 | 13 | type NetworkCache struct { 14 | redisClient *redis.Client 15 | ttl time.Duration 16 | } 17 | 18 | func NewNetworkCache(redisAddr string, ttl time.Duration) *NetworkCache { 19 | client := redis.NewClient(&redis.Options{ 20 | Addr: redisAddr, 21 | }) 22 | 23 | return &NetworkCache{ 24 | redisClient: client, 25 | ttl: ttl, 26 | } 27 | } 28 | 29 | func (nc *NetworkCache) StoreSecondDegree(ctx context.Context, memberID models.MemberID, connections []models.MemberID) error { 30 | key := fmt.Sprintf("second_degree:%d", memberID) 31 | 32 | pipe := nc.redisClient.Pipeline() 33 | for _, conn := range connections{ 34 | pipe.ZAdd(ctx, key, &redis.Z{ 35 | Score: float64(conn), 36 | Member: conn, 37 | }) 38 | } 39 | pipe.Expire(ctx, key, nc.ttl) 40 | 41 | _, err := pipe.Exec(ctx) 42 | return err 43 | } 44 | 45 | func (nc *NetworkCache) GetSecondDegree(ctx context.Context, memberId models.MemberID) ([] models.MemberID, error){ 46 | key := fmt.Sprintf("second_degree:%d", memberId) 47 | results, err := nc.redisClient.ZRange(ctx, key, 0, -1).Result() 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | connections := make([]models.MemberID, len(results)) 53 | for i, result := range results { 54 | memberId, _ := strconv.ParseInt(result, 10, 64) 55 | connections[i] = models.MemberID(memberId) 56 | } 57 | 58 | return connections, nil 59 | } -------------------------------------------------------------------------------- /internal/ncs/setcover.go: -------------------------------------------------------------------------------- 1 | package ncs 2 | 3 | type GraphNode struct { 4 | ID string 5 | Partitions map[int]struct{} 6 | } 7 | 8 | func FindMinimumNodeSet(requiredPartitions map[int]struct{}, nodes []GraphNode) []GraphNode { 9 | var result []GraphNode 10 | uncovered := make(map[int]struct{}) 11 | 12 | 13 | for partition := range requiredPartitions { 14 | uncovered[partition] = struct{}{} 15 | } 16 | 17 | for len(uncovered) > 0 { 18 | bestNode := findBestCoveringNode(uncovered, nodes) 19 | if bestNode == nil { 20 | break 21 | } 22 | 23 | result = append(result, *bestNode) 24 | for partition := range bestNode.Partitions { 25 | delete(uncovered, partition) 26 | } 27 | } 28 | 29 | return result 30 | } 31 | 32 | func findBestCoveringNode(uncovered map[int]struct{}, nodes []GraphNode) *GraphNode { 33 | var bestNode *GraphNode 34 | maxCovered := 0 35 | 36 | for i := range nodes { 37 | covered := 0 38 | for partition := range nodes[i].Partitions { 39 | if _, exists := uncovered[partition]; exists { 40 | covered++ 41 | } 42 | } 43 | 44 | if covered > maxCovered { 45 | maxCovered = covered 46 | bestNode = &nodes[i] 47 | } 48 | } 49 | 50 | return bestNode 51 | } -------------------------------------------------------------------------------- /pkg/models/graph.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type MemberID int64 4 | 5 | type Connection struct { 6 | SourceID MemberID 7 | DestID MemberID 8 | Metadata map[string]interface{} 9 | } 10 | 11 | type GraphDistance struct { 12 | SourceID MemberID 13 | DestID MemberID 14 | Distance int 15 | } 16 | --------------------------------------------------------------------------------