├── settings.gradle ├── src ├── main │ ├── lombok,.config │ └── java │ │ ├── model │ │ ├── Direction.java │ │ ├── TimestampedPropertyValue.java │ │ ├── GraphEntity.java │ │ ├── Edge.java │ │ ├── Vertex.java │ │ ├── TimeSpace.java │ │ ├── Property.java │ │ ├── PropertyAwareEntity.java │ │ ├── TemporalProperty.java │ │ └── Graph.java │ │ ├── core │ │ ├── utils │ │ │ └── CommonUtils.java │ │ ├── GraphEntityLabelManager.java │ │ ├── propertystore │ │ │ └── PropertyStore.java │ │ ├── InMemoryGraphStorage.java │ │ └── TemporalGraph.java │ │ ├── exceptions │ │ └── PropertyNotFoundException.java │ │ ├── transformers │ │ ├── EncoderDecoder.java │ │ ├── IntZigZagEncoderDecoder.java │ │ ├── VarLongEncoderDecoder.java │ │ ├── VarIntEncoderDecoder.java │ │ ├── VarIntArrayEncoderDecoder.java │ │ ├── VarLongArrayEncoderDecoder.java │ │ ├── MetricsDecoderEncoderHandler.java │ │ └── DeltaEncoderDecoder.java │ │ └── Driver.java ├── test │ └── java │ │ ├── core │ │ ├── LongPropertyTimeSeriesGenerator.java │ │ ├── IntegerPropertyTimeSeriesGenerator.java │ │ ├── AbstractPropertyTimeSeriesGenerator.java │ │ ├── InMemoryGraphStoragePerfTest.java │ │ └── TemporalGraphTests.java │ │ ├── transformers │ │ ├── IntZigZagEncoderDecoderTest.java │ │ ├── VarIntEncoderDecoderTest.java │ │ ├── VarLongEncoderDecoderTest.java │ │ ├── VarLongArrayEncoderDecoderTest.java │ │ ├── VarIntArrayEncoderDecoderTest.java │ │ └── DeltaEncoderDecoderTest.java │ │ └── model │ │ └── TemporalPropertyTests.java └── jmh │ └── java │ └── transformers │ ├── VarLongArrayEncoderDecoderBenchmark.java │ ├── VarIntArrayEncoderDecoderBenchmark.java │ └── DeltaEncoderDecoderBenchmark.java ├── .gitignore ├── NOTICE ├── LICENSE ├── CONTRIBUTING_DCO.md ├── gradlew.bat ├── README.md ├── CODE_OF_CONDUCT.md └── gradlew /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'temporalgraph' 2 | 3 | -------------------------------------------------------------------------------- /src/main/lombok,.config: -------------------------------------------------------------------------------- 1 | lombok.addLombokGeneratedAnnotation = true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ projects 2 | .idea 3 | # Mac system files 4 | .DS_store 5 | # Intermittent build files 6 | build/ 7 | # gradle directories 8 | .gradle/ 9 | gradle/ 10 | -------------------------------------------------------------------------------- /src/main/java/model/Direction.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * The direction of the relation 10 | */ 11 | public enum Direction { 12 | IN, 13 | OUT 14 | ; 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/core/utils/CommonUtils.java: -------------------------------------------------------------------------------- 1 | package core.utils; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | public class CommonUtils { 9 | public static long pack2IntsInLong(final int int1, final int int2) { 10 | return (((long) int1) << 32) | (int2 & 0xffffffffL); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/exceptions/PropertyNotFoundException.java: -------------------------------------------------------------------------------- 1 | package exceptions; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | public class PropertyNotFoundException extends Exception { 9 | public PropertyNotFoundException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/model/TimestampedPropertyValue.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import lombok.AllArgsConstructor; 9 | import lombok.Data; 10 | 11 | @Data 12 | @AllArgsConstructor 13 | public final class TimestampedPropertyValue { 14 | private final String name; 15 | private final Object value; 16 | } 17 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2023 VMware, Inc. 2 | 3 | This product is licensed to you under the BSD 2 clause (the "License"). You may not use this product except in compliance with the License. 4 | 5 | This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. -------------------------------------------------------------------------------- /src/main/java/transformers/EncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * Represents a contract for encoding and decoding a value to 10 | * be stored in the graph. 11 | * Encoders and decoders are generally defined in pairs to ensure 12 | * data integrity and consistency during the encoding and decoding process 13 | * 14 | * @param The source data type before the encoding process 15 | * @param The destination data type of the encoding process 16 | */ 17 | public interface EncoderDecoder { 18 | 19 | U encode(T data); 20 | 21 | T decode(U encoded); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/Driver.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 VMware, Inc. 3 | SPDX-License-Identifier: BSD-2-Clause 4 | */ 5 | 6 | public class Driver { 7 | public static void main(String[] args) { 8 | 9 | } 10 | 11 | private static long packTwoInts(int integer1, int integer2) { 12 | final long lsbMask = 0x0000_0000_ffff_ffff; 13 | 14 | return ((integer1 & lsbMask) << 32) | (integer2 & lsbMask); 15 | 16 | } 17 | 18 | private static int[] unpackTwoInts(long value) { 19 | final long lsbMask = 0x0000_0000_ffff_ffff; 20 | int[] values = new int[2]; 21 | values[1] = (int)(value & lsbMask); 22 | values[0] = (int)(value >>> 32); 23 | return values; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/core/LongPropertyTimeSeriesGenerator.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | import java.util.TreeMap; 12 | 13 | public class LongPropertyTimeSeriesGenerator extends AbstractPropertyTimeSeriesGenerator { 14 | 15 | @Override 16 | public Map getValueTimeSeries(int resolutionInMins, long startTime) { 17 | List timeUnits = getTimeUnits(startTime, resolutionInMins); 18 | Map valueTimeSeries = new TreeMap<>(); 19 | Random r = new Random(); 20 | for (Long timeUnit : timeUnits) { 21 | valueTimeSeries.put(timeUnit, r.nextLong()); 22 | } 23 | return valueTimeSeries; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/model/GraphEntity.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * An object that represents a core entity within a graph whoch could be one of:
    Vertex
,
    Edge
10 | *
    Property
11 | *
12 | * Every graph entity object has a unique identifier and provides the time-instant at which 13 | * it got created 14 | */ 15 | public interface GraphEntity { 16 | 17 | /** 18 | * Gets the Identifier associated with the graph entity. 19 | * @return the entity identifier 20 | */ 21 | int getId(); 22 | 23 | /** 24 | * Gets the time instance at which the graph entity was created. 25 | * @return the time instant in milliseconds at which the entity got created. 26 | */ 27 | long getTime(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/core/IntegerPropertyTimeSeriesGenerator.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | import java.util.TreeMap; 12 | 13 | public class IntegerPropertyTimeSeriesGenerator extends AbstractPropertyTimeSeriesGenerator { 14 | 15 | @Override 16 | public Map getValueTimeSeries(int resolutionInMins, long startTime) { 17 | List timeUnits = getTimeUnits(startTime, resolutionInMins); 18 | Map valueTimeSeries = new TreeMap<>(); 19 | Random r = new Random(); 20 | for (Long timeUnit : timeUnits) { 21 | valueTimeSeries.put(timeUnit, Math.abs(r.nextInt())); 22 | } 23 | return valueTimeSeries; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/model/Edge.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * Contract representing an edge between two vertices. Edge gave a direction as specified by 10 | * the {@link Direction} enum and can have one or more properties and a label. 11 | * A label represents a classifier for an edge. For e.g. there could be edges with\ 12 | * the label as NetworkConnectivity which is the set of all edges that represent 13 | * a network communication between two workloads or IP addresses. 14 | * The association between an edge, and it's set of properties is time aware. 15 | * It exposes mechanisms to retrieve the value of a given property at a particular time. 16 | * Similarly, it is also exposes mechanism to retrieve a list of properties at a particular time instant. 17 | */ 18 | public interface Edge extends PropertyAwareEntity { 19 | 20 | String getLabel(); 21 | 22 | Vertex getSrcVertex(); 23 | 24 | Vertex getDestVertex(); 25 | 26 | Direction getDirection(); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/core/GraphEntityLabelManager.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.apache.commons.collections4.BidiMap; 9 | import org.apache.commons.collections4.bidimap.DualHashBidiMap; 10 | 11 | import java.util.Objects; 12 | 13 | class GraphEntityLabelManager { 14 | 15 | private final BidiMap nameToIdMap = new DualHashBidiMap<>(); 16 | 17 | Long addLabel(final String label) { 18 | if (nameToIdMap.containsKey(label)) { 19 | throw new IllegalArgumentException(String.format("Label %s already exists. Labels names must be " + 20 | "unique in the edge and vertex space", label)); 21 | } 22 | long labelId = Objects.hash(System.currentTimeMillis(), label); 23 | nameToIdMap.put(label, labelId); 24 | return labelId; 25 | } 26 | 27 | String getLabel(final long labelId) { 28 | return nameToIdMap.getKey(labelId); 29 | } 30 | 31 | void deleteLabel(final String label) { 32 | nameToIdMap.remove(label); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/core/AbstractPropertyTimeSeriesGenerator.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | public abstract class AbstractPropertyTimeSeriesGenerator { 14 | 15 | public abstract Map getValueTimeSeries(final int resolutionInMins, final long startTime); 16 | 17 | protected List getTimeUnits(final long startTime, final int resolutionMins) { 18 | // divide 24 hours into the specified resolution 19 | long resolutionMillis = TimeUnit.MILLISECONDS.convert(resolutionMins, TimeUnit.MINUTES); 20 | long millisInDay = TimeUnit.MILLISECONDS.convert(1L, TimeUnit.DAYS); 21 | int numUnits = Long.valueOf(millisInDay / resolutionMillis).intValue(); 22 | List timeUnits = new ArrayList<>(); 23 | timeUnits.add(startTime); 24 | for (int i = 1; i < numUnits; ++i) { 25 | timeUnits.add(startTime + (resolutionMillis * i)); 26 | } 27 | return timeUnits; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Redistribution and use in source and binary forms, with or without 2 | modification, are permitted provided that the following conditions are 3 | met: 4 | 5 | 1. Redistributions of source code must retain the above copyright 6 | notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above 9 | copyright notice, this list of conditions and the following 10 | disclaimer in the documentation and/or other materials provided 11 | with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 14 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 15 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 16 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 17 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 19 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /src/main/java/transformers/IntZigZagEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | import com.google.common.annotations.VisibleForTesting; 4 | 5 | import java.io.ByteArrayOutputStream; 6 | import java.io.IOException; 7 | 8 | /* 9 | Copyright 2023 VMware, Inc. 10 | SPDX-License-Identifier: BSD-2-Clause 11 | */ 12 | 13 | /** 14 | * An encoder/decoder that decodes an arbitrary collection of positive and negative integers 15 | * using the ZigZag encoding/decoding scheme. 16 | *
17 | * Reference : https://golb.hplar.ch/2019/06/variable-length-int-java.html 18 | */ 19 | public class IntZigZagEncoderDecoder implements EncoderDecoder { 20 | 21 | private final VarIntEncoderDecoder intEncoderDecoder; 22 | 23 | public IntZigZagEncoderDecoder() { 24 | this(new VarIntEncoderDecoder()); 25 | } 26 | 27 | @VisibleForTesting 28 | IntZigZagEncoderDecoder(VarIntEncoderDecoder encoderDecoder) { 29 | this.intEncoderDecoder = encoderDecoder; 30 | } 31 | 32 | 33 | @Override 34 | public byte[] encode(Integer data) { 35 | int temp = (data << 1) ^ (data >> 31); 36 | return intEncoderDecoder.encode(temp); 37 | } 38 | 39 | @Override 40 | public Integer decode(byte[] encoded) { 41 | int temp = intEncoderDecoder.decode(encoded); 42 | return (temp >>> 1) ^ -(temp & 1); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/transformers/IntZigZagEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import static org.junit.jupiter.api.Assertions.*; 12 | 13 | class IntZigZagEncoderDecoderTest { 14 | 15 | private IntZigZagEncoderDecoder underTest; 16 | 17 | @BeforeEach 18 | public void init() { 19 | underTest = new IntZigZagEncoderDecoder(); 20 | } 21 | 22 | @Test 23 | public void encodeAndDecode() { 24 | { 25 | int value = 3; 26 | byte[] encoded = underTest.encode(value); 27 | assertEquals(value, underTest.decode(encoded)); 28 | } 29 | 30 | { 31 | int value = -3; 32 | byte[] encoded = underTest.encode(value); 33 | assertEquals(value, underTest.decode(encoded)); 34 | } 35 | 36 | { 37 | int value = -345623588; 38 | byte[] encoded = underTest.encode(value); 39 | assertEquals(value, underTest.decode(encoded)); 40 | } 41 | { 42 | int value = (-1 * 10_000_000); 43 | byte[] encoded = underTest.encode(value); 44 | assertEquals(value, underTest.decode(encoded)); 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /src/test/java/transformers/VarIntEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import static org.junit.jupiter.api.Assertions.*; 12 | 13 | class VarIntEncoderDecoderTest { 14 | 15 | private VarIntEncoderDecoder underTest; 16 | 17 | @BeforeEach 18 | public void init() { 19 | underTest = new VarIntEncoderDecoder(); 20 | } 21 | 22 | @Test 23 | public void encode() { 24 | byte[] encoded = underTest.encode(300); 25 | assertEquals(2, encoded.length); 26 | assertEquals((byte)-84, encoded[0]); 27 | assertEquals((byte)2, encoded[1]); 28 | 29 | encoded = underTest.encode(Integer.MAX_VALUE - 1); 30 | assertEquals(5, encoded.length); 31 | 32 | encoded = underTest.encode(45); 33 | assertEquals(1, encoded.length); 34 | } 35 | 36 | @Test 37 | public void decode() { 38 | byte[] encoded = underTest.encode(300); 39 | int decoded = underTest.decode(encoded); 40 | assertEquals(300, decoded); 41 | 42 | encoded = underTest.encode(Integer.MAX_VALUE - 1); 43 | decoded = underTest.decode(encoded); 44 | assertEquals(Integer.MAX_VALUE - 1, decoded); 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /src/test/java/transformers/VarLongEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | 13 | class VarLongEncoderDecoderTest { 14 | 15 | private VarLongEncoderDecoder underTest; 16 | 17 | @BeforeEach 18 | public void init() { 19 | underTest = new VarLongEncoderDecoder(); 20 | } 21 | 22 | @Test 23 | public void encode() { 24 | byte[] encoded = underTest.encode(300L); 25 | assertEquals(2, encoded.length); 26 | assertEquals((byte)-84, encoded[0]); 27 | assertEquals((byte)2, encoded[1]); 28 | 29 | encoded = underTest.encode(Integer.MAX_VALUE + 1L); 30 | assertEquals(5, encoded.length); 31 | 32 | encoded = underTest.encode(45L); 33 | assertEquals(1, encoded.length); 34 | } 35 | 36 | @Test 37 | public void decode() { 38 | byte[] encoded = underTest.encode(300L); 39 | long decoded = underTest.decode(encoded); 40 | assertEquals(300, decoded); 41 | 42 | encoded = underTest.encode(Integer.MAX_VALUE + 1L); 43 | decoded = underTest.decode(encoded); 44 | assertEquals(Integer.MAX_VALUE + 1L, decoded); 45 | 46 | long time = System.currentTimeMillis(); 47 | encoded = underTest.encode(time); 48 | decoded = underTest.decode(encoded); 49 | assertEquals(decoded, time); 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /src/main/java/transformers/VarLongEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * A variable length integer encoder/decoder for unsigned or positive integer values 10 | * The implementation does not currently support the varint encoding of negative/signed numbers 11 | * The encoding mechanism used is a simple varint encoding 12 | */ 13 | public class VarLongEncoderDecoder implements EncoderDecoder { 14 | 15 | @Override 16 | public byte[] encode(Long data) { 17 | if (data < 0) { 18 | throw new IllegalArgumentException("VarInt encoding only supports non-negative/unsigned integers"); 19 | } 20 | byte[] buffer = new byte[9]; 21 | int position = 0; 22 | while (true) { 23 | if ((data & 0b1111111111111111111111111111111111111111111111111111111110000000L) == 0) { 24 | buffer[position++] = (byte)(data.intValue()); 25 | break; 26 | } 27 | buffer[position++] = (byte)((data & 0b1111111) | 0b10000000); 28 | data >>>= 7; 29 | } 30 | byte[] encoded = new byte[position]; 31 | System.arraycopy(buffer, 0, encoded, 0, position); 32 | return encoded; 33 | } 34 | 35 | @Override 36 | public Long decode(byte[] encoded) { 37 | long result = 0; 38 | int shift = 0; 39 | for (byte b : encoded) { 40 | result |= (long) (b & 0b1111111) << shift; 41 | shift += 7; 42 | if ((b & 0b10000000) == 0) { 43 | return result; 44 | } 45 | } 46 | return result; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/transformers/VarIntEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * A variable length integer encoder/decoder for unsigned or positive integer values 10 | * The implementation does not currently support the varint encoding of negative/signed numbers 11 | * The encoding mechanism used is a simple varint encoding. 12 | *
13 | * Reference: https://golb.hplar.ch/2019/06/variable-length-int-java.html 14 | */ 15 | public class VarIntEncoderDecoder implements EncoderDecoder { 16 | 17 | @Override 18 | public byte[] encode(Integer data) { 19 | if (data < 0) { 20 | throw new IllegalArgumentException("VarInt encoding only supports non-negative/unsigned integers"); 21 | } 22 | byte[] buffer = new byte[5]; 23 | int position = 0; 24 | while (true) { 25 | if ((data & 0b11111111111111111111111110000000) == 0) { 26 | buffer[position++] = (byte)(data.intValue()); 27 | break; 28 | } 29 | buffer[position++] = (byte)((data & 0b1111111) | 0b10000000); 30 | data >>>= 7; 31 | } 32 | byte[] encoded = new byte[position]; 33 | System.arraycopy(buffer, 0, encoded, 0, position); 34 | return encoded; 35 | } 36 | 37 | @Override 38 | public Integer decode(byte[] encoded) { 39 | int result = 0; 40 | int shift = 0; 41 | for (byte b : encoded) { 42 | result |= (b & 0b1111111) << shift; 43 | shift += 7; 44 | if ((b & 0b10000000) == 0) { 45 | return result; 46 | } 47 | } 48 | return result; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/model/Vertex.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.Collection; 9 | 10 | /** 11 | * An object representing a vertex within a graph. 12 | * {@link Vertex} instances have labels which are used to classifiy the vertices. For e.g. 13 | * there could be vertices in a graph with label VM representing the Virtual machine 14 | * workloads in a data-center graph. 15 | * Vertices can have one or more properties. Vertices can also have one more incoming or cutgoing edges 16 | * The association between {@link Vertex} and the set of properties and incoming or outgoing edges is time aware 17 | */ 18 | public interface Vertex extends PropertyAwareEntity { 19 | 20 | /** 21 | * Get the label associated with a vertex 22 | * @return the label associated with the vertex 23 | */ 24 | String getLabel(); 25 | 26 | /** 27 | * Get the outgoing edges from a vertex at specific timestamp 28 | * @param timestamp the timestamp at which the outgoing edges need to be retrieved 29 | * @return the collection of {@link Edge} instances representing outgoing edges 30 | */ 31 | Collection getOutEdgesAtTime(long timestamp); 32 | 33 | /** 34 | * Get the incoming edges from a vertex at specific timestamp 35 | * @param timestamp the timestamp at which the incoming edges need to be retrieved 36 | * @return the collection of {@link Edge} instances representing incoming edges 37 | */ 38 | Collection getInEdgesAtTime(long timestamp); 39 | 40 | /** 41 | * Add an edge to the vertex at the specified time. 42 | * @param e the edge instance to be added 43 | * @param timestamp the timestamp at which the edge should be added to the vertex 44 | */ 45 | void addEdgeAtTime(Edge e, long timestamp); 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/transformers/VarLongArrayEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.util.Random; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 14 | 15 | class VarLongArrayEncoderDecoderTest { 16 | 17 | private VarLongArrayEncoderDecoder underTest; 18 | 19 | @BeforeEach 20 | public void init() { 21 | underTest = new VarLongArrayEncoderDecoder(); 22 | } 23 | 24 | @Test 25 | public void encode() { 26 | byte[] bytes = underTest.encode(new long[]{127L, 126L, 32767L, 32764L}); 27 | } 28 | 29 | @Test 30 | public void decode() { 31 | long[] input = new long[]{127L, 126L, 32767L, 32764L}; 32 | byte[] bytes = underTest.encode(input); 33 | long[] decoded = underTest.decode(bytes); 34 | assertArrayEquals(decoded, input); 35 | } 36 | 37 | @Test 38 | public void largeInput() { 39 | int limit = 7000000; 40 | Random r = new Random(); 41 | long[] input = new long[limit]; 42 | for (int i = 0; i < limit; i++) { 43 | input[i] = Math.abs(r.nextLong()); 44 | } 45 | long now = System.currentTimeMillis(); 46 | byte[] bytes = underTest.encode(input); 47 | long later = System.currentTimeMillis(); 48 | System.out.println("Size of array in bytes for 7M integers:" + bytes.length + " in " + (later-now) + " " + 49 | "ms"); 50 | 51 | now = System.currentTimeMillis(); 52 | long[] decoded = underTest.decode(bytes); 53 | later = System.currentTimeMillis(); 54 | System.out.println("Size of array in bytes for 7M integers:" + bytes.length + " in " + (later-now) + " " + 55 | "ms"); 56 | } 57 | } -------------------------------------------------------------------------------- /src/main/java/model/TimeSpace.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | import java.util.Set; 11 | import java.util.TreeMap; 12 | 13 | public class TimeSpace { 14 | 15 | private final Map> verticesByTime = new TreeMap<>(); 16 | private final Map> edgesByTime = new TreeMap<>(); 17 | private final Map> propertiesByTime = new TreeMap<>(); 18 | 19 | public Set getVerticesAtTime(final long timestamp) { 20 | return Collections.emptySet(); 21 | } 22 | 23 | public Set getEdgesAtTime(final long timestamp) { 24 | return Collections.emptySet(); 25 | } 26 | 27 | public Set getPropertiesAtTime(final long timestamp) { 28 | return Collections.emptySet(); 29 | } 30 | 31 | public void createOrUpdateGraphEntityAtTime(final GraphEntity entity, final long timestamp) { 32 | if (entity instanceof Vertex) { 33 | addToSet(verticesByTime, entity.getId(), timestamp); 34 | return; 35 | } 36 | if (entity instanceof Edge) { 37 | addToSet(edgesByTime, entity.getId(), timestamp); 38 | return; 39 | } 40 | if (entity instanceof Property) { 41 | addToSet(propertiesByTime, entity.getId(), timestamp); 42 | return; 43 | } 44 | throw new IllegalArgumentException(String.format("Unrecognized entity type : %s", 45 | entity.getClass().getCanonicalName())); 46 | } 47 | 48 | 49 | private void addToSet(Map> map, final long entityId, final long timestamp) { 50 | Set entityIds = map.getOrDefault(timestamp, Collections.emptySet()); 51 | entityIds.add(entityId); 52 | map.put(timestamp, entityIds); 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /src/test/java/transformers/VarIntArrayEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.util.Random; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 14 | 15 | class VarIntArrayEncoderDecoderTest { 16 | 17 | private VarIntArrayEncoderDecoder underTest; 18 | 19 | @BeforeEach 20 | public void init() { 21 | underTest = new VarIntArrayEncoderDecoder(); 22 | } 23 | 24 | @Test 25 | public void encode() { 26 | byte[] bytes = underTest.encode(new int[]{127, 126, 32767, 32764}); 27 | } 28 | 29 | @Test 30 | public void decode() { 31 | int[] input = new int[]{127, 126, 32767, 32764}; 32 | byte[] bytes = underTest.encode(input); 33 | int[] decoded = underTest.decode(bytes); 34 | assertArrayEquals(decoded, input); 35 | } 36 | 37 | @Test 38 | public void largeInput() { 39 | int limit = 70_000_000; 40 | Random r = new Random(); 41 | int[] input = new int[limit]; 42 | for (int i = 0; i < limit; i++) { 43 | input[i] = Math.abs(r.nextInt()); 44 | } 45 | byte[] bytes = underTest.encode(input); 46 | System.out.println("Size of array in bytes for 70M integers:" + bytes.length); 47 | } 48 | 49 | @Test 50 | public void largeInput1() throws InterruptedException { 51 | int limit = 70_000_000; 52 | Random r = new Random(); 53 | int[] input = new int[limit]; 54 | for (int i = 0; i < limit; i++) { 55 | input[i] = Math.abs(r.nextInt() % 100_000); 56 | } 57 | byte[] bytes = underTest.encode(input); 58 | System.out.println("TAKE heap dump"); 59 | Thread.sleep(240000); 60 | //System.out.println("Size of array in bytes for 70M integers:" + bytes.length); 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/java/transformers/VarIntArrayEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import com.google.common.annotations.VisibleForTesting; 9 | 10 | import java.io.ByteArrayOutputStream; 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | public class VarIntArrayEncoderDecoder implements EncoderDecoder { 17 | 18 | private final VarIntEncoderDecoder intEncoderDecoder; 19 | 20 | public VarIntArrayEncoderDecoder() { 21 | this(new VarIntEncoderDecoder()); 22 | } 23 | 24 | @VisibleForTesting 25 | VarIntArrayEncoderDecoder(VarIntEncoderDecoder intEncoderDecoder) { 26 | this.intEncoderDecoder = intEncoderDecoder; 27 | } 28 | 29 | @Override 30 | public byte[] encode(int[] data) { 31 | try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { 32 | for (int datum : data) { 33 | bos.write(intEncoderDecoder.encode(datum)); 34 | } 35 | return bos.toByteArray(); 36 | } catch (IOException e) { 37 | throw new IllegalStateException(e); 38 | } 39 | } 40 | 41 | @Override 42 | public int[] decode(byte[] encoded) { 43 | List buffer = new ArrayList<>(); 44 | int index = 0; 45 | int range = 0; 46 | for (byte datum : encoded) { 47 | if ((datum & 0b10000000) != 0) { 48 | range++; 49 | continue; 50 | } 51 | 52 | byte[] toDecode = Arrays.copyOfRange(encoded, index, range + 1); 53 | buffer.add(intEncoderDecoder.decode(toDecode)); 54 | index = range + 1; 55 | range = index; 56 | } 57 | int[] decoded = new int[buffer.size()]; 58 | for (int i = 0; i < buffer.size(); i++) { 59 | decoded[i] = buffer.get(i); 60 | } 61 | return decoded; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/transformers/VarLongArrayEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import com.google.common.annotations.VisibleForTesting; 9 | 10 | import java.io.ByteArrayOutputStream; 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | public class VarLongArrayEncoderDecoder implements EncoderDecoder { 17 | 18 | private final VarLongEncoderDecoder longEncoderDecoder; 19 | 20 | public VarLongArrayEncoderDecoder() { 21 | this(new VarLongEncoderDecoder()); 22 | } 23 | 24 | @VisibleForTesting 25 | VarLongArrayEncoderDecoder(VarLongEncoderDecoder longEncoderDecoder) { 26 | this.longEncoderDecoder = longEncoderDecoder; 27 | } 28 | 29 | @Override 30 | public byte[] encode(long[] data) { 31 | try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { 32 | for (long datum : data) { 33 | bos.write(longEncoderDecoder.encode(datum)); 34 | } 35 | return bos.toByteArray(); 36 | } catch (IOException e) { 37 | throw new IllegalStateException(e); 38 | } 39 | } 40 | 41 | @Override 42 | public long[] decode(byte[] encoded) { 43 | List buffer = new ArrayList<>(); 44 | int index = 0; 45 | int range = 0; 46 | for (byte datum : encoded) { 47 | if ((datum & 0b10000000) != 0) { 48 | range++; 49 | continue; 50 | } 51 | 52 | byte[] toDecode = Arrays.copyOfRange(encoded, index, range + 1); 53 | buffer.add(longEncoderDecoder.decode(toDecode)); 54 | index = range + 1; 55 | range = index; 56 | } 57 | long[] decoded = new long[buffer.size()]; 58 | for (int i = 0; i < buffer.size(); i++) { 59 | decoded[i] = buffer.get(i); 60 | } 61 | return decoded; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/model/Property.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | /** 9 | * An object that represents a property for a {@link GraphEntity} 10 | * Properties themselves are graph entities and are stored as first class 11 | * entities within the storage layer as opposed to attributes or tags on higher level 12 | * entities like {@link Vertex} and {@link Edge}
13 | * Implementations of this interface expose mechanisms to get or set the value of the property 14 | * at a specific timestamp. 15 | *
16 | * For both get and set methods, the implementations are responsible 17 | * for interpreting the timestamp correctly and retrieving/storing the property value against the timestamp. 18 | * For e.g. an implementation of this interface can define the semantics of the {@code get} method to return the 19 | * value of property at the largest time instant less than or equal to the specified time instance. 20 | * Alternatively, another implementation of {@code get} may choose to return the value of the property at 21 | * exactly the specified time instance. 22 | */ 23 | public interface Property extends GraphEntity { 24 | 25 | /** 26 | * Gets the name of the property 27 | * @return the name of the property 28 | */ 29 | String getName(); 30 | 31 | /** 32 | * Returns the value of the property at the specific timestamp. 33 | * @param timestamp the timestamp at which the value needs to be retrieved. 34 | * @return the value of the property at the timeinstant. 35 | */ 36 | Object getValueAtTime(long timestamp); 37 | 38 | /** 39 | * Set the value of the property at a specified time. 40 | * @param timestamp the time instant at which to set the value 41 | * @param value the value to be set. 42 | */ 43 | void setValueAtTime(long timestamp, Object value); 44 | 45 | /** 46 | * Purge the values in the property timeseries till timestamp. 47 | * @param timestamp the time till which values needs to be removed 48 | * @return true of the timeseries is purged else false 49 | */ 50 | boolean purgeTimeSeriesUntilTime(final long timestamp); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/model/PropertyAwareEntity.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import java.util.Collection; 9 | import java.util.Set; 10 | import java.util.TreeMap; 11 | 12 | /** 13 | * Represents an graph entity that can contain properties. 14 | *
15 | * Currently the types of {@link GraphEntity} instances that 16 | * can contain properties are restricted to {@link Vertex} and {@link Edge} instances 17 | * Objects implementing this interface expose mechanisms to retrieve the mapping between 18 | * a particular time instance and the complete set of properties associated with the entity 19 | * at that time instance. 20 | * The time series based representation for the set of properties allows clients of this 21 | * interface to retrieve the full history of property associations with the {@link GraphEntity} instance. 22 | */ 23 | public interface PropertyAwareEntity extends GraphEntity { 24 | 25 | /** 26 | * Gets the full history of the associated properties with the current 27 | * {@link GraphEntity} instance. The history is chronologically ordered 28 | * from oldest to the newest. 29 | * @return A {@link TreeMap} with time instant as the key and a map 30 | * of properties indexed by property name existing at that time instant as a value. 31 | */ 32 | Set getProperties(); 33 | 34 | /** 35 | * Get the properties associated with the vertex at a specified timestamp. 36 | * @param timestamp the timestamp at which the properties need to be retrieved. 37 | * @return the collection of {@link Property} instances at the specified time stamp. 38 | */ 39 | Collection getPropertiesAtTime(long timestamp); 40 | 41 | /** 42 | * Get the property identified by name associated with the vertex at a specified timestamp 43 | * @param name the name of the property to be retrieved 44 | * @param timestamp the timestamp at which the property needs to be retrieved 45 | * @return the {@link Property} instance as available within the vertex at the specified time. 46 | */ 47 | Property getPropertyAtTime(String name, long timestamp); 48 | 49 | /** 50 | * Add a collection of properties to the vertex 51 | * @param properties The collection of properties to be dded. 52 | */ 53 | void addProperties(Collection properties); 54 | 55 | /** 56 | * Removes a property with the specified name from the vertex 57 | * @param propertyName 58 | */ 59 | void removeProperty(String propertyName); 60 | 61 | } 62 | -------------------------------------------------------------------------------- /CONTRIBUTING_DCO.md: -------------------------------------------------------------------------------- 1 | # Contributing to in-memory-property-aware-temporal-graph 2 | 3 | We welcome contributions from the community and first want to thank you for taking the time to contribute! 4 | 5 | Please familiarize yourself with the [Code of Conduct](https://github.com/vmware/.github/blob/main/CODE_OF_CONDUCT.md) before contributing. 6 | 7 | Before you start working with in-memory-property-aware-temporal-graph, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch. 8 | 9 | ## Ways to contribute 10 | 11 | We welcome many different types of contributions and not all of them need a Pull request. Contributions may include: 12 | 13 | * New features and proposals 14 | * Documentation 15 | * Bug fixes 16 | * Issue Triage 17 | * Answering questions and giving feedback 18 | * Helping to onboard new contributors 19 | * Other related activities 20 | 21 | ## Getting started 22 | 23 | Please refer the [README.md] file for the detailed steps for setting up the project. 24 | 25 | ## Contribution Flow 26 | 27 | This is a rough outline of what a contributor's workflow looks like: 28 | 29 | * Make a fork of the repository within your GitHub account 30 | * Create a topic branch in your fork from where you want to base your work 31 | * Make commits of logical units 32 | * Make sure your commit messages are with the proper format, quality and descriptiveness (see below) 33 | * Push your changes to the topic branch in your fork 34 | * Create a pull request containing that commit 35 | 36 | We follow the GitHub workflow and you can find more details on the [GitHub flow documentation](https://docs.github.com/en/get-started/quickstart/github-flow). 37 | 38 | ### Pull Request Checklist 39 | 40 | Before submitting your pull request, we advise you to use the following: 41 | 42 | 1. Check if your code changes will pass both code linting checks and unit tests. 43 | 2. Ensure your commit messages are descriptive. We follow the conventions on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/). Be sure to include any related GitHub issue references in the commit message. See [GFM syntax](https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown) for referencing issues and commits. 44 | 3. Check the commits and commits messages and ensure they are free from typos. 45 | 4. If you are making changes in the core data structure, please include **jmh** memory and cpu benchmark report pre and post your change. 46 | 47 | 48 | ## Reporting Bugs and Creating Issues 49 | 50 | While creating an issue, please try to follow commit message format mentioned in step 2 of "**Pull Request Checklist**" 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/test/java/transformers/DeltaEncoderDecoderTest.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | import static org.junit.jupiter.api.Assertions.*; 15 | 16 | class DeltaEncoderDecoderTest { 17 | 18 | private DeltaEncoderDecoder underTest; 19 | 20 | @BeforeEach 21 | public void init() { 22 | underTest = new DeltaEncoderDecoder(); 23 | } 24 | 25 | @Test 26 | void encode() { 27 | List numberList = new ArrayList<>(); 28 | for (int i = 100; i > 0; i--) { 29 | numberList.add(i); 30 | } 31 | DeltaEncoderDecoder.Result encoded = underTest.encode(numberList); 32 | assertTrue(encoded.getBaseValue() instanceof Integer); 33 | assertSame(encoded.getBaseValue(), (Integer) 100); 34 | long[] encodedData = encoded.getEncodedData(); 35 | assertEquals(encodedData[0], (int) (Integer) 100); 36 | for (int i = 1; i < encodedData.length; i++) { 37 | assertEquals(-1, encodedData[i]); 38 | } 39 | 40 | } 41 | 42 | @Test 43 | void decode() { 44 | List numberList = new ArrayList<>(); 45 | for (int i = 100; i > 0; i--) { 46 | numberList.add(i); 47 | } 48 | DeltaEncoderDecoder.Result encoded = underTest.encode(numberList); 49 | assertTrue(encoded.getBaseValue() instanceof Integer); 50 | assertSame(encoded.getBaseValue(), (Integer) 100); 51 | long[] encodedData = encoded.getEncodedData(); 52 | assertEquals(encodedData[0], (int) (Integer) 100); 53 | for (int i = 1; i < encodedData.length; i++) { 54 | assertEquals(-1, encodedData[i]); 55 | } 56 | List decoded = underTest.decode(encoded); 57 | assertEquals(numberList, decoded); 58 | 59 | } 60 | 61 | @Test 62 | void decodeLong() { 63 | List numberList = new ArrayList<>(); 64 | for (long i = 100L; i > 0L; i--) { 65 | numberList.add(i); 66 | } 67 | DeltaEncoderDecoder.Result encoded = underTest.encode(numberList); 68 | assertTrue(encoded.getBaseValue() instanceof Long); 69 | assertSame(encoded.getBaseValue(), (Long) 100L); 70 | long[] encodedData = encoded.getEncodedData(); 71 | assertEquals(encodedData[0], (long) (Long) 100L); 72 | for (int i = 1; i < encodedData.length; i++) { 73 | assertEquals(-1, encodedData[i]); 74 | } 75 | List decoded = underTest.decode(encoded); 76 | assertEquals(numberList, decoded); 77 | 78 | } 79 | } -------------------------------------------------------------------------------- /src/main/java/transformers/MetricsDecoderEncoderHandler.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import lombok.Getter; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.stream.Collectors; 14 | 15 | public class MetricsDecoderEncoderHandler { 16 | 17 | public Result encode(List data) { 18 | if (data == null) { 19 | return null; 20 | } 21 | if (data.size() == 0) { 22 | return null; 23 | } 24 | 25 | Number baseValue = data.get(0); 26 | if (!(baseValue instanceof Long) && !(baseValue instanceof Integer)) { 27 | throw new IllegalArgumentException(String.format("Delta encoding not supported for type: %s", baseValue.getClass())); 28 | } 29 | 30 | byte[] encoded; 31 | if (baseValue instanceof Long) { 32 | encoded = encodeVarLongSeries(data); 33 | return new Result(METRIC_TYPE.LONG, encoded); 34 | } else { 35 | encoded = encodeIntVarSeries(data); 36 | return new Result(METRIC_TYPE.INTEGER, encoded); 37 | } 38 | } 39 | 40 | private byte[] encodeIntVarSeries(List data) { 41 | return new VarIntArrayEncoderDecoder().encode(data.stream().mapToInt(Number::intValue).toArray()); 42 | } 43 | 44 | private byte[] encodeVarLongSeries(List data) { 45 | return new VarLongArrayEncoderDecoder().encode(data.stream().mapToLong(Number::longValue).toArray()); 46 | } 47 | 48 | public List decode(Result encoded) { 49 | List retVal = new ArrayList<>(); 50 | if (encoded.metric_type == METRIC_TYPE.LONG) { 51 | decodeVarLongSeries(encoded.getEncodedData(), retVal); 52 | } else if (encoded.metric_type == METRIC_TYPE.INTEGER) { 53 | decodeVarIntSeries(encoded.getEncodedData(), retVal); 54 | } 55 | return retVal; 56 | } 57 | 58 | private void decodeVarLongSeries(byte[] encodedData, List retVal) { 59 | retVal.addAll(Arrays.stream(new VarLongArrayEncoderDecoder().decode(encodedData)).boxed().collect(Collectors.toList())); 60 | } 61 | 62 | private void decodeVarIntSeries(byte[] encodedData, List retVal) { 63 | retVal.addAll(Arrays.stream(new VarIntArrayEncoderDecoder().decode(encodedData)).boxed().collect(Collectors.toList())); 64 | } 65 | 66 | enum METRIC_TYPE { 67 | INTEGER, 68 | LONG, 69 | OTHER 70 | } 71 | 72 | public static class Result { 73 | @Getter 74 | private byte[] encodedData; 75 | @Getter 76 | private METRIC_TYPE metric_type = METRIC_TYPE.OTHER; 77 | 78 | private Result(METRIC_TYPE type, byte[] encodedData) { 79 | this.metric_type = type; 80 | this.encodedData = encodedData; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/jmh/java/transformers/VarLongArrayEncoderDecoderBenchmark.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | import org.openjdk.jmh.annotations.Benchmark; 4 | import org.openjdk.jmh.annotations.BenchmarkMode; 5 | import org.openjdk.jmh.annotations.Fork; 6 | import org.openjdk.jmh.annotations.Level; 7 | import org.openjdk.jmh.annotations.Measurement; 8 | import org.openjdk.jmh.annotations.Mode; 9 | import org.openjdk.jmh.annotations.OutputTimeUnit; 10 | import org.openjdk.jmh.annotations.Param; 11 | import org.openjdk.jmh.annotations.Scope; 12 | import org.openjdk.jmh.annotations.Setup; 13 | import org.openjdk.jmh.annotations.State; 14 | import org.openjdk.jmh.annotations.TearDown; 15 | import org.openjdk.jmh.annotations.Warmup; 16 | 17 | import java.util.Random; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | public class VarLongArrayEncoderDecoderBenchmark { 21 | 22 | @State(Scope.Thread) 23 | public static class MyState { 24 | // specify the step size for the benchmarking 25 | @Param({"1000", "10000", "100000"}) 26 | private int sampleSize; 27 | private long[] samples; 28 | private VarLongArrayEncoderDecoder underBenchmark; 29 | private byte[] result; 30 | 31 | @Setup(Level.Trial) 32 | public void setup() { 33 | underBenchmark = new VarLongArrayEncoderDecoder(); 34 | int limit = sampleSize; 35 | samples = new long[limit]; 36 | System.out.println("Generating random data points of size " + limit); 37 | Random r = new Random(); 38 | for (int i = 0; i < limit; i++) { 39 | samples[i] = Math.abs(r.nextInt()); 40 | } 41 | // precompute some encoded data before the benchmarking so that 42 | // decode can also be benchmarked 43 | result = underBenchmark.encode(samples); 44 | } 45 | 46 | @TearDown(Level.Trial) 47 | public void tearDown() { 48 | System.out.println("Cleaning up random samples"); 49 | samples = null; 50 | result = null; 51 | } 52 | 53 | public long[] getSamples() { 54 | return samples; 55 | } 56 | 57 | public VarLongArrayEncoderDecoder getUnderBenchmark() { 58 | return underBenchmark; 59 | } 60 | 61 | public byte[] getEncoded() { 62 | return result; 63 | } 64 | } 65 | 66 | @Benchmark 67 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 68 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 69 | @Fork(value = 3, warmups = 2) 70 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 71 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 72 | public byte[] encode(MyState state) { 73 | return state.getUnderBenchmark().encode(state.getSamples()); 74 | } 75 | 76 | 77 | @Benchmark 78 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 79 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 80 | @Fork(value = 3, warmups = 2) 81 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 82 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 83 | public long[] decode(MyState state) { 84 | return state.getUnderBenchmark().decode(state.getEncoded()); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/jmh/java/transformers/VarIntArrayEncoderDecoderBenchmark.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | import org.openjdk.jmh.annotations.Benchmark; 4 | import org.openjdk.jmh.annotations.BenchmarkMode; 5 | import org.openjdk.jmh.annotations.Fork; 6 | import org.openjdk.jmh.annotations.Level; 7 | import org.openjdk.jmh.annotations.Measurement; 8 | import org.openjdk.jmh.annotations.Mode; 9 | import org.openjdk.jmh.annotations.OutputTimeUnit; 10 | import org.openjdk.jmh.annotations.Param; 11 | import org.openjdk.jmh.annotations.Scope; 12 | import org.openjdk.jmh.annotations.Setup; 13 | import org.openjdk.jmh.annotations.State; 14 | import org.openjdk.jmh.annotations.TearDown; 15 | import org.openjdk.jmh.annotations.Warmup; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.Random; 20 | import java.util.concurrent.TimeUnit; 21 | 22 | public class VarIntArrayEncoderDecoderBenchmark { 23 | 24 | @State(Scope.Thread) 25 | public static class MyState { 26 | // specify the step size for the benchmarking 27 | @Param({"1000", "10000", "100000"}) 28 | private int sampleSize; 29 | private int[] samples; 30 | private VarIntArrayEncoderDecoder underBenchmark; 31 | private byte[] result; 32 | 33 | @Setup(Level.Trial) 34 | public void setup() { 35 | underBenchmark = new VarIntArrayEncoderDecoder(); 36 | int limit = sampleSize; 37 | samples = new int[limit]; 38 | System.out.println("Generating random data points of size " + limit); 39 | Random r = new Random(); 40 | for (int i = 0; i < limit; i++) { 41 | samples[i] = Math.abs(r.nextInt()); 42 | } 43 | // precompute some encoded data before the benchmarking so that 44 | // decode can also be benchmarked 45 | result = underBenchmark.encode(samples); 46 | } 47 | 48 | @TearDown(Level.Trial) 49 | public void tearDown() { 50 | System.out.println("Cleaning up random samples"); 51 | samples = null; 52 | result = null; 53 | } 54 | 55 | public int[] getSamples() { 56 | return samples; 57 | } 58 | 59 | public VarIntArrayEncoderDecoder getUnderBenchmark() { 60 | return underBenchmark; 61 | } 62 | 63 | public byte[] getEncoded() { 64 | return result; 65 | } 66 | } 67 | 68 | @Benchmark 69 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 70 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 71 | @Fork(value = 3, warmups = 2) 72 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 73 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 74 | public byte[] encode(MyState state) { 75 | return state.getUnderBenchmark().encode(state.getSamples()); 76 | } 77 | 78 | 79 | @Benchmark 80 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 81 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 82 | @Fork(value = 3, warmups = 2) 83 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 84 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 85 | public int[] decode(MyState state) { 86 | return state.getUnderBenchmark().decode(state.getEncoded()); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/jmh/java/transformers/DeltaEncoderDecoderBenchmark.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | import org.openjdk.jmh.annotations.Benchmark; 4 | import org.openjdk.jmh.annotations.BenchmarkMode; 5 | import org.openjdk.jmh.annotations.Fork; 6 | import org.openjdk.jmh.annotations.Level; 7 | import org.openjdk.jmh.annotations.Measurement; 8 | import org.openjdk.jmh.annotations.Mode; 9 | import org.openjdk.jmh.annotations.OutputTimeUnit; 10 | import org.openjdk.jmh.annotations.Param; 11 | import org.openjdk.jmh.annotations.Scope; 12 | import org.openjdk.jmh.annotations.Setup; 13 | import org.openjdk.jmh.annotations.State; 14 | import org.openjdk.jmh.annotations.TearDown; 15 | import org.openjdk.jmh.annotations.Warmup; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.Random; 20 | import java.util.concurrent.TimeUnit; 21 | 22 | public class DeltaEncoderDecoderBenchmark { 23 | 24 | @State(Scope.Thread) 25 | public static class MyState { 26 | @Param({"1000", "10000", "100000"}) 27 | private int sampleSize; 28 | private List samples; 29 | private DeltaEncoderDecoder underBenchmark; 30 | private DeltaEncoderDecoder.Result result; 31 | 32 | @Setup(Level.Trial) 33 | public void setup() { 34 | underBenchmark = new DeltaEncoderDecoder(); 35 | samples = new ArrayList<>(); 36 | Random r = new Random(); 37 | int limit = sampleSize; 38 | System.out.println("Generating random data points of size " + limit); 39 | List samples = new ArrayList<>(); 40 | for (int i = 0; i < limit; i++) { 41 | samples.add(r.nextLong()); 42 | } 43 | // precompute some encoded data before the benchmarking so that 44 | // decode can also be benchmarked 45 | result = underBenchmark.encode(samples); 46 | } 47 | 48 | @TearDown(Level.Trial) 49 | public void tearDown() { 50 | System.out.println("Cleaning up random samples"); 51 | samples.clear(); 52 | samples = null; 53 | result = null; 54 | } 55 | 56 | public List getSamples() { 57 | return samples; 58 | } 59 | 60 | public DeltaEncoderDecoder getUnderBenchmark() { 61 | return underBenchmark; 62 | } 63 | 64 | public DeltaEncoderDecoder.Result getEncoded() { 65 | return result; 66 | } 67 | } 68 | 69 | @Benchmark 70 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 71 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 72 | @Fork(value = 3, warmups = 2) 73 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 74 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 75 | public void encode(MyState state) { 76 | state.getUnderBenchmark().encode(state.getSamples()); 77 | } 78 | 79 | 80 | @Benchmark 81 | @BenchmarkMode({Mode.Throughput, Mode.AverageTime}) 82 | @OutputTimeUnit(TimeUnit.MILLISECONDS) 83 | @Fork(value = 3, warmups = 2) 84 | @Warmup(iterations = 5, time = 60, timeUnit = TimeUnit.MILLISECONDS) 85 | @Measurement(iterations = 4, time = 60, timeUnit = TimeUnit.MILLISECONDS) 86 | public void decode(MyState state) { 87 | state.getUnderBenchmark().decode(state.getEncoded()); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/transformers/DeltaEncoderDecoder.java: -------------------------------------------------------------------------------- 1 | package transformers; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import lombok.Getter; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.function.BiFunction; 15 | 16 | /** 17 | * Implements a delta encoding scheme where-in {@link List} containing {@link Number} instances of actual type {@link Integer} 18 | * or {@link Long} are reduced to {@link Integer} deltas. 19 | *
20 | * 21 | * Delta encoding does not result in significant savings when the underlying values are floating point types. The current 22 | * implementation of the encoder decoder does not support floating point data types. 23 | * 24 | */ 25 | public class DeltaEncoderDecoder implements EncoderDecoder, DeltaEncoderDecoder.Result> { 26 | 27 | private static final BiFunction intReducer = (aInt, aInt2) -> aInt2 - aInt; 28 | private static final BiFunction longReducer = (aLong, aLong2) -> Long.valueOf(aLong2 - aLong).intValue(); 29 | 30 | private static final HashMap> classReducerMap = new HashMap<>(); 31 | 32 | static { 33 | classReducerMap.put(Long.class, longReducer); 34 | classReducerMap.put(Integer.class, intReducer); 35 | } 36 | 37 | @Override 38 | public Result encode(List data) { 39 | 40 | // get the underlying data type for the input data 41 | // it is assumed that the ArrayList contains data items 42 | // of homogenous types 43 | if (data == null) { 44 | return null; 45 | } 46 | if (data.size() == 0) { 47 | return null; 48 | } 49 | 50 | Number baseValue = data.get(0); 51 | if (!(baseValue instanceof Long) && !(baseValue instanceof Integer) ) { 52 | throw new IllegalArgumentException(String.format("Delta encoding not supported for type: %s", baseValue.getClass())); 53 | } 54 | 55 | long[] encoded = encodeLongSeries(data); 56 | return new Result<>(baseValue, encoded); 57 | } 58 | 59 | @Override 60 | public List decode(Result encoded) { 61 | Number baseValue = encoded.getBaseValue(); 62 | long[] encodedData = encoded.getEncodedData(); 63 | List retVal = new ArrayList<>(); 64 | retVal.add(baseValue); 65 | for (int i = 1; i < encodedData.length; i++) { 66 | if (baseValue instanceof Long) { 67 | long prevValue = retVal.get(i - 1).longValue(); 68 | retVal.add(prevValue + encodedData[i]); 69 | } else { 70 | int prevValue = retVal.get(i - 1).intValue(); 71 | retVal.add(prevValue + Long.valueOf(encodedData[i]).intValue()); 72 | } 73 | 74 | } 75 | return retVal; 76 | } 77 | 78 | private long[] encodeLongSeries(List data) { 79 | long[] encoded = new long[data.size()]; 80 | long prevValue = 0L; 81 | for (int i = 0; i < data.size(); i++) { 82 | long current = data.get(i).longValue(); 83 | encoded[i] = Long.valueOf(((Number) (data.get(i))).longValue() - prevValue); 84 | prevValue = current; 85 | } 86 | return encoded; 87 | } 88 | 89 | private int[] encodeIntSeries(List data) { 90 | int[] encoded = new int[data.size()]; 91 | int prevValue = 0; 92 | for (int i = 0; i < data.size(); i++) { 93 | int current = data.get(i).intValue(); 94 | encoded[i] = Long.valueOf(((Number)(data.get(i))).longValue() - prevValue).intValue(); 95 | prevValue = current; 96 | } 97 | return encoded; 98 | } 99 | 100 | public static class Result { 101 | @Getter 102 | private final T baseValue; 103 | @Getter 104 | private final long[] encodedData; 105 | 106 | private Result(T baseValue, long[] encodedData) { 107 | this.baseValue = baseValue; 108 | this.encodedData = Arrays.copyOf(encodedData, encodedData.length); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # in-memory-property-aware-temporal-graph 2 | 3 | **in-memory-property-aware-temporal-graph** aims to provide an in-memory graph database that stores nodes and relationships between nodes in a versioned/time-aware manner. 4 | 5 | ### Motivation 6 | 7 | Large datacenters contain tens of thousands of related entities for e.g. a host is a container for potentially hundreds of virtual machine workloads executing on the host. Similarly, a kubernetes service is an abstraction for one or more pod replicas servicing the requests directed to the service. 8 | Also in a data center workloads communicate with each thereby creating an implicit communication graph where the vertices correspond to workloads and the edges correspond to the network traffic between the workloads. Such relations can be naturally modeled as graphs which can then be used for efficient querying based on relationship between entities. 9 | 10 | For large data-center environments, it is often beneficial to maintain the history of the attribute values for various relationships. This helps in-depth troubleshooting, forensics and root cause analysis of problems within the data center. This brings forth a critical requirement of maintaining temporal information with respect to the edges and the vertices of the graph along with their properties. 11 | 12 | While traditional graph databases such as TinkerGraph, Neo4J and JanusGraph expose a robust graph subsystem along with traversal and query facilities such as TinkerPop Gremlin, they also introduce the following complications: 13 | * Varied support for features such as multi-valued attributes, control over id generation etc. For e.g. JanusGraph does not support multi-valued attributes whereas Neo4j does. 14 | * Each of these databases have varied level of support for TinkerPop standard. 15 | * Have a significantly heavy footprint in terms of deployment, management and resource consumption patterns. 16 | * Consuming these graph database services within other applications would require large number of dependencies in terms of database specific librairies and other software requirement 17 | 18 | The above limitations prompted the design pointed to the requirement of a light-weight graph database framework that exposes: 19 | * A simple, compact API interface to consumers to work with time-aware graph entities. 20 | * Rely on the facilties provided by the core Java language as much as possible along with some common third party dependencies whereever required. 21 | * Facilitate consumption of the graph database/storage as a library within the application itself. 22 | 23 | It should be noted that while the motivation for the graph database was primarily problems encountered when troubleshooting datacenters, the graph database implemented here is agnostic of the domain and could be used as a generic library whereever maintaining the history of relationships between entities along with the relationship and entity attributes is required. 24 | 25 | ### Technical details 26 | **in-memory-property-aware-temporal-graph** exposes the following primitives to consumers: 27 | * Property - represents a property (a name-value pair) associated with an edge or a vertex. The values for a property are stored in a time-series aware manner. Hence it is possible to retrieve the values of the property over a time range 28 | * Vertex - represents a traditional graph vertex. Each vertex could have zero or more properties associated with it. Additionally, every vertex could potentially have zero or more edges (incoming or outgoing) associated with itself. The properties and edges associated with a vertex are stored in a time-aware fashion making it possible to retrieve the set of edges or properties for a vertex at a specific point in time or over a time range* 29 | * Edge - represents a traditional edge within a graph. Each edge could have zero or more properties associated with it. The properties of the edge are stored in a time aware fashion, making it possible to retrieve the set of properties associated with an edge either at a specific point in time or over a time range. 30 | 31 | ### Prerequisites 32 | 33 | * jdk11 or above 34 | * gradle 35 | 36 | ### Build & Run 37 | 38 | 1. To Build: ./gradlew clean build 39 | 2. To run the jmh benchmarking: ./gradlew jmh 40 | 41 | 42 | ## Contributing 43 | 44 | The in-memory-property-aware-temporal-graph project team welcomes contributions from the community. Before you start working with in-memory-property-aware-temporal-graph, please 45 | read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be 46 | signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on 47 | as an open-source patch. For more detailed information, refer to [CONTRIBUTING.md](CONTRIBUTING_DCO.md). 48 | -------------------------------------------------------------------------------- /src/main/java/model/TemporalProperty.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import lombok.AllArgsConstructor; 9 | import lombok.Builder; 10 | import lombok.Data; 11 | import lombok.EqualsAndHashCode; 12 | 13 | import java.util.ArrayList; 14 | import java.util.Random; 15 | import java.util.TreeMap; 16 | 17 | /** 18 | * An implementation of the {@link Property}. 19 | * Maintains the property time series within a {@link TreeMap} instance 20 | * in a chronological ascending order. 21 | */ 22 | @Data 23 | @Builder(toBuilder = true) 24 | @EqualsAndHashCode 25 | public class TemporalProperty implements Property { 26 | private final int id; 27 | 28 | @EqualsAndHashCode.Exclude 29 | private final String name; 30 | 31 | @EqualsAndHashCode.Exclude 32 | private final long time; 33 | 34 | @EqualsAndHashCode.Exclude 35 | private long latestTimestamp ; 36 | 37 | @Builder.Default 38 | private final ArrayList values = new ArrayList<>(); 39 | 40 | 41 | @Override 42 | public Object getValueAtTime(long timestamp) { 43 | if (this.time == 0L) { 44 | throw new IllegalStateException("Property create time is required but not set."); 45 | } 46 | return floorValue(timestamp); 47 | } 48 | 49 | @Override 50 | public void setValueAtTime(long timestamp, Object value) { 51 | if (this.time == 0L) { 52 | throw new IllegalStateException("Property create time is required but not set."); 53 | } 54 | if (timestamp < latestTimestamp) { 55 | throw new IllegalArgumentException(String.format("Incoming timestamp: %d for property %s is less " + 56 | "than last known timestamp: %d", timestamp, 57 | name, latestTimestamp)); 58 | } 59 | int timeDifferential = (int)(timestamp - this.time); 60 | TimeStampWithValue tsv = new TimeStampWithValue(timeDifferential, value); 61 | values.add(tsv); 62 | latestTimestamp = timestamp; 63 | } 64 | 65 | @Override 66 | public boolean purgeTimeSeriesUntilTime(final long timestamp) { 67 | int floorIndex = floorIndex(timestamp); 68 | // no values removed from the series 69 | if (-1 == floorIndex) 70 | return false; 71 | values.subList(0, floorIndex+1).clear(); 72 | return true; 73 | } 74 | 75 | private int floorIndex(final long timestamp) { 76 | 77 | int floorEntry = Integer.MIN_VALUE; 78 | int baseline = (int)(timestamp - this.time); 79 | int left = 0; 80 | int right = values.size(); 81 | 82 | // return null in case if the time is less thant the first value 83 | if (baseline < values.get(left).timeDifferential) { 84 | return -1; 85 | } 86 | while (left < (right - 1)) { 87 | int mid = (left + (right)) / 2; 88 | int currentMid = values.get(mid).timeDifferential; 89 | if (currentMid == baseline) { 90 | return mid; 91 | } 92 | if (currentMid > baseline) { 93 | right = mid; 94 | } else { 95 | floorEntry = Math.max(floorEntry, currentMid); 96 | left = mid; 97 | } 98 | } 99 | return left; 100 | } 101 | 102 | private Object floorValue(final long timestamp) { 103 | final int floorIndex = floorIndex(timestamp); 104 | if (-1 == floorIndex) { 105 | return null; 106 | } 107 | return values.get(floorIndex).object; 108 | } 109 | 110 | public static class TemporalPropertyBuilder { 111 | public TemporalProperty build() { 112 | if (id == 0L) { 113 | id = Math.abs(new Random().nextInt()); 114 | } 115 | if (null == name || name.isEmpty()) { 116 | throw new IllegalArgumentException(String.format("Name is null or empty")); 117 | } 118 | if (this.time <= 0L) { 119 | throw new IllegalArgumentException(String.format("Time parameter is invalid")); 120 | } 121 | 122 | return new TemporalProperty(id, name, time, latestTimestamp, (this.values$value == null) ? new ArrayList<>() : 123 | this.values$value); 124 | } 125 | } 126 | 127 | @Data 128 | @AllArgsConstructor 129 | public static final class TimeStampWithValue { 130 | private final int timeDifferential; 131 | private final Object object; 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /src/test/java/model/TemporalPropertyTests.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import org.junit.jupiter.api.Test; 9 | import org.mockito.Mockito; 10 | 11 | import java.util.Random; 12 | 13 | import static org.junit.jupiter.api.Assertions.*; 14 | 15 | class TemporalPropertyTests { 16 | 17 | @Test 18 | void getValueAtTime() { 19 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 20 | final long time = System.currentTimeMillis(); 21 | b.id(Math.abs(new Random().nextInt())); 22 | b.time(time).name("bandwidth"); 23 | 24 | Vertex v = Mockito.mock(Vertex.class); 25 | 26 | TemporalProperty prop = b.build(); 27 | 28 | prop.setValueAtTime(time, 1L); 29 | prop.setValueAtTime(time + 1L, 2L); 30 | prop.setValueAtTime(time + 10L, 50L); 31 | 32 | assertEquals(1L, prop.getValueAtTime(time)); 33 | assertEquals(2L, prop.getValueAtTime(time + 4L)); 34 | assertEquals(50L, prop.getValueAtTime(time + 12L)); 35 | } 36 | 37 | @Test 38 | void getValueAtTimeInitTimeNotSet() { 39 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 40 | final long time = System.currentTimeMillis(); 41 | int id = Math.abs(new Random().nextInt()); 42 | System.out.printf("ID: %s", id); 43 | b.id(id).time(time).name("bandwidth"); 44 | Vertex v = Mockito.mock(Vertex.class); 45 | TemporalProperty prop = b.build(); 46 | 47 | prop.setValueAtTime(time, 1L); 48 | prop.setValueAtTime(time + 1L, 2L); 49 | prop.setValueAtTime(time + 10L, 50L); 50 | 51 | assertEquals(1L, prop.getValueAtTime(time)); 52 | assertEquals(2L, prop.getValueAtTime(time + 4L)); 53 | assertEquals(50L, prop.getValueAtTime(time + 12L)); 54 | } 55 | 56 | void setValueAtTimeInitTimeNotSet() { 57 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 58 | final long time = System.currentTimeMillis(); 59 | b.id(new Random().nextInt()); 60 | 61 | Vertex v = Mockito.mock(Vertex.class); 62 | 63 | TemporalProperty prop = b.build(); 64 | 65 | prop.setValueAtTime(time, 1L); 66 | prop.setValueAtTime(time + 1L, 2L); 67 | prop.setValueAtTime(time + 10L, 50L); 68 | 69 | assertEquals(1L, prop.getValueAtTime(time)); 70 | assertEquals(2L, prop.getValueAtTime(time + 4L)); 71 | assertEquals(50L, prop.getValueAtTime(time + 12L)); 72 | } 73 | 74 | @Test 75 | void setValueAtTimeInitTimeOutOfOrder() { 76 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 77 | final long time = System.currentTimeMillis(); 78 | b.id(new Random().nextInt()).name("Test property").time(time); 79 | 80 | Vertex v = Mockito.mock(Vertex.class); 81 | 82 | TemporalProperty prop = b.build(); 83 | 84 | prop.setValueAtTime(time, 1L); 85 | prop.setValueAtTime(time + 1L, 2L); 86 | prop.setValueAtTime(time + 10L, 50L); 87 | 88 | assertEquals(1L, prop.getValueAtTime(time)); 89 | assertEquals(2L, prop.getValueAtTime(time + 4L)); 90 | assertEquals(50L, prop.getValueAtTime(time + 12L)); 91 | 92 | assertThrows(IllegalArgumentException.class, () -> prop.setValueAtTime(time + 8L, 100L)); 93 | } 94 | 95 | @Test 96 | void purgePropertyValuesTillTime() { 97 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 98 | final long time = System.currentTimeMillis(); 99 | b.id(new Random().nextInt()).name("Test property").time(time); 100 | 101 | TemporalProperty prop = b.build(); 102 | 103 | prop.setValueAtTime(time, 1L); 104 | prop.setValueAtTime(time + 1L, 2L); 105 | prop.setValueAtTime(time + 10L, 50L); 106 | prop.setValueAtTime(time + 14L, 70L); 107 | prop.setValueAtTime(time + 18L, 90L); 108 | prop.setValueAtTime(time + 20L, 101L); 109 | 110 | prop.purgeTimeSeriesUntilTime(time - 1L); 111 | assertEquals(1L, prop.getValueAtTime(time)); 112 | assertEquals(2L, prop.getValueAtTime(time + 4L)); 113 | assertEquals(50L, prop.getValueAtTime(time + 12L)); 114 | 115 | prop.purgeTimeSeriesUntilTime(time + 12L); 116 | assertEquals(null, prop.getValueAtTime(time)); 117 | assertEquals(null, prop.getValueAtTime(time + 11L)); 118 | assertEquals(null, prop.getValueAtTime(time + 13L)); 119 | assertEquals(70L, prop.getValueAtTime(time + 14L)); 120 | assertThrows(IllegalArgumentException.class, () -> prop.setValueAtTime(time + 8L, 100L)); 121 | } 122 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in in-memory-property-aware-temporal-graph project and our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at oss-coc@vmware.com. 63 | All complaints will be reviewed and investigated promptly and fairly. 64 | 65 | All community leaders are obligated to respect the privacy and security of the 66 | reporter of any incident. 67 | 68 | ## Enforcement Guidelines 69 | 70 | Community leaders will follow these Community Impact Guidelines in determining 71 | the consequences for any action they deem in violation of this Code of Conduct: 72 | 73 | ### 1. Correction 74 | 75 | **Community Impact**: Use of inappropriate language or other behavior deemed 76 | unprofessional or unwelcome in the community. 77 | 78 | **Consequence**: A private, written warning from community leaders, providing 79 | clarity around the nature of the violation and an explanation of why the 80 | behavior was inappropriate. A public apology may be requested. 81 | 82 | ### 2. Warning 83 | 84 | **Community Impact**: A violation through a single incident or series 85 | of actions. 86 | 87 | **Consequence**: A warning with consequences for continued behavior. No 88 | interaction with the people involved, including unsolicited interaction with 89 | those enforcing the Code of Conduct, for a specified period of time. This 90 | includes avoiding interactions in community spaces as well as external channels 91 | like social media. Violating these terms may lead to a temporary or 92 | permanent ban. 93 | 94 | ### 3. Temporary Ban 95 | 96 | **Community Impact**: A serious violation of community standards, including 97 | sustained inappropriate behavior. 98 | 99 | **Consequence**: A temporary ban from any sort of interaction or public 100 | communication with the community for a specified period of time. No public or 101 | private interaction with the people involved, including unsolicited interaction 102 | with those enforcing the Code of Conduct, is allowed during this period. 103 | Violating these terms may lead to a permanent ban. 104 | 105 | ### 4. Permanent Ban 106 | 107 | **Community Impact**: Demonstrating a pattern of violation of community 108 | standards, including sustained inappropriate behavior, harassment of an 109 | individual, or aggression toward or disparagement of classes of individuals. 110 | 111 | **Consequence**: A permanent ban from any sort of public interaction within 112 | the community. 113 | 114 | ## Attribution 115 | 116 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 117 | version 2.0, available at 118 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 119 | 120 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 121 | enforcement ladder](https://github.com/mozilla/diversity). 122 | 123 | [homepage]: https://www.contributor-covenant.org 124 | 125 | For answers to common questions about this code of conduct, see the FAQ at 126 | https://www.contributor-covenant.org/faq. Translations are available at 127 | https://www.contributor-covenant.org/translations. 128 | -------------------------------------------------------------------------------- /src/main/java/core/propertystore/PropertyStore.java: -------------------------------------------------------------------------------- 1 | package core.propertystore; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; 9 | import it.unimi.dsi.fastutil.ints.IntSet; 10 | import lombok.AllArgsConstructor; 11 | import lombok.Builder; 12 | import lombok.Data; 13 | import model.Property; 14 | import model.TemporalProperty; 15 | import transformers.MetricsDecoderEncoderHandler; 16 | 17 | import java.util.ArrayList; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | import java.util.function.Consumer; 22 | 23 | public class PropertyStore { 24 | private final static int DEFAULT_NUM_PROPERTIES = 0; 25 | private final Int2ObjectOpenHashMap propertyStore; 26 | 27 | public PropertyStore() { 28 | this(DEFAULT_NUM_PROPERTIES); 29 | } 30 | 31 | public PropertyStore(final int totalExpectedProperties) { 32 | this.propertyStore = new Int2ObjectOpenHashMap<>(totalExpectedProperties); 33 | } 34 | 35 | public void put(int propertyId, Property p) { 36 | TemporalProperty tp = (TemporalProperty)p; 37 | ArrayList valuesWithTime = tp.getValues(); 38 | CompressedProperty.CompressedPropertyBuilder b = CompressedProperty.builder(); 39 | b.id(propertyId); 40 | b.name(new String(p.getName())); 41 | b.time(p.getTime()); 42 | b.latestTimestamp(((TemporalProperty) p).getLatestTimestamp()); 43 | int[] timeDiffs = new int[valuesWithTime.size()]; 44 | // perform delta encoding for integer and long values. 45 | if ((valuesWithTime.get(0).getObject() instanceof Long) || (valuesWithTime.get(0).getObject() instanceof Integer)) { 46 | List toEncode = new ArrayList<>(); 47 | valuesWithTime.stream().forEach(new Consumer() { 48 | @Override 49 | public void accept(TemporalProperty.TimeStampWithValue timeStampWithValue) { 50 | 51 | if (timeStampWithValue.getObject() instanceof Long) { 52 | toEncode.add((Long)timeStampWithValue.getObject()); 53 | } else { 54 | toEncode.add((Integer)timeStampWithValue.getObject()); 55 | } 56 | } 57 | }); 58 | // the resultant object is a result object that contains 59 | // sufficient information to decode the result back. 60 | b.valueSeries(new MetricsDecoderEncoderHandler().encode(toEncode)); 61 | } else { 62 | // non-integer/long values - retain objects as from source. 63 | Object[] values = new Object[valuesWithTime.size()]; 64 | for (int i = 0; i < valuesWithTime.size(); i++) { 65 | values[i] = valuesWithTime.get(i); 66 | } 67 | b.valueSeries(values); 68 | } 69 | for (int i = 0; i < valuesWithTime.size(); i++) { 70 | timeDiffs[i] = valuesWithTime.get(i).getTimeDifferential(); 71 | } 72 | b.timeDiffs(timeDiffs); 73 | propertyStore.put(propertyId, b.build()); 74 | } 75 | 76 | public Property get(final int propertyId) { 77 | if (!propertyStore.containsKey(propertyId)) { 78 | return null; 79 | } 80 | CompressedProperty cp = (propertyStore.get(propertyId)); 81 | TemporalProperty.TemporalPropertyBuilder b = TemporalProperty.builder(); 82 | b.id(propertyId); 83 | b.name(cp.name); 84 | b.time(cp.time); 85 | b.latestTimestamp(cp.latestTimestamp); 86 | ArrayList valuesWithTime = new ArrayList<>(); 87 | // the data was encoded using delta encoding 88 | if (cp.valueSeries instanceof MetricsDecoderEncoderHandler.Result) { 89 | MetricsDecoderEncoderHandler.Result res = (MetricsDecoderEncoderHandler.Result)(cp.valueSeries); 90 | List decoded = new MetricsDecoderEncoderHandler().decode(res); 91 | 92 | for (int i = 0; i < cp.timeDiffs.length; i++) { 93 | TemporalProperty.TimeStampWithValue tsv = new TemporalProperty.TimeStampWithValue(cp.timeDiffs[i], decoded.get(i)); 94 | valuesWithTime.add(tsv); 95 | } 96 | } else { 97 | for (int i = 0; i < cp.timeDiffs.length; i++) { 98 | Object[] values = (Object[])(cp.valueSeries); 99 | valuesWithTime.add((TemporalProperty.TimeStampWithValue)values[i]); 100 | } 101 | } 102 | b.values(valuesWithTime); 103 | 104 | return b.build(); 105 | } 106 | 107 | public boolean containsKey(int propertyId) { 108 | return propertyStore.containsKey(propertyId); 109 | } 110 | 111 | public Map getProperties() { 112 | Map properties = new HashMap<>(); 113 | for (Map.Entry entry : propertyStore.entrySet()) { 114 | Property tp = get(entry.getKey()); 115 | properties.put(entry.getKey(), tp); 116 | } 117 | return properties; 118 | } 119 | 120 | public boolean trim() { 121 | return this.propertyStore.trim(); 122 | } 123 | 124 | public void purgePropertiesTillTime(final IntSet propertyIds, long timestamp) { 125 | for (int propId : propertyIds) { 126 | final Property p = this.get(propId); 127 | if (null == p) { 128 | continue; 129 | } 130 | if (p.purgeTimeSeriesUntilTime(timestamp)) { 131 | this.put(propId, p); 132 | } 133 | } 134 | } 135 | 136 | @Data 137 | @AllArgsConstructor 138 | @Builder(toBuilder = true) 139 | private static final class CompressedProperty { 140 | private final int id; 141 | private final String name; 142 | private final long time; 143 | private final int[] timeDiffs; 144 | private final Object valueSeries; 145 | private final long latestTimestamp; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MSYS* | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /src/test/java/core/InMemoryGraphStoragePerfTest.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import model.Property; 9 | import model.TemporalProperty; 10 | import org.junit.jupiter.api.BeforeEach; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.time.ZonedDateTime; 14 | import java.util.ArrayList; 15 | import java.util.Collections; 16 | import java.util.HashMap; 17 | import java.util.HashSet; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.Objects; 21 | import java.util.Random; 22 | import java.util.Set; 23 | 24 | class InMemoryGraphStoragePerfTest { 25 | 26 | private static final Set VERTEX_PROPETIES = new HashSet<>(); 27 | private static final Set EDGE_PROPERTIES = new HashSet<>(); 28 | private static final Map EDGE_PROPERTIES_WITH_TYPE = new HashMap<>(); 29 | 30 | // test parameters are driven using environment variables. 31 | private static final String ENV_PROPERTY_RESOLUTION_MINS = "resolutionMins"; 32 | private static final Integer DEFAULT_RESOULTION_MINS = 5; 33 | 34 | // number of vertices to be created 35 | private static final Integer[] NUM_VERTICES = new Integer[] {73000}; 36 | private static final Integer[] NUM_EDGES = new Integer[] {20000000}; 37 | 38 | // the percentage of vertices that talk to each other. 39 | private static final Integer EDGE_PCT = 75; 40 | 41 | static { 42 | VERTEX_PROPETIES.add("Name"); 43 | VERTEX_PROPETIES.add("IPAddress"); 44 | EDGE_PROPERTIES.add("PacketCount"); 45 | EDGE_PROPERTIES.add("Bandwidth"); 46 | EDGE_PROPERTIES.add("Volume"); 47 | EDGE_PROPERTIES.add("SessionCount"); 48 | 49 | // initialize the edge properties with type hashmap 50 | /* 51 | EDGE_PROPERTIES_WITH_TYPE.put("PacketCount", Long.class); 52 | 53 | EDGE_PROPERTIES_WITH_TYPE.put("Bandwidth", Long.class); 54 | EDGE_PROPERTIES_WITH_TYPE.put("Volume", Long.class); 55 | EDGE_PROPERTIES_WITH_TYPE.put("SessionCount", Long.class); 56 | 57 | */ 58 | EDGE_PROPERTIES_WITH_TYPE.put("SessionCount", Integer.class); 59 | 60 | 61 | 62 | } 63 | 64 | private Random random; 65 | private Integer resolution; 66 | private TemporalGraph underTest; 67 | private long initTs; 68 | 69 | @BeforeEach 70 | public void init() { 71 | this.random = new Random(); 72 | String resFromProperty = System.getProperty(ENV_PROPERTY_RESOLUTION_MINS); 73 | this.resolution = resFromProperty != null ? Integer.parseInt(resFromProperty) : DEFAULT_RESOULTION_MINS; 74 | this.initTs = System.currentTimeMillis(); 75 | this.underTest = new TemporalGraph(initTs); 76 | } 77 | 78 | 79 | // @Test 80 | public void memoryConsumption() throws InterruptedException { 81 | populateGraph(); 82 | System.out.println("Ready for heap dump process ..."); 83 | Thread.sleep(240000); 84 | System.out.println("done"); 85 | } 86 | 87 | private void populateGraph() { 88 | List generatedVertexIds = addVertices(); 89 | long time; 90 | 91 | List generatedEdgeIds = new ArrayList<>(); 92 | 93 | long startTime = System.currentTimeMillis(); 94 | 95 | for (int i = 0; i < NUM_EDGES.length; i++) { 96 | int numEdges = NUM_EDGES[i]; 97 | time = System.currentTimeMillis(); 98 | // pick the vertex and generate edges to every other vertex 99 | for (int j = 0; j < numEdges; j++) { 100 | int srcVertexIndex = random.nextInt(generatedVertexIds.size()); 101 | int destVertexIndex = random.nextInt(generatedVertexIds.size()); 102 | if (srcVertexIndex == destVertexIndex) { 103 | continue; 104 | } 105 | Set properties = new HashSet<>(); 106 | /* 107 | for (Map.Entry e : EDGE_PROPERTIES_WITH_TYPE.entrySet()) { 108 | Property p = createPropertyTimeSeries(e.getValue(), e.getKey(), time); 109 | properties.add(p); 110 | }*/ 111 | final long timeBeforeEdgeAdd = System.currentTimeMillis(); 112 | int edgeId = underTest.addEdge(srcVertexIndex, destVertexIndex, properties, time); 113 | final long timeAfterEdgeAdd = System.currentTimeMillis(); 114 | generatedEdgeIds.add(edgeId); 115 | // System.out.println("TotalEdges : " + generatedEdgeIds.size() + "; Single edge add time : " + (timeAfterEdgeAdd - timeBeforeEdgeAdd) + "; Total Elapsed time : " + (timeAfterEdgeAdd - startTime)); 116 | } 117 | } 118 | long completionTime = System.currentTimeMillis(); 119 | 120 | System.out.println(String.format("Num Vertices : %d, Num Edges : %d, Time elapsed (ms) : %d", 121 | generatedVertexIds.size(), 122 | generatedEdgeIds.size(), (completionTime - startTime))); 123 | } 124 | 125 | 126 | 127 | private List addVertices() { 128 | List generatedVertexIds = new ArrayList<>(); 129 | long time = System.currentTimeMillis(); 130 | for (int i = 0; i < NUM_VERTICES.length; i++) { 131 | final int numVertex = NUM_VERTICES[i]; 132 | for (int j = 1; j <= numVertex; j++) { 133 | 134 | Property p = createProperty("Name", time); 135 | p.setValueAtTime(time, String.format("VirtualMachine_%d", j)); 136 | 137 | Property p1 = createProperty("IPAddress", time); 138 | p1.setValueAtTime(time, String.format("IpAddress_%d", j)); 139 | int vertexId = underTest.addVertex(Collections.singleton(p), time); 140 | generatedVertexIds.add(vertexId); 141 | } 142 | } 143 | long completionTime = System.currentTimeMillis(); 144 | System.out.printf("Num vertices : %d, Time elapsed (ms): %d\n", generatedVertexIds.size(), 145 | (completionTime - time)); 146 | return generatedVertexIds; 147 | } 148 | 149 | @Test 150 | public void floorValue() { 151 | long[] values = new long[11]; 152 | for (int i = 0; i <= 100; i = i + 10) { 153 | values[i/10] = i; 154 | } 155 | 156 | long floorEntry = Integer.MIN_VALUE; 157 | long baseline = 40L; 158 | int left = 0; 159 | int right = values.length; 160 | 161 | while (left < (right - 1)) { 162 | int mid = (left + (right)) / 2; 163 | if (values[mid] == baseline) { 164 | break; 165 | } 166 | if (values[mid] > baseline) { 167 | right = mid; 168 | } else { 169 | floorEntry = Math.max(floorEntry, values[mid]); 170 | left = mid; 171 | } 172 | } 173 | System.out.println("Done"); 174 | } 175 | 176 | @Test 177 | public void currentTimeLessThanMaxInt() { 178 | ZonedDateTime today = ZonedDateTime.parse("2022-02-28T00:00:00+05:30"); 179 | ZonedDateTime history = ZonedDateTime.parse("2022-02-21T00:00:00+05:30"); 180 | long diff = (today.toInstant().toEpochMilli() - history.toInstant().toEpochMilli()); 181 | long intRep = (long) Long.valueOf(diff).intValue(); 182 | System.out.println("Precision loss : " + (diff != intRep)); 183 | System.out.println("Assumption = current time is less than Int MAX VALUE : " + (diff < System.currentTimeMillis())); 184 | } 185 | 186 | private Property createPropertyTimeSeries(final Class valueType, 187 | final String name, 188 | final long startTime) { 189 | Property p = createProperty(name, startTime); 190 | if (valueType == Long.class) { 191 | LongPropertyTimeSeriesGenerator generator = new LongPropertyTimeSeriesGenerator(); 192 | Map valueTimeSeries = generator.getValueTimeSeries(this.resolution, startTime); 193 | for (Map.Entry value : valueTimeSeries.entrySet()) { 194 | p.setValueAtTime(value.getKey(), value.getValue()); 195 | } 196 | return p; 197 | } 198 | if (valueType == Integer.class) { 199 | IntegerPropertyTimeSeriesGenerator generator = new IntegerPropertyTimeSeriesGenerator(); 200 | Map valueTimeSeries = generator.getValueTimeSeries(this.resolution, startTime); 201 | for (Map.Entry value : valueTimeSeries.entrySet()) { 202 | p.setValueAtTime(value.getKey(), value.getValue()); 203 | } 204 | return p; 205 | } 206 | throw new IllegalArgumentException(String.format("Property of type : %s is not supported", 207 | valueType.getName())); 208 | } 209 | 210 | 211 | 212 | private Property createProperty(final String name, 213 | final long initTime) { 214 | return TemporalProperty.builder().name(name).time(initTime).build(); 215 | } 216 | 217 | private long getPropertyId(final long entityId, final long propertyId) { 218 | return Objects.hash(entityId, propertyId); 219 | } 220 | 221 | private static final class Value { 222 | long timestamp; 223 | long value; 224 | } 225 | } -------------------------------------------------------------------------------- /src/main/java/model/Graph.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import exceptions.PropertyNotFoundException; 9 | 10 | import java.util.Collection; 11 | import java.util.Iterator; 12 | import java.util.Map; 13 | 14 | /** 15 | * Contract representing a time-aware graph. Clients are expected to interact with implementations 16 | * of this contract to create vertices and edges with time-awareness
17 | * The values returned by the addXXX methods represent the internal graph identifiers assigned to the vertex 18 | * and edge entity. It is the responsibility of the clients consuming the graph to maintain a mapping between 19 | * the returned identifier and the identifier for the entity as present within their model. Such an enforcement 20 | * ensures simplicity of the graph interface without having the client to maintain various representations of the 21 | * graph where the domain entities are the same but domain entity identifiers could be different based on the 22 | * use case. 23 | * Generally, it is recommended to embed the domain identifier of the entity as a property within the vertex or 24 | * the edge itself so that it is readily available in the desired format.
25 | * The getVertices and getEdges (including the direction aware methods) methods return an iterator instead of the 26 | * entire vertex or edge identifier set. 27 | * This allows the clients to consume the vertex or edge entities in an on-demand manner while also alleviating 28 | * memory 29 | * consumption concerns when the graph has a high cardinality of vertices or edges.
30 | */ 31 | public interface Graph { 32 | 33 | /** 34 | * add a vertex to the graph with the specified {@link Property} collection and time. 35 | * 36 | * @param properties the properties associated with the vertex 37 | * @param timestamp the timestamp at which the vertex would be inserted in the graph 38 | * @return an identifier for the vertex as present within the graph storage layer 39 | */ 40 | int addVertex(Collection properties, long timestamp); 41 | 42 | /** 43 | * add an edge to the graph between source and destination vertices with the specified properties and time 44 | * 45 | * @param srcVertexId the identifier of the source vertex as known to the graph storage layer 46 | * @param destVertexId the identifier of the destination vertex as known to the graph storage layer 47 | * @param properties the properties associated with the edge 48 | * @param timestamp the timestamp at which the edge would be inserted in the graph 49 | * @return an identifier for the edge as present within the graph storage layer 50 | */ 51 | int addEdge(int srcVertexId, int destVertexId, Collection properties, long timestamp); 52 | 53 | /** 54 | * get the vertices present in the graph at a specified time. 55 | * 56 | * @param timestamp the timesatmp at which the vertex information needs to be retrieved 57 | * @return An {@link Iterator} that can be used to iterate through the vertex ids as known to the graph 58 | * storage layer 59 | */ 60 | Iterator getVerticesAtTime(long timestamp); 61 | 62 | /** 63 | * get the edges present in the graph between a source vertex and a destination vertex at a specified time 64 | * 65 | * @param srcVertexId the id of the source vertex as known to the graph storage layer 66 | * @param destVertexId the id of the destination vertex as known to the graph storage layer 67 | * @param timestamp the timestamp at which the edges need to be retrieved 68 | * @return An {@link Iterator} that can be used to iterate through the edge ids as known to the graph 69 | * storage layer 70 | */ 71 | Iterator getEdgesAtTime(int srcVertexId, int destVertexId, long timestamp); 72 | 73 | /** 74 | * get the outgoing edges present in the graph for a given source vertex at a specified 75 | * time 76 | * 77 | * @param vertexId the id of the source vertex as known to the graph storage layer 78 | * @param timestamp the timestamp at which the edges need to be retrieved 79 | * @return An {@link Iterator} that can be used to iterate through the edge ids as known to the graph 80 | * storage layer 81 | */ 82 | Iterator getOutEdgesAtTime(int vertexId, long timestamp); 83 | 84 | /** 85 | * get the outgoing edges present in the graph for a given source vertex at a specified 86 | * time 87 | * 88 | * @param vertexId the id of the destination vertex as known to the graph storage layer 89 | * @param timestamp the timestamp at which the edges need to be retrieved 90 | * @return An {@link Iterator} that can be used to iterate through the edge ids as known to the graph 91 | * storage layer 92 | */ 93 | Iterator getInEdgesAtTime(int vertexId, long timestamp); 94 | 95 | /** 96 | * get the properties associated with the vertex at the specified timestamp 97 | * 98 | * @param vertexId the identifier of the vertex as known to the graph storage layer 99 | * @param timestamp the timestamp at which the properties need to be retrieved 100 | * @return a {@link Collection} of {@link Property} associated with the vertex at the specified time 101 | */ 102 | Collection getVertexPropertiesAtTime(int vertexId, long timestamp); 103 | 104 | /** 105 | * get the properties associated with the vertex at the specified timestamp 106 | * 107 | * @param vertexId the identifier of the vertex as known to the graph storage layer 108 | * @param propertyName the name of the property for which the value needs to be retrieved. 109 | * @param timestamp the timestamp at which the properties need to be retrieved 110 | * @return a {@link Collection} of {@link Property} associated with the vertex at the specified time 111 | */ 112 | TimestampedPropertyValue getVertexPropertyAtTime(int vertexId, String propertyName, long timestamp); 113 | 114 | /** 115 | * get the properties associated with the edge at the specified timestamp 116 | * 117 | * @param edgeId the identifier of the vertex as known to the graph storage layer 118 | * @param timestamp the timestamp at which the properties need to be retrieved 119 | * @return a {@link Collection} of {@link Property} associated with the edge at the specified time 120 | */ 121 | Collection getEdgePropertiesAtTime(int edgeId, long timestamp); 122 | 123 | /** 124 | * get the properties associated with the edge at the specified timestamp 125 | * 126 | * @param edgeId the identifier of the vertex as known to the graph storage layer 127 | * @param propertyName the name of the property for which the value needs to be retrieved. 128 | * @param timestamp the timestamp at which the properties need to be retrieved 129 | * @return a {@link Collection} of {@link Property} associated with the edge at the specified time 130 | */ 131 | TimestampedPropertyValue getEdgePropertyAtTime(int edgeId, String propertyName, long timestamp); 132 | 133 | /** 134 | * get the properties associated with the edge at the specified timestamp 135 | * 136 | * @param srcVertexId the identifier of the source vertex as known to the graph storage layer 137 | * @param destVertexId the identifier of the destination vertex as known to the graph storage layer 138 | * @param timestamp the timestamp at which the properties need to be retrieved 139 | * @return a {@link Map} of {@link Collection} of {@link Property} associated with the corresponding edge 140 | * id at the specified time. 141 | */ 142 | Map> getEdgePropertiesAtTime(int srcVertexId, int destVertexId, long timestamp); 143 | 144 | /** 145 | * get the property associated with the edge at the specified timestamp 146 | * 147 | * @param srcVertexId the identifier of the source vertex as known to the graph storage layer 148 | * @param destVertexId the identifier of the destination vertex as known to the graph storage layer 149 | * @param propertyName the name of the property which needs to be retrieved 150 | * @param timestamp the timestamp at which the properties need to be retrieved 151 | * @return a {@link Map} of {@link Property} associated with the corresponding edge 152 | * id at the specified time. 153 | */ 154 | Map getEdgePropertyAtTime(int srcVertexId, int destVertexId, 155 | String propertyName, long timestamp); 156 | 157 | /** 158 | * add the new value at the new timestamp to the existing property of vertex. 159 | * 160 | * @param vertexId the identifier of the vertex as known to the graph storage layer 161 | * @param propertyName the property name for which the new value needs to be added 162 | * @param value property value that needs to be added 163 | * @param timestamp the timestamp at which the property value needs to be added 164 | */ 165 | void addVertexProperty(int vertexId, String propertyName, Object value, long timestamp) throws PropertyNotFoundException; 166 | 167 | /** 168 | * add the new value at the new timestamp to the existing property of edge. 169 | * 170 | * @param edgeId the identifier of the edge as known to the graph storage layer 171 | * @param propertyName the property name for which the new value needs to be added 172 | * @param value property value that needs to be added 173 | * @param timestamp the timestamp at which the property value needs to be added 174 | */ 175 | void addEdgeProperty(int edgeId, String propertyName, Object value, long timestamp) throws PropertyNotFoundException; 176 | 177 | /** 178 | * get all the edges at a time. 179 | * 180 | * @param timestamp the timestamp at which the edges needs to be returned. 181 | */ 182 | Iterator getAllEdgesAtTime(long timestamp); 183 | 184 | /** 185 | * Purge the graph till the timestamp provided as a parameter. 186 | * This would compute intensive method and would be used if required only. 187 | * @param timestamp the timestamp till which graph needs to be purged. 188 | */ 189 | void purgeAtTime(long timestamp); 190 | } 191 | -------------------------------------------------------------------------------- /src/test/java/core/TemporalGraphTests.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import com.google.common.collect.Iterators; 9 | import com.google.common.collect.Lists; 10 | import com.google.common.collect.Sets; 11 | import exceptions.PropertyNotFoundException; 12 | import model.TemporalProperty; 13 | import model.TimestampedPropertyValue; 14 | import org.junit.jupiter.api.BeforeEach; 15 | import org.junit.jupiter.api.Test; 16 | 17 | import java.util.Arrays; 18 | import java.util.Collection; 19 | import java.util.HashSet; 20 | import java.util.Iterator; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Random; 24 | import java.util.Set; 25 | import java.util.concurrent.TimeUnit; 26 | import static org.junit.jupiter.api.Assertions.*; 27 | 28 | public class TemporalGraphTests { 29 | private TemporalGraph underTest; 30 | private Random random; 31 | private static final long initTime = System.currentTimeMillis(); 32 | private static final String property1 = "Bandwidth"; 33 | private static final String property2 = "cpuUsage"; 34 | private static final String property3 = "packetCount"; 35 | 36 | @BeforeEach 37 | void init() { 38 | underTest = new TemporalGraph(initTime); 39 | random = new Random(); 40 | } 41 | 42 | @Test 43 | void testAddProperty() { 44 | int id = Math.abs(random.nextInt()); 45 | long nowTs = initTime + TimeUnit.MINUTES.toMillis(5); 46 | // out of order property addition 47 | // no name present exception 48 | // no time present exception 49 | { 50 | assertThrows(IllegalArgumentException.class, () -> TemporalProperty.builder().id(id).time(initTime).build()); 51 | assertThrows(IllegalArgumentException.class, () -> TemporalProperty.builder().name(property1).id(id).build()); 52 | TemporalProperty prop1 = TemporalProperty.builder().name(property1).id(id).time(initTime).build(); 53 | prop1.setValueAtTime(nowTs, 100L); 54 | prop1.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 100L); 55 | assertThrows(IllegalArgumentException.class, () -> prop1.setValueAtTime(nowTs + 10 * TimeUnit.MINUTES.toMillis(5), 100L)); 56 | } 57 | // getting property at a time 58 | { 59 | TemporalProperty prop1 = TemporalProperty.builder().name(property1).id(id).time(initTime).build(); 60 | prop1.setValueAtTime(nowTs, 100L); 61 | prop1.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 300L); 62 | prop1.setValueAtTime(nowTs + 30 * TimeUnit.MINUTES.toMillis(5), 200L); 63 | long val = (long) prop1.getValueAtTime(nowTs + 10 * TimeUnit.MINUTES.toMillis(5)); 64 | assertEquals(100, val); 65 | val = (long) prop1.getValueAtTime(nowTs); 66 | assertEquals(100, val); 67 | assertNull(prop1.getValueAtTime(nowTs - 1L)); 68 | System.out.println("val: "+val); 69 | val = (long) prop1.getValueAtTime(nowTs + 31 * TimeUnit.MINUTES.toMillis(5)); 70 | assertEquals(200, val); 71 | val = (long) prop1.getValueAtTime(nowTs + 16 * TimeUnit.MINUTES.toMillis(5)); 72 | assertEquals(300, val); 73 | } 74 | } 75 | 76 | @Test 77 | void testAddVertex() throws PropertyNotFoundException { 78 | int id = Math.abs(random.nextInt()); 79 | long nowTs = initTime + TimeUnit.MINUTES.toMillis(5); 80 | //test all Exceptions 81 | { 82 | TemporalProperty prop1 = TemporalProperty.builder().name(property1).id(id).time(initTime).build(); 83 | assertThrows(IllegalArgumentException.class, () -> underTest.addVertex(Sets.newHashSet(prop1), nowTs)); 84 | } 85 | //add vertex 86 | { 87 | int idProp1 = getRandomId(); 88 | TemporalProperty prop1 = TemporalProperty.builder().name(property1).id(idProp1).time(nowTs).build(); 89 | prop1.setValueAtTime(nowTs + 5, 100L); 90 | prop1.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 300L); 91 | prop1.setValueAtTime(nowTs + 30 * TimeUnit.MINUTES.toMillis(5), 200L); 92 | 93 | int idProp2 = getRandomId(); 94 | TemporalProperty prop2 = TemporalProperty.builder().name(property2).id(idProp2).time(nowTs).build(); 95 | prop2.setValueAtTime(nowTs + 10, 600L); 96 | prop2.setValueAtTime(nowTs + 20 * TimeUnit.MINUTES.toMillis(5), 700L); 97 | prop2.setValueAtTime(nowTs + 40 * TimeUnit.MINUTES.toMillis(5), 900L); 98 | int vertexId = underTest.addVertex(Sets.newHashSet(prop1, prop2), nowTs); 99 | assertTrue(vertexId >= 0); 100 | Iterator vertices = underTest.getVerticesAtTime(nowTs); 101 | assertNotNull(vertices); 102 | assertTrue(vertices.hasNext()); 103 | assertEquals(2, Iterators.size(vertices)); 104 | vertices = underTest.getVerticesAtTime(nowTs - 10L); 105 | assertNotNull(vertices); 106 | assertEquals(0, Iterators.size(vertices)); 107 | 108 | int idProp3 = getRandomId(); 109 | TemporalProperty prop3 = 110 | TemporalProperty.builder().name(property1).id(idProp3).time(nowTs + TimeUnit.MINUTES.toMillis(5)).build(); 111 | prop3.setValueAtTime(nowTs + 10 * TimeUnit.MINUTES.toMillis(5), 400L); 112 | prop3.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 600L); 113 | prop3.setValueAtTime(nowTs + 30 * TimeUnit.MINUTES.toMillis(5), 200L); 114 | 115 | int idProp4 = getRandomId(); 116 | TemporalProperty prop4 = 117 | TemporalProperty.builder().name(property2).id(idProp4).time(nowTs + TimeUnit.MINUTES.toMillis(10)).build(); 118 | prop4.setValueAtTime(nowTs + 4 * TimeUnit.MINUTES.toMillis(5), 400L); 119 | prop4.setValueAtTime(nowTs + 8 * TimeUnit.MINUTES.toMillis(5), 600L); 120 | prop4.setValueAtTime(nowTs + 12 * TimeUnit.MINUTES.toMillis(5), 200L); 121 | 122 | int vertexId1 = underTest.addVertex(Sets.newHashSet(prop3, prop4), nowTs + 5); 123 | assertTrue(vertexId1 >= 0); 124 | vertices = underTest.getVerticesAtTime(nowTs); 125 | assertNotNull(vertices); 126 | assertEquals(2, Iterators.size(vertices)); 127 | vertices = underTest.getVerticesAtTime(nowTs + 6); 128 | assertNotNull(vertices); 129 | assertEquals(3, Iterators.size(vertices)); 130 | vertices = underTest.getVerticesAtTime(nowTs - 5); 131 | assertNotNull(vertices); 132 | assertEquals(0, Iterators.size(vertices)); 133 | Set properties = (Set) underTest.getVertexPropertiesAtTime(vertexId, nowTs + 10); 134 | checkPropertyValues(properties, 2, 100L, 600L); 135 | properties = (Set) underTest.getVertexPropertiesAtTime(vertexId, nowTs + 1); 136 | assertNotNull(properties); 137 | assertEquals(0, properties.size()); 138 | properties = (Set) underTest.getVertexPropertiesAtTime(vertexId, nowTs + 50 * TimeUnit.MINUTES.toMillis(5)); 139 | assertNotNull(properties); 140 | assertEquals(2, properties.size()); 141 | TimestampedPropertyValue p = underTest.getVertexPropertyAtTime(vertexId1, property1, nowTs + 11 * TimeUnit.MINUTES.toMillis(5)); 142 | assertNotNull(p); 143 | assertEquals(400L, p.getValue()); 144 | p = underTest.getVertexPropertyAtTime(vertexId1, property1, nowTs + 9 * TimeUnit.MINUTES.toMillis(5)); 145 | assertNull(p); 146 | p = underTest.getVertexPropertyAtTime(vertexId1, property1+"1", nowTs + 9 * TimeUnit.MINUTES.toMillis(5)); 147 | assertNull(p); 148 | assertThrows(PropertyNotFoundException.class, () -> underTest.addVertexProperty(vertexId1, property1+"1", 100L, nowTs)); 149 | assertThrows(IllegalArgumentException.class, () -> underTest.addVertexProperty(vertexId1, property1, 100L, nowTs + 9 * TimeUnit.MINUTES.toMillis(5))); 150 | underTest.addVertexProperty(vertexId1, property1, 1000L, nowTs + 31 * TimeUnit.MINUTES.toMillis(5)); 151 | p = underTest.getVertexPropertyAtTime(vertexId1, property1, nowTs + 32 * TimeUnit.MINUTES.toMillis(5)); 152 | assertNotNull(p); 153 | assertEquals(1000L, p.getValue()); 154 | } 155 | } 156 | 157 | @Test 158 | void testAddEdge() throws PropertyNotFoundException { 159 | long nowTs = initTime + TimeUnit.MINUTES.toMillis(5); 160 | 161 | int idProp1 = getRandomId(); 162 | TemporalProperty prop1 = TemporalProperty.builder().name(property1).id(idProp1).time(nowTs).build(); 163 | prop1.setValueAtTime(nowTs + 5, 100L); 164 | prop1.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 300L); 165 | prop1.setValueAtTime(nowTs + 30 * TimeUnit.MINUTES.toMillis(5), 200L); 166 | 167 | int idProp2 = getRandomId(); 168 | TemporalProperty prop2 = TemporalProperty.builder().name(property2).id(idProp2).time(nowTs).build(); 169 | prop2.setValueAtTime(nowTs + 10, 600L); 170 | prop2.setValueAtTime(nowTs + 20 * TimeUnit.MINUTES.toMillis(5), 700L); 171 | prop2.setValueAtTime(nowTs + 40 * TimeUnit.MINUTES.toMillis(5), 900L); 172 | 173 | int idProp3 = getRandomId(); 174 | TemporalProperty prop3 = 175 | TemporalProperty.builder().name(property1).id(idProp3).time(nowTs + TimeUnit.MINUTES.toMillis(5)).build(); 176 | prop3.setValueAtTime(nowTs + 10 * TimeUnit.MINUTES.toMillis(5), 400L); 177 | prop3.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 600L); 178 | prop3.setValueAtTime(nowTs + 30 * TimeUnit.MINUTES.toMillis(5), 200L); 179 | 180 | int idProp4 = getRandomId(); 181 | TemporalProperty prop4 = 182 | TemporalProperty.builder().name(property2).id(idProp4).time(nowTs + TimeUnit.MINUTES.toMillis(10)).build(); 183 | prop4.setValueAtTime(nowTs + 4 * TimeUnit.MINUTES.toMillis(5), 400L); 184 | prop4.setValueAtTime(nowTs + 8 * TimeUnit.MINUTES.toMillis(5), 600L); 185 | prop4.setValueAtTime(nowTs + 12 * TimeUnit.MINUTES.toMillis(5), 200L); 186 | 187 | int idPropEdge = getRandomId(); 188 | TemporalProperty propEdge = 189 | TemporalProperty.builder().name(property2).id(idPropEdge).time(nowTs + TimeUnit.MINUTES.toMillis(10)).build(); 190 | propEdge.setValueAtTime(nowTs + 3 * TimeUnit.MINUTES.toMillis(5), 400L); 191 | propEdge.setValueAtTime(nowTs + 6 * TimeUnit.MINUTES.toMillis(5), 600L); 192 | propEdge.setValueAtTime(nowTs + 11 * TimeUnit.MINUTES.toMillis(5), 200L); 193 | 194 | int idPropEdge1 = getRandomId(); 195 | TemporalProperty propEdge1 = 196 | TemporalProperty.builder().name(property2).id(idPropEdge1).time(nowTs + 2 * TimeUnit.MINUTES.toMillis(5)).build(); 197 | propEdge1.setValueAtTime(nowTs + 4 * TimeUnit.MINUTES.toMillis(5), 200L); 198 | propEdge1.setValueAtTime(nowTs + 6 * TimeUnit.MINUTES.toMillis(5), 400L); 199 | propEdge1.setValueAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5), 800L); 200 | // test exception / edge addition / getting an edge 201 | { 202 | int vertexId1 = underTest.addVertex(Sets.newHashSet(prop1, prop2), nowTs); 203 | int vertexId2 = underTest.addVertex(Sets.newHashSet(prop3, prop4), nowTs + 5); 204 | // null properties 205 | assertDoesNotThrow(() -> underTest.addEdge(vertexId1, vertexId2, null, nowTs)); 206 | // Edge Addition time > property addition time 207 | assertThrows(IllegalArgumentException.class, () -> underTest.addEdge(vertexId1, vertexId2, Sets.newHashSet(propEdge), nowTs + TimeUnit.MINUTES.toMillis(11))); 208 | int edgeId = underTest.addEdge(vertexId1, vertexId2, Sets.newHashSet(propEdge), nowTs); 209 | assertTrue(edgeId >= 0); 210 | Iterator edges = underTest.getEdgesAtTime(vertexId1, vertexId2, nowTs - 1); 211 | assertNotNull(edges); 212 | assertEquals(0, Iterators.size(edges)); 213 | edges = underTest.getEdgesAtTime(vertexId2, vertexId1, nowTs); 214 | assertNotNull(edges); 215 | assertEquals(0, Iterators.size(edges)); 216 | edges = underTest.getEdgesAtTime(vertexId1, vertexId2, nowTs); 217 | assertNotNull(edges); 218 | List edgeList = Lists.newArrayList(edges); 219 | assertEquals(2, edgeList.size()); 220 | assertTrue(new HashSet(edgeList).contains(edgeId)); 221 | int edgeId1 = underTest.addEdge(vertexId1, vertexId2, Sets.newHashSet(propEdge1), nowTs + 2); 222 | assertTrue(edgeId >= 0); 223 | edges = underTest.getEdgesAtTime(vertexId1, vertexId2, nowTs + 3); 224 | assertNotNull(edges); 225 | edgeList = Lists.newArrayList(edges); 226 | assertEquals(3, edgeList.size()); 227 | Set expectedEdges = new HashSet<>(Arrays.asList(edgeId, edgeId1)); 228 | Set actualEdgeSet = new HashSet<>(edgeList); 229 | expectedEdges.stream().forEach(e -> assertTrue(actualEdgeSet.contains(e))); 230 | 231 | assertEquals(2, Iterators.size(underTest.getInEdgesAtTime(vertexId2, nowTs))); 232 | assertEquals(3, Iterators.size(underTest.getInEdgesAtTime(vertexId2, nowTs + 2))); 233 | assertEquals(3, Iterators.size(underTest.getOutEdgesAtTime(vertexId1, nowTs + 4))); 234 | assertEquals(2, Iterators.size(underTest.getOutEdgesAtTime(vertexId1, nowTs))); 235 | assertEquals(0, Iterators.size(underTest.getOutEdgesAtTime(vertexId1, nowTs - 1))); 236 | 237 | // test edge properties 238 | Map> properties = underTest.getEdgePropertiesAtTime(vertexId1, vertexId2, nowTs + 3); 239 | assertEquals(3, properties.size()); 240 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId)), 0, 0, 0); 241 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId1)), 0, 0, 0); 242 | 243 | properties = underTest.getEdgePropertiesAtTime(vertexId1, vertexId2, nowTs + 3 * TimeUnit.MINUTES.toMillis(5)); 244 | assertEquals(4, properties.size()); 245 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId)), 1, 0, 400L); 246 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId1)), 0, 0, 0); 247 | 248 | properties = underTest.getEdgePropertiesAtTime(vertexId1, vertexId2, nowTs + 5 * TimeUnit.MINUTES.toMillis(5)); 249 | assertEquals(4, properties.size()); 250 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId)), 1, 0, 400L); 251 | checkPropertyValues(Sets.newHashSet(properties.get(edgeId1)), 1, 0, 200L); 252 | 253 | properties = underTest.getEdgePropertiesAtTime(vertexId1, vertexId2, nowTs - 1L); 254 | assertEquals(0, properties.size()); 255 | 256 | Map propertyMap = 257 | underTest.getEdgePropertyAtTime(vertexId1, vertexId2, property2, nowTs + 3 * TimeUnit.MINUTES.toMillis(5)); 258 | assertEquals(4, propertyMap.size()); 259 | checkPropertyValues(Sets.newHashSet(propertyMap.get(edgeId)), 1, 0, 400L); 260 | assertNull(propertyMap.get(edgeId1).getValue()); 261 | 262 | propertyMap = underTest.getEdgePropertyAtTime(vertexId1, vertexId2, property2, nowTs + 5 * TimeUnit.MINUTES.toMillis(5)); 263 | assertEquals(4, propertyMap.size()); 264 | checkPropertyValues(Sets.newHashSet(propertyMap.get(edgeId)), 1, 0, 400L); 265 | checkPropertyValues(Sets.newHashSet(propertyMap.get(edgeId1)), 1, 0, 200L); 266 | 267 | assertThrows(IllegalArgumentException.class, () -> underTest.addEdgeProperty(edgeId, property2, 700L, nowTs + 8 * TimeUnit.MINUTES.toMillis(5))); 268 | underTest.addEdgeProperty(edgeId, property2, 700L, nowTs + 15 * TimeUnit.MINUTES.toMillis(5)); 269 | TimestampedPropertyValue value = underTest.getEdgePropertyAtTime(edgeId, property2, nowTs + 16 * TimeUnit.MINUTES.toMillis(5)); 270 | assertNotNull(value); 271 | assertEquals(700L, value.getValue()); 272 | 273 | edges = underTest.getAllEdgesAtTime(nowTs + 15 * TimeUnit.MINUTES.toMillis(5)); 274 | assertNotNull(edges); 275 | edgeList = Lists.newArrayList(edges); 276 | assertEquals(4, edgeList.size()); 277 | 278 | underTest.purgeAtTime(nowTs + 10 * TimeUnit.MINUTES.toMillis(5)); 279 | Collection props = underTest.getVertexPropertiesAtTime(vertexId1, nowTs + 5 * TimeUnit.MINUTES.toMillis(5)); 280 | assertTrue(props.isEmpty()); 281 | props = underTest.getVertexPropertiesAtTime(vertexId1, nowTs + 21 * TimeUnit.MINUTES.toMillis(5)); 282 | checkPropertyValues(Sets.newHashSet(props), 2, 300L, 700L); 283 | 284 | props = underTest.getVertexPropertiesAtTime(vertexId2, nowTs + 5 * TimeUnit.MINUTES.toMillis(5)); 285 | assertTrue(props.isEmpty()); 286 | props = underTest.getVertexPropertiesAtTime(vertexId2, nowTs + 16 * TimeUnit.MINUTES.toMillis(5)); 287 | checkPropertyValues(Sets.newHashSet(props), 2, 600L, 200L); 288 | 289 | props = underTest.getEdgePropertiesAtTime(edgeId, nowTs + 6 * TimeUnit.MINUTES.toMillis(5)); 290 | assertTrue(props.isEmpty()); 291 | props = underTest.getEdgePropertiesAtTime(edgeId, nowTs + 12 * TimeUnit.MINUTES.toMillis(5)); 292 | checkPropertyValues(Sets.newHashSet(props), 1, 0L, 200L); 293 | 294 | 295 | props = underTest.getEdgePropertiesAtTime(edgeId1, nowTs + 6 * TimeUnit.MINUTES.toMillis(5)); 296 | assertTrue(props.isEmpty()); 297 | props = underTest.getEdgePropertiesAtTime(edgeId1, nowTs + 16 * TimeUnit.MINUTES.toMillis(5)); 298 | checkPropertyValues(Sets.newHashSet(props), 1, 0L, 800L); 299 | } 300 | } 301 | 302 | private void checkPropertyValues(Set properties, int size, long p1Val, long p2Val) { 303 | assertNotNull(properties); 304 | assertEquals(size, properties.size()); 305 | for (TimestampedPropertyValue p : properties) { 306 | if (p.getName().equals(property1)) { 307 | assertEquals(p1Val, p.getValue()); 308 | } 309 | if (p.getName().equals(property2)) { 310 | assertEquals(p2Val, p.getValue()); 311 | } 312 | } 313 | } 314 | 315 | private int getRandomId() { 316 | return Math.abs(random.nextInt()); 317 | } 318 | 319 | } 320 | -------------------------------------------------------------------------------- /src/main/java/core/InMemoryGraphStorage.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import com.google.common.annotations.VisibleForTesting; 9 | import core.propertystore.PropertyStore; 10 | import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; 11 | import it.unimi.dsi.fastutil.ints.IntOpenHashSet; 12 | import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; 13 | import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; 14 | import it.unimi.dsi.fastutil.longs.LongOpenHashSet; 15 | import it.unimi.dsi.fastutil.longs.LongSet; 16 | import it.unimi.dsi.fastutil.objects.ObjectArrayList; 17 | import lombok.AllArgsConstructor; 18 | import lombok.Builder; 19 | import lombok.Data; 20 | import lombok.EqualsAndHashCode; 21 | import model.Edge; 22 | import model.Property; 23 | import model.PropertyAwareEntity; 24 | import model.Vertex; 25 | 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | import java.util.NavigableMap; 29 | import java.util.Set; 30 | import java.util.TreeMap; 31 | import java.util.concurrent.atomic.AtomicInteger; 32 | 33 | @SuppressWarnings("rawtypes") 34 | @EqualsAndHashCode 35 | @AllArgsConstructor 36 | class InMemoryGraphStorage { 37 | private static final int maxGraphStorageDuration = Integer.MAX_VALUE; 38 | private final long initTs; 39 | // atomic running counter for an index 40 | private final AtomicInteger vertexInsertIndex = new AtomicInteger(0); 41 | 42 | // time based canonical entity maps. 43 | 44 | // private final Map vertices = new Long2ObjectOpenHashMap<>(); 45 | private final ObjectArrayList vertices = new ObjectArrayList<>(500000); 46 | private final Long2IntOpenHashMap vertexIdToInsertionPoint = new Long2IntOpenHashMap(500000); 47 | private final Map edges = new Long2ObjectOpenHashMap<>(4000000); 48 | //private final Map properties = new HashMap<>(); 49 | private final PropertyStore properties = new PropertyStore(); 50 | private final NavigableMap verticesAtTime = new TreeMap<>(); 51 | private final NavigableMap edgesAtTime = new TreeMap<>(); 52 | // private final NavigableMap propertiesAtTime = new TreeMap<>(); 53 | 54 | // lookup maps -> entity based view by time 55 | private final Map> outgoingEdgesByTimeForVertex = new Long2ObjectOpenHashMap<>(); 56 | private final Map> incomingEdgesByTimeForVertex = new Long2ObjectOpenHashMap<>(); 57 | private final Map> propertiesByTimeForVertex = new Long2ObjectOpenHashMap<>(); 58 | private final Map> propertiesByTimeForEdge = new Long2ObjectOpenHashMap<>(); 59 | private final Map> verticesByTimeForEdge = new Long2ObjectOpenHashMap<>(); 60 | 61 | private final Map vertexProperties = new Long2ObjectOpenHashMap<>(400000); 62 | private final Map edgeProperties = new Long2ObjectOpenHashMap<>(4000000); 63 | 64 | // lookup maps -> time based view by entity 65 | // TODO: check if these are really useful. 66 | private final NavigableMap>> vertexOutgoingEdgesAtTime = new TreeMap<>(); 67 | private final NavigableMap>> vertexIncomingEdgesAtTime = new TreeMap<>(); 68 | private final NavigableMap>> vertexPropertiesAtTime = new TreeMap<>(); 69 | private final NavigableMap>> edgePropertiesAtTime = new TreeMap<>(); 70 | private final NavigableMap>> edgeVerticesAtTime = new TreeMap<>(); 71 | 72 | public long getInitTs() { 73 | return initTs; 74 | } 75 | 76 | // Todo: Upasana call this method while adding Edge, Vertex, Property. 77 | // This needs to be done while changing "long time" to "int time" 78 | public boolean validateTimestamp(final long ts) { 79 | if (ts < initTs) { 80 | return false; 81 | } 82 | if (ts - initTs >= maxGraphStorageDuration) { 83 | return false; 84 | } 85 | return true; 86 | } 87 | 88 | void addVertex(final Vertex v, final long timestamp) { 89 | /*Preconditions.checkArgument((v instanceof TemporalVertex), 90 | String.format("Vertex supplied is not of type %s ", 91 | TemporalVertex.class.getName())); 92 | TemporalVertex tv = (TemporalVertex)v; 93 | 94 | 95 | // make a light-weight copy of the incoming vertex by storing only the 96 | // basic properties 97 | TemporalVertex.TemporalVertexBuilder b = TemporalVertex.builder(); 98 | b.id(tv.getId()); 99 | b.label(tv.getLabel()); 100 | b.time(tv.getTime()); 101 | //vertices.put(tv.getId(), b.build()); 102 | if (!vertexIdToInsertionPoint.containsKey(tv.getId())) { 103 | vertices.add(b.build()); 104 | vertexIdToInsertionPoint.put(tv.getId(), vertexInsertIndex.getAndIncrement()); 105 | } else { 106 | int insertIndex = vertexIdToInsertionPoint.get(tv.getId()); 107 | vertices.set(insertIndex, b.build()); 108 | } 109 | 110 | LongSet vertexIds = new LongOpenHashSet(); 111 | 112 | Map.Entry prevVerticesEntry = verticesAtTime.floorEntry(timestamp); 113 | if (prevVerticesEntry != null) { 114 | vertexIds.addAll(prevVerticesEntry.getValue()); 115 | } 116 | vertexIds.add(tv.getId()); 117 | verticesAtTime.put(timestamp, vertexIds); 118 | 119 | storeGraphEntityProps(tv, propertiesByTimeForVertex, vertexProperties);*/ 120 | 121 | } 122 | 123 | void addEdge(final Edge e, long timestamp) { 124 | /*Preconditions.checkArgument((e instanceof TemporalEdge), 125 | String.format("Edge supplied is not of type %s ", 126 | TemporalEdge.class.getName())); 127 | TemporalEdge te = (TemporalEdge) e; 128 | 129 | // store lightweight edge representations 130 | int sourceVertexIndex = Integer.MIN_VALUE; 131 | int destVertexIndex = Integer.MIN_VALUE; 132 | 133 | for (int i = 0; i < vertices.size(); i++) { 134 | if (vertices.get(i).getId() == te.getSrcVertex().getId()) { 135 | sourceVertexIndex = i; 136 | } 137 | if (vertices.get(i).getId() == te.getDestVertex().getId()) { 138 | destVertexIndex = i; 139 | } 140 | } 141 | if (sourceVertexIndex == Integer.MIN_VALUE || destVertexIndex == Integer.MIN_VALUE) { 142 | throw new IllegalStateException(String.format("Edge Id : %d has missing source or " + 143 | "destination vertix information", 144 | te.getId())); 145 | } 146 | edges.put(e.getId(), new int[] {sourceVertexIndex, destVertexIndex}); 147 | 148 | Map.Entry prevEdgesEntry = edgesAtTime.floorEntry(timestamp); 149 | LongSet edgeIds = new LongOpenHashSet(); 150 | 151 | if (prevEdgesEntry != null) { 152 | edgeIds.addAll(prevEdgesEntry.getValue()); 153 | } 154 | edgeIds.add(e.getId()); 155 | edgesAtTime.put(timestamp, edgeIds); 156 | 157 | storeGraphEntityProps(te, propertiesByTimeForEdge, edgeProperties); 158 | storeVertexEdgeRelations(te, timestamp);*/ 159 | } 160 | 161 | Vertex getVertexAtTime(final long vertexId, final long timestamp) { 162 | /* 163 | if (!vertices.containsKey(vertexId)) { 164 | return null; 165 | } 166 | 167 | */ 168 | /*if (!vertexIdToInsertionPoint.containsKey(vertexId)) { 169 | return null; 170 | } 171 | 172 | Map.Entry vertexIdsAtTime = verticesAtTime.floorEntry(timestamp); 173 | if (vertexIdsAtTime == null) { 174 | return null; 175 | } 176 | if (!vertexIdsAtTime.getValue().contains(vertexId)) { 177 | return null; 178 | } 179 | TemporalVertex tv = (TemporalVertex) vertices.get(vertexIdToInsertionPoint.get(vertexId)); 180 | 181 | TemporalVertex.TemporalVertexBuilder builder = tv.toBuilder(); 182 | 183 | if (propertiesByTimeForVertex.containsKey(vertexId)) { 184 | // get the properties associated with the vertex at the provided time 185 | NavigableMap vertexProps = propertiesByTimeForVertex.get(vertexId); 186 | Map.Entry propEntry = vertexProps.floorEntry(timestamp); 187 | HashSet finalProps = new HashSet<>(); 188 | propEntry.getValue().stream().forEach(propId -> { 189 | if (properties.containsKey(propId)) { 190 | Property p = properties.get(propId); 191 | finalProps.add(p); 192 | } 193 | }); 194 | TreeMap> propertiesAtTime = new TreeMap<>(); 195 | propertiesAtTime.put(vertexIdsAtTime.getKey(), finalProps); 196 | // builder.properties(propertiesAtTime); 197 | 198 | HashMap> propertiesByTime = new HashMap<>(); 199 | 200 | for (Property p : finalProps) { 201 | TreeMap propertyByTime = new TreeMap<>(); 202 | propertyByTime.put(timestamp, p); 203 | propertiesByTime.put(p.getName(), propertyByTime); 204 | } 205 | // builder.properties(propertiesByTime); 206 | } 207 | return builder.build();*/ 208 | return null; 209 | } 210 | 211 | Edge getEdgeAtTime(final long edgeId, final long timestamp) { 212 | return null; 213 | /* 214 | if (!edges.containsKey(edgeId)) { 215 | return null; 216 | } */ 217 | /*Map.Entry edgeIdsAtTime = edgesAtTime.floorEntry(timestamp); 218 | if (edgeIdsAtTime == null) { 219 | return null; 220 | } 221 | if (!edgeIdsAtTime.getValue().contains(edgeId)) { 222 | return null; 223 | } 224 | TemporalEdge.TemporalEdgeBuilder builder = TemporalEdge.builder(); 225 | 226 | // get the vertices corresponding to an edge id. 227 | int[] vertexIds = edges.get(edgeId); 228 | Vertex srcVertex = vertices.get(vertexIds[0]); 229 | Vertex dstVertex = vertices.get(vertexIds[1]); 230 | builder.srcVertex(srcVertex); 231 | builder.destVertex(dstVertex); 232 | 233 | // TODO: remove DRY violation with respect to persisting entity properties 234 | if (propertiesByTimeForEdge.containsKey(edgeId)) { 235 | // get the properties associated with the vertex at the provided time 236 | NavigableMap edgeProps = propertiesByTimeForEdge.get(edgeId); 237 | Map.Entry propEntry = edgeProps.floorEntry(timestamp); 238 | Set finalProps = new HashSet<>(); 239 | propEntry.getValue().stream().forEach(propId -> { 240 | if (properties.containsKey(propId)) { 241 | Property p = properties.get(propId); 242 | finalProps.add(p); 243 | } 244 | }); 245 | Long2ObjectAVLTreeMap> propertiesAtTime = new Long2ObjectAVLTreeMap<>(); 246 | propertiesAtTime.put(edgeIdsAtTime.getKey(), finalProps); 247 | // builder.allPropertiesWithTime(propertiesAtTime); 248 | 249 | HashMap> propertiesByTime = new HashMap<>(); 250 | 251 | for (Property p : finalProps) { 252 | TreeMap propertyByTime = new TreeMap<>(); 253 | propertyByTime.put(timestamp, p); 254 | propertiesByTime.put(p.getName(), propertyByTime); 255 | } 256 | // builder.allPropertiesByTime(propertiesByTime); 257 | } 258 | 259 | return builder.build();*/ 260 | } 261 | 262 | Property getEdgePropertyAtTime(final long edgeId, final String propertyName, final long timestamp) { 263 | 264 | return getPropertyAtTime(edgeId, propertyName, timestamp, propertiesByTimeForEdge, edgeProperties); 265 | } 266 | 267 | Property getVertexPropertyAtTime(final long vertexId, final String propertyName, final long timestamp) { 268 | 269 | return getPropertyAtTime(vertexId, propertyName, timestamp, propertiesByTimeForVertex, vertexProperties); 270 | } 271 | 272 | @VisibleForTesting 273 | /* 274 | Map getVertices() { 275 | return vertices; 276 | }*/ 277 | 278 | ObjectArrayList getVertices() { 279 | return vertices; 280 | } 281 | 282 | @VisibleForTesting 283 | Map getEdges() { 284 | return edges; 285 | } 286 | 287 | @VisibleForTesting 288 | Map getProperties() { 289 | return properties.getProperties(); 290 | } 291 | 292 | @VisibleForTesting 293 | NavigableMap getVerticesAtTime() { 294 | return verticesAtTime; 295 | } 296 | 297 | @VisibleForTesting 298 | NavigableMap getEdgesAtTime() { 299 | return edgesAtTime; 300 | } 301 | 302 | /* 303 | @VisibleForTesting 304 | NavigableMap getPropertiesAtTime() { 305 | return propertiesAtTime; 306 | } 307 | 308 | */ 309 | 310 | @VisibleForTesting 311 | Map> getOutgoingEdgesByTimeForVertex() { 312 | return outgoingEdgesByTimeForVertex; 313 | } 314 | 315 | @VisibleForTesting 316 | Map> getIncomingEdgesByTimeForVertex() { 317 | return incomingEdgesByTimeForVertex; 318 | } 319 | 320 | @VisibleForTesting 321 | Map> getPropertiesByTimeForVertex() { 322 | return propertiesByTimeForVertex; 323 | } 324 | 325 | @VisibleForTesting 326 | Map> getPropertiesByTimeForEdge() { 327 | return propertiesByTimeForEdge; 328 | } 329 | 330 | @VisibleForTesting 331 | Map> getVerticesByTimeForEdge() { 332 | return verticesByTimeForEdge; 333 | } 334 | 335 | @VisibleForTesting 336 | Map>> getVertexOutgoingEdgesAtTime() { 337 | return vertexOutgoingEdgesAtTime; 338 | } 339 | 340 | @VisibleForTesting 341 | Map>> getVertexIncomingEdgesAtTime() { 342 | return vertexIncomingEdgesAtTime; 343 | } 344 | 345 | @VisibleForTesting 346 | Map>> getVertexPropertiesAtTime() { 347 | return vertexPropertiesAtTime; 348 | } 349 | 350 | @VisibleForTesting 351 | Map>> getEdgePropertiesAtTime() { 352 | return edgePropertiesAtTime; 353 | } 354 | 355 | @VisibleForTesting 356 | Map>> getEdgeVerticesAtTime() { 357 | return edgeVerticesAtTime; 358 | } 359 | 360 | @VisibleForTesting 361 | Long2IntOpenHashMap getVertexIdToInsertionPoint() { 362 | return this.vertexIdToInsertionPoint; 363 | } 364 | 365 | @VisibleForTesting 366 | Map getEdgeProperties() { 367 | return edgeProperties; 368 | } 369 | 370 | @VisibleForTesting 371 | Map getVertexProperties() { 372 | return vertexProperties; 373 | } 374 | 375 | 376 | private void storeGraphEntityProps(PropertyAwareEntity entity, 377 | Map> propertiesByTimeForEntity, 378 | Int2ObjectOpenHashMap entityProperties) { 379 | 380 | // check if the entity exists. If yes, then the current property set 381 | // should build upon the historical property set. 382 | TreeMap entityPropsByTime = propertiesByTimeForEntity.getOrDefault(entity.getId(), 383 | new TreeMap<>()); 384 | 385 | LongOpenHashSet propertiesForEntity = entityProperties.getOrDefault(entity.getId(), 386 | new LongOpenHashSet()); 387 | 388 | TreeMap> allProps = new TreeMap<>(); // entity 389 | // .getAllPropertiesWithTime(); 390 | for (Map.Entry> propertyTimeEntry : allProps.entrySet()) { 391 | // record properties. also record the historical properties along with 392 | // the current ones 393 | 394 | Set props = propertyTimeEntry.getValue(); 395 | props.forEach((property) -> { 396 | // if the property already exists then 397 | // add the passed in property values 398 | // to the values already existing. 399 | if (properties.containsKey(property.getId())) { 400 | Property p = properties.get(property.getId()); 401 | p.setValueAtTime(propertyTimeEntry.getKey(), 402 | property.getValueAtTime(propertyTimeEntry.getKey())); 403 | properties.put(property.getId(), p); 404 | } else { 405 | properties.put(property.getId(), property); 406 | } 407 | propertiesForEntity.add(property.getId()); 408 | }); 409 | } 410 | entityProperties.put(entity.getId(), propertiesForEntity); 411 | } 412 | 413 | /*private void storeVertexEdgeRelations(final TemporalEdge te, final long timestamp) { 414 | Vertex srcVertex = te.getSrcVertex(); 415 | Vertex dstVertex = te.getDestVertex(); 416 | 417 | storeEdgeRelationInDirection(te, timestamp, srcVertex, outgoingEdgesByTimeForVertex); 418 | storeEdgeRelationInDirection(te, timestamp, dstVertex, incomingEdgesByTimeForVertex); 419 | }*/ 420 | 421 | /*private void storeEdgeRelationInDirection(TemporalEdge te, long timestamp, Vertex vertex, 422 | Map> edgesForVertexInDirection) { 423 | TreeMap edgesInDirectionAtTime = 424 | edgesForVertexInDirection.getOrDefault(vertex.getId(), new TreeMap<>()); 425 | Map.Entry prevEdgesInDirection = edgesInDirectionAtTime.floorEntry(timestamp); 426 | LongSet currentEdgesInDirection = new LongOpenHashSet(); 427 | if (prevEdgesInDirection != null) { 428 | currentEdgesInDirection.addAll(prevEdgesInDirection.getValue()); 429 | } 430 | currentEdgesInDirection.add(te.getId()); 431 | edgesInDirectionAtTime.put(timestamp, currentEdgesInDirection); 432 | edgesForVertexInDirection.put(vertex.getId(), edgesInDirectionAtTime); 433 | }*/ 434 | 435 | private Property getPropertyAtTime(long entityId, String propertyName, long timestamp, 436 | final Map> propertiesByTimeForEntity, 437 | final Map entityProperties) { 438 | /* 439 | if (!propertiesByTimeForEntity.containsKey(entityId)) { 440 | return null; 441 | } 442 | 443 | */ 444 | if (!entityProperties.containsKey(entityId)) { 445 | return null; 446 | } 447 | IntOpenHashSet propertySet = entityProperties.get(entityId); 448 | 449 | // get the property corresponding to the specified name 450 | Property targetProperty = null; 451 | 452 | for (Integer propertyId : propertySet) { 453 | if (!propertyName.equals(properties.get(propertyId).getName())) { 454 | continue; 455 | } 456 | targetProperty = properties.get(propertyId); 457 | break; 458 | } 459 | 460 | return targetProperty; 461 | /* 462 | Map.Entry propertiesByTime = propertiesByTimeForEntity.get(entityId).floorEntry(timestamp); 463 | if (propertiesByTime == null) { 464 | return null; 465 | } 466 | 467 | // currently there is no reverse index stored to map the property name and the property id 468 | // at a specific point in time. Hence we loop through all the properties found 469 | // at a particular point in time and filter the one with the name equal to the specified 470 | // name. 471 | // TODO; prahaladd - analyze the memory impact of maintaining a reverse index between the 472 | // the property name and the corresponding property in a time series based fashion. 473 | Set filtered = propertiesByTime.getValue().stream().filter(pid -> { 474 | return (properties.containsKey(pid) && properties.get(pid).getName().equals(propertyName)); 475 | }).collect(Collectors.toSet()); 476 | 477 | if (filtered.size() == 0) { 478 | return null; 479 | } 480 | // if there is more than one property, it is an error since 481 | // every edge can have only one property with a specific name 482 | if (filtered.size() > 1) { 483 | throw new IllegalStateException(String.format("Edge Id : %d has multiple properties with" + 484 | " the same name : %s", entityId, 485 | propertyName)); 486 | } 487 | return properties.get(filtered.iterator().next()); 488 | 489 | */ 490 | } 491 | 492 | 493 | @Data 494 | @AllArgsConstructor 495 | @EqualsAndHashCode 496 | @Builder(toBuilder = true) 497 | public static class IdPair { 498 | private final long srcEntityId; 499 | private final long dstEntityId; 500 | } 501 | } 502 | -------------------------------------------------------------------------------- /src/main/java/core/TemporalGraph.java: -------------------------------------------------------------------------------- 1 | package core; 2 | 3 | /* 4 | Copyright 2023 VMware, Inc. 5 | SPDX-License-Identifier: BSD-2-Clause 6 | */ 7 | 8 | import com.google.common.annotations.VisibleForTesting; 9 | import com.google.common.base.Preconditions; 10 | import com.google.common.collect.Lists; 11 | import com.google.common.collect.Sets; 12 | import core.propertystore.PropertyStore; 13 | import exceptions.PropertyNotFoundException; 14 | import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; 15 | import it.unimi.dsi.fastutil.ints.Int2ObjectMap; 16 | import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; 17 | import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; 18 | import it.unimi.dsi.fastutil.ints.IntOpenHashSet; 19 | import it.unimi.dsi.fastutil.ints.IntSet; 20 | import it.unimi.dsi.fastutil.longs.Long2ObjectMap; 21 | import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; 22 | import it.unimi.dsi.fastutil.objects.ObjectSortedSet; 23 | import lombok.AllArgsConstructor; 24 | import lombok.EqualsAndHashCode; 25 | import model.Graph; 26 | import model.Property; 27 | import model.TimestampedPropertyValue; 28 | 29 | import java.util.Collection; 30 | import java.util.Collections; 31 | import java.util.HashSet; 32 | import java.util.Iterator; 33 | import java.util.Map; 34 | import java.util.Set; 35 | import java.util.TreeMap; 36 | import java.util.concurrent.atomic.AtomicInteger; 37 | import java.util.stream.Collectors; 38 | 39 | @EqualsAndHashCode 40 | @AllArgsConstructor 41 | public class TemporalGraph implements Graph { 42 | static final int MAX_GRAPH_STORAGE_DURATION = Integer.MAX_VALUE; 43 | private static final String TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE = "Timestamp supplied %s falls beyond supported range."; 44 | private static final String SRC_DEST_NOT_FOUND_ERR_MSG_TEMPLATE = "Source vertex Id %d or destination vertex Id %d not found at time: %d."; 45 | private static final String VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE = "Vertex with Id %d not found at time: %d."; 46 | private static final String EDGE_NOT_FOUND_ERR_MSG_TEMPLATE = "Edge with Id %d not found at time: %d."; 47 | private static final String NULL_EMPTY_PROPERTIES_ERR_MSG_TEMPLATE = "Properties can't be empty or null"; 48 | private static final String PROPERTIES_TIME_ERR_MSG_TEMPLATE = "Property addition time: %d should be greater than %s addition time %d for property %s"; 49 | private final long initTs; 50 | // atomic running counter for a vertex index 51 | private final AtomicInteger vertexIndex = new AtomicInteger(0); 52 | 53 | // atomic running counter for an edge index 54 | private final AtomicInteger edgeIndex = new AtomicInteger(0); 55 | 56 | // map maintaining vertices as per time Map> 57 | private final TreeMap verticesByTime = new TreeMap<>(); 58 | 59 | // stores outgoing edges of a vertex map>> 60 | private final Map> outgoingEdgesByTimeForVertex = new Int2ObjectOpenHashMap<>(); 61 | // stores incoming edges of a vertex 62 | private final Map> incomingEdgesByTimeForVertex = new Int2ObjectOpenHashMap<>(); 63 | 64 | // time differential to edges Ids set 65 | private final Int2ObjectAVLTreeMap edgesByTime = new Int2ObjectAVLTreeMap<>(); 66 | 67 | // vertex to property map > 68 | private final Map vertexProperties = new Int2ObjectOpenHashMap<>(); 69 | 70 | // edge to property map > 71 | private final Map edgeProperties = new Int2ObjectOpenHashMap<>(); 72 | 73 | private final PropertyStore propertyStore = new PropertyStore(); 74 | 75 | // This needs to be checked while finding time differential from the user specified timestamp 76 | @VisibleForTesting 77 | protected boolean validateTimestamp(final long ts) { 78 | if (ts < initTs) { 79 | return false; 80 | } 81 | return ts - initTs < MAX_GRAPH_STORAGE_DURATION; 82 | } 83 | 84 | @Override 85 | public int addVertex(final Collection properties, final long timestamp) { 86 | Preconditions.checkArgument(validateTimestamp(timestamp), 87 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 88 | // Preconditions.checkArgument(null != properties && !properties.isEmpty(), NULL_EMPTY_PROPERTIES_ERR_MSG_TEMPLATE); 89 | final int vertexId = vertexIndex.getAndIncrement(); 90 | IntSet vertexIds = verticesByTime.computeIfAbsent((int) (timestamp - initTs), t -> new IntOpenHashSet(1)); 91 | vertexIds.add(vertexId); 92 | //adding property for vertex if properties are specified. first compress the property 93 | if (properties != null && !properties.isEmpty()) { 94 | for (final Property p : properties) { 95 | Preconditions.checkArgument(timestamp <= p.getTime(), 96 | String.format(PROPERTIES_TIME_ERR_MSG_TEMPLATE, p.getTime(), "Vertex", timestamp, p.getName())); 97 | } 98 | IntOpenHashSet propertyIds = new IntOpenHashSet(properties.size()); 99 | properties.stream().forEach( p -> { 100 | propertyIds.add(p.getId()); 101 | propertyStore.put(p.getId(), p); 102 | }); 103 | vertexProperties.put(vertexId, propertyIds); 104 | } 105 | 106 | return vertexId; 107 | } 108 | 109 | @Override 110 | public int addEdge(int srcVertexId, int destVertexId, Collection properties, long timestamp) { 111 | Preconditions.checkArgument(validateTimestamp(timestamp), 112 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 113 | Preconditions.checkArgument((srcVertexId < vertexIndex.get() && destVertexId < vertexIndex.get()), 114 | String.format(SRC_DEST_NOT_FOUND_ERR_MSG_TEMPLATE, srcVertexId, destVertexId, timestamp)); 115 | 116 | final int edgeId = edgeIndex.getAndIncrement(); 117 | final int timeDifferential = (int) (timestamp - initTs); 118 | final IntSet edgeIds = edgesByTime.computeIfAbsent(timeDifferential, t -> new IntOpenHashSet(1)); 119 | edgeIds.add(edgeId); 120 | 121 | //add outgoing edges in the map 122 | addEdgeForVertices(srcVertexId, timeDifferential, edgeId, outgoingEdgesByTimeForVertex); 123 | 124 | //add incoming edges in the map 125 | addEdgeForVertices(destVertexId, timeDifferential, edgeId, incomingEdgesByTimeForVertex); 126 | 127 | //adding property for edge if properties are specified 128 | if (properties != null && !properties.isEmpty()) { 129 | for (final Property p : properties) { 130 | Preconditions.checkArgument(timestamp <= p.getTime(), 131 | String.format(PROPERTIES_TIME_ERR_MSG_TEMPLATE, p.getTime(), "Edge", timestamp, p.getName())); 132 | } 133 | IntOpenHashSet propertyIds = new IntOpenHashSet(properties.size()); 134 | properties.stream().forEach( p -> { 135 | propertyIds.add(p.getId()); 136 | propertyStore.put(p.getId(), p); 137 | }); 138 | edgeProperties.put(edgeId, propertyIds); 139 | } 140 | 141 | return edgeId; 142 | } 143 | 144 | private void addEdgeForVertices(final int vertexId, 145 | final int timeDifferential, 146 | final int edgeId, 147 | final Map> edgesByTimeForVertex) { 148 | final TreeMap edgeMap = edgesByTimeForVertex.computeIfAbsent(vertexId, t -> new TreeMap<>()); 149 | final Set outgoingEdges = edgeMap.computeIfAbsent(timeDifferential, t -> new IntOpenHashSet(1)); 150 | outgoingEdges.add(edgeId); 151 | } 152 | 153 | @Override 154 | public Iterator getVerticesAtTime(final long timestamp) { 155 | Preconditions.checkArgument(validateTimestamp(timestamp), 156 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 157 | final int timeDifferential = (int) (timestamp - initTs); 158 | final Map.Entry floorEntry = verticesByTime.floorEntry(timeDifferential); 159 | if (null == floorEntry || null == floorEntry.getKey()) { 160 | return Collections.emptyIterator(); 161 | } 162 | return getIntIterator(verticesByTime.firstKey(), floorEntry.getKey(), verticesByTime).iterator(); 163 | } 164 | 165 | @Override 166 | public Iterator getEdgesAtTime(final int srcVertexId, 167 | final int destVertexId, 168 | final long timestamp) { 169 | Preconditions.checkArgument(validateTimestamp(timestamp), 170 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 171 | Preconditions.checkArgument((srcVertexId < vertexIndex.get() && destVertexId < vertexIndex.get()), 172 | String.format(SRC_DEST_NOT_FOUND_ERR_MSG_TEMPLATE, srcVertexId, destVertexId, timestamp)); 173 | Set outEdges = Sets.newHashSet(getOutEdgesAtTime(srcVertexId, timestamp)); 174 | if (outEdges.isEmpty()) { 175 | return Collections.emptyIterator(); 176 | } 177 | Set inEdges = Sets.newHashSet(getInEdgesAtTime(destVertexId, timestamp)); 178 | if (inEdges.isEmpty()) { 179 | return Collections.emptyIterator(); 180 | } 181 | outEdges.retainAll(inEdges); 182 | return outEdges.iterator(); 183 | } 184 | 185 | @Override 186 | public Iterator getOutEdgesAtTime(final int vertexId, final long timestamp) { 187 | Preconditions.checkArgument(validateTimestamp(timestamp), 188 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 189 | Preconditions.checkArgument(vertexId < vertexIndex.get(), 190 | String.format(VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE, vertexId, timestamp)); 191 | final TreeMap outgoingEdgesForVertex = outgoingEdgesByTimeForVertex.get(vertexId); 192 | if (null == outgoingEdgesForVertex) { 193 | return Collections.emptyIterator(); 194 | } 195 | final int timeDifferential = (int) (timestamp - initTs); 196 | final Map.Entry floorEntry = outgoingEdgesForVertex.floorEntry(timeDifferential); 197 | if (null == floorEntry || null == floorEntry.getKey()) { 198 | return Collections.emptyIterator(); 199 | } 200 | return getIntIterator(outgoingEdgesForVertex.firstKey(), floorEntry.getKey(), outgoingEdgesForVertex).iterator(); 201 | } 202 | 203 | @Override 204 | public Iterator getInEdgesAtTime(final int vertexId, final long timestamp) { 205 | Preconditions.checkArgument(validateTimestamp(timestamp), 206 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 207 | Preconditions.checkArgument(vertexId < vertexIndex.get(), 208 | String.format(VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE, vertexId, timestamp)); 209 | final TreeMap incomingEdgesForVertex = incomingEdgesByTimeForVertex.get(vertexId); 210 | if (null == incomingEdgesForVertex) { 211 | return null; 212 | } 213 | final int timeDifferential = (int) (timestamp - initTs); 214 | final Integer floorKey = incomingEdgesForVertex.floorEntry(timeDifferential).getKey(); 215 | if (null == floorKey) { 216 | return null; 217 | } 218 | return getIntIterator(incomingEdgesForVertex.firstKey(), floorKey, incomingEdgesForVertex).iterator(); 219 | } 220 | 221 | @Override 222 | public Collection getVertexPropertiesAtTime(final int vertexId, final long timestamp) { 223 | Preconditions.checkArgument(validateTimestamp(timestamp), 224 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 225 | Preconditions.checkArgument(vertexId < vertexIndex.get(), 226 | String.format(VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE, vertexId, timestamp)); 227 | final Set allProperties = getEntityProperties(vertexProperties, vertexId); 228 | final Set propertiesWithValue = new HashSet<>(); 229 | for (final Property p : allProperties) { 230 | final Object value = p.getValueAtTime(timestamp); 231 | if (null == value) { 232 | continue; 233 | } 234 | propertiesWithValue.add(new TimestampedPropertyValue(p.getName(), value)); 235 | } 236 | return propertiesWithValue; 237 | } 238 | 239 | @Override 240 | public TimestampedPropertyValue getVertexPropertyAtTime(final int vertexId, final String propertyName, final long timestamp) { 241 | Preconditions.checkArgument(validateTimestamp(timestamp), 242 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 243 | Preconditions.checkArgument(vertexId < vertexIndex.get(), 244 | String.format(VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE, vertexId, timestamp)); 245 | final Set allProperties = getEntityProperties(vertexProperties, vertexId); 246 | for (final Property p : allProperties) { 247 | if (!propertyName.equals(p.getName())) { 248 | continue; 249 | } 250 | final Object value = p.getValueAtTime(timestamp); 251 | if (null == value) { 252 | return null; 253 | } 254 | return new TimestampedPropertyValue(p.getName(), value); 255 | } 256 | return null; 257 | } 258 | 259 | @Override 260 | public Collection getEdgePropertiesAtTime(final int edgeId, final long timestamp) { 261 | Preconditions.checkArgument(validateTimestamp(timestamp), 262 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 263 | Preconditions.checkArgument(edgeId < edgeIndex.get(), 264 | String.format(EDGE_NOT_FOUND_ERR_MSG_TEMPLATE, edgeId, timestamp)); 265 | final Set allProperties = getEntityProperties(edgeProperties, edgeId); 266 | final Set propertiesWithValue = new HashSet<>(); 267 | for (final Property p : allProperties) { 268 | final Object value = p.getValueAtTime(timestamp); 269 | if (null == value) { 270 | continue; 271 | } 272 | propertiesWithValue.add(new TimestampedPropertyValue(p.getName(), value)); 273 | } 274 | return propertiesWithValue; 275 | } 276 | 277 | @Override 278 | public TimestampedPropertyValue getEdgePropertyAtTime(final int edgeId, final String propertyName, final long timestamp) { 279 | Preconditions.checkArgument(validateTimestamp(timestamp), 280 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 281 | Preconditions.checkArgument(edgeId < edgeIndex.get(), 282 | String.format(EDGE_NOT_FOUND_ERR_MSG_TEMPLATE, edgeId, timestamp)); 283 | final Set allProperties = getEntityProperties(edgeProperties, edgeId); 284 | for (final Property p : allProperties) { 285 | if (!propertyName.equals(p.getName())) { 286 | continue; 287 | } 288 | final Object value = p.getValueAtTime(timestamp); 289 | return new TimestampedPropertyValue(p.getName(), value); 290 | } 291 | return null; 292 | } 293 | 294 | @Override 295 | public Map> getEdgePropertiesAtTime(final int srcVertexId, 296 | final int destVertexId, 297 | final long timestamp) { 298 | Preconditions.checkArgument(validateTimestamp(timestamp), 299 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 300 | Preconditions.checkArgument((srcVertexId < vertexIndex.get() && destVertexId < vertexIndex.get()), 301 | String.format(SRC_DEST_NOT_FOUND_ERR_MSG_TEMPLATE, srcVertexId, destVertexId, timestamp)); 302 | final Iterator edgesBetweenVertices = getEdgesAtTime(srcVertexId, destVertexId, timestamp); 303 | final Map> edgeToProperties = new Int2ObjectOpenHashMap<>(); 304 | if (null == edgesBetweenVertices) { 305 | return edgeToProperties; 306 | } 307 | for (final int edgeId : Sets.newHashSet(edgesBetweenVertices)) { 308 | edgeToProperties.put(edgeId, getEdgePropertiesAtTime(edgeId, timestamp)); 309 | } 310 | return edgeToProperties; 311 | } 312 | 313 | @Override 314 | public Map getEdgePropertyAtTime(final int srcVertexId, 315 | final int destVertexId, 316 | final String propertyName, 317 | final long timestamp) { 318 | Preconditions.checkArgument(validateTimestamp(timestamp), 319 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 320 | Preconditions.checkArgument((srcVertexId < vertexIndex.get() && destVertexId < vertexIndex.get()), 321 | String.format(SRC_DEST_NOT_FOUND_ERR_MSG_TEMPLATE, srcVertexId, destVertexId, timestamp)); 322 | final Iterator edgesBetweenVertices = getEdgesAtTime(srcVertexId, destVertexId, timestamp); 323 | final Map edgeToProperties = new Int2ObjectOpenHashMap<>(); 324 | if (null == edgesBetweenVertices) { 325 | return edgeToProperties; 326 | } 327 | for (final int edgeId : Sets.newHashSet(edgesBetweenVertices)) { 328 | edgeToProperties.put(edgeId, getEdgePropertyAtTime(edgeId, propertyName, timestamp)); 329 | } 330 | return edgeToProperties; 331 | } 332 | 333 | @Override 334 | public void addVertexProperty(final int vertexId, 335 | final String propertyName, 336 | final Object value, 337 | long timestamp) throws PropertyNotFoundException { 338 | Preconditions.checkArgument(validateTimestamp(timestamp), 339 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 340 | Preconditions.checkArgument(vertexId < vertexIndex.get(), 341 | String.format(VERTEX_NOT_FOUND_ERR_MSG_TEMPLATE, vertexId, timestamp)); 342 | final Set allProperties = getEntityProperties(vertexProperties, vertexId); 343 | final Property p = getPropertyFromSet(propertyName, allProperties); 344 | if (null == p) { 345 | throw new PropertyNotFoundException(String.format("Property %s not found for the vertex: %d", propertyName, vertexId)); 346 | } else { 347 | p.setValueAtTime(timestamp, value); 348 | propertyStore.put(p.getId(), p); 349 | } 350 | } 351 | 352 | @Override 353 | public void addEdgeProperty(final int edgeId, 354 | final String propertyName, 355 | final Object value, 356 | long timestamp) throws PropertyNotFoundException { 357 | Preconditions.checkArgument(validateTimestamp(timestamp), 358 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 359 | Preconditions.checkArgument(edgeId < edgeIndex.get(), 360 | String.format(EDGE_NOT_FOUND_ERR_MSG_TEMPLATE, edgeId, timestamp)); 361 | final Set allProperties = getEntityProperties(edgeProperties, edgeId); 362 | final Property p = getPropertyFromSet(propertyName, allProperties); 363 | if (null == p) { 364 | throw new PropertyNotFoundException(String.format("Property %s not found for the edge: %d", propertyName, edgeId)); 365 | } else { 366 | p.setValueAtTime(timestamp, value); 367 | propertyStore.put(p.getId(), p); 368 | } 369 | } 370 | 371 | @Override 372 | public Iterator getAllEdgesAtTime(long timestamp) { 373 | Preconditions.checkArgument(validateTimestamp(timestamp), 374 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 375 | final int timeDifferential = (int) (timestamp - initTs); 376 | 377 | final Set allEdgesAtTime = getIntIterator(timeDifferential, edgesByTime); 378 | if (allEdgesAtTime.isEmpty()) { 379 | return Collections.emptyIterator(); 380 | } 381 | return allEdgesAtTime.iterator(); 382 | } 383 | 384 | /* 385 | * Algorithm 386 | * Graph init time : t0, purging time of graph: tp 387 | * Get all vertices till time floor(tp). => v 388 | * Get all the outgoing / incoming edges for those vertices. => e 389 | * From Property store, remove all the data points for each property of the e, v from init time till floor(tp).*/ 390 | @Override 391 | public void purgeAtTime(final long timestamp) { 392 | Preconditions.checkArgument(validateTimestamp(timestamp), 393 | String.format(TIME_RANGE_CROSSED_ERR_MSG_TEMPLATE, timestamp)); 394 | purgeEdgeProperties(timestamp); 395 | purgeVertexProperties(timestamp); 396 | 397 | } 398 | 399 | private void purgeVertexProperties(long timestamp) { 400 | Iterator verticesIterator = getVerticesAtTime(timestamp); 401 | if (null == verticesIterator) { 402 | return; 403 | } 404 | for (final int vertexId : Lists.newArrayList(verticesIterator)) { 405 | IntOpenHashSet propertyIdsPerVertex = vertexProperties.get(vertexId); 406 | if (null == propertyIdsPerVertex) { 407 | continue; 408 | } 409 | propertyStore.purgePropertiesTillTime(propertyIdsPerVertex 410 | , timestamp); 411 | } 412 | } 413 | 414 | private void purgeEdgeProperties(final long timestamp) { 415 | Iterator edgesIterator = getAllEdgesAtTime(timestamp); 416 | if (null == edgesIterator) { 417 | return; 418 | } 419 | for (final int edgeId : Lists.newArrayList(edgesIterator)) { 420 | IntOpenHashSet propertyIdsPerEdge = edgeProperties.get(edgeId); 421 | if (null == propertyIdsPerEdge) { 422 | continue; 423 | } 424 | propertyStore.purgePropertiesTillTime(propertyIdsPerEdge, timestamp); 425 | } 426 | } 427 | 428 | private void trimInternalMapsForVertexEdgeDirectionMapping(Int2ObjectOpenHashMap mapping) { 429 | Int2ObjectMap.FastEntrySet> entries = ((Int2ObjectOpenHashMap>) mapping).int2ObjectEntrySet(); 430 | for (Int2ObjectMap.Entry> entry : entries) { 431 | TreeMap tm = entry.getValue(); 432 | for (Map.Entry intSet : tm.entrySet()) { 433 | ((IntOpenHashSet)(intSet.getValue())).trim(); 434 | } 435 | } 436 | } 437 | 438 | private void trimInternalMapsForVertexToEdgeMapping(Long2ObjectOpenHashMap mapping) { 439 | Long2ObjectMap.FastEntrySet entries = mapping.long2ObjectEntrySet(); 440 | for (Long2ObjectMap.Entry entry : entries) { 441 | IntSet is = entry.getValue(); 442 | ((IntOpenHashSet)(is)).trim(); 443 | } 444 | } 445 | 446 | private void trimInternalMapsForProperties(Int2ObjectOpenHashMap mapping) { 447 | Int2ObjectMap.FastEntrySet entries = mapping.int2ObjectEntrySet(); 448 | for (Int2ObjectMap.Entry entry : entries) { 449 | IntSet is = entry.getValue(); 450 | ((IntOpenHashSet)(is)).trim(); 451 | } 452 | } 453 | 454 | private Property getPropertyFromSet(final String propertyName, final Set allProperties) { 455 | if (null == allProperties) { 456 | return null; 457 | } 458 | for (final Property p : allProperties) { 459 | if (propertyName.equals(p.getName())) { 460 | return p; 461 | } 462 | } 463 | return null; 464 | } 465 | 466 | private Set getIntIterator(final int firstKey, 467 | final int floorKey, 468 | final TreeMap map) { 469 | final Set entityCollection = new HashSet<>(); 470 | final Map subMap = map.subMap(firstKey, true, floorKey, true); 471 | if (null == subMap) { 472 | return entityCollection; 473 | } 474 | for (final Map.Entry entry : subMap.entrySet()) { 475 | entityCollection.addAll(entry.getValue()); 476 | } 477 | return entityCollection; 478 | } 479 | 480 | private Set getIntIterator(final int floorKey, 481 | final Int2ObjectAVLTreeMap map) { 482 | final Set entityCollection = new HashSet<>(); 483 | Int2ObjectSortedMap headMap = map.headMap(floorKey); 484 | ObjectSortedSet> headMapEntries = headMap.int2ObjectEntrySet(); 485 | for (Int2ObjectMap.Entry e : headMapEntries) { 486 | entityCollection.addAll(e.getValue()); 487 | } 488 | IntSet floorKeyEntry = map.get(floorKey); 489 | if (floorKeyEntry != null) { 490 | entityCollection.addAll(floorKeyEntry); 491 | } 492 | return entityCollection; 493 | } 494 | 495 | private Set getEntityProperties(Map entityProperties, final int entityId) { 496 | IntOpenHashSet propertyIds = entityProperties.getOrDefault(entityId, new IntOpenHashSet()); 497 | if (propertyIds.isEmpty()) { 498 | return Collections.emptySet(); 499 | } 500 | Set properties = 501 | propertyIds.stream().map(propertyStore::get).collect(Collectors.toSet()); 502 | 503 | return properties; 504 | } 505 | } 506 | --------------------------------------------------------------------------------