├── .github ├── labeler.yml ├── release-drafter.yml └── workflows │ ├── changelog-release-drafter.yml │ ├── codeql-analysis.yml │ ├── labeler.yml │ ├── maven.yml │ ├── release.yml │ ├── renovate.yml │ └── required-labels.yml ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── LICENSE ├── README.md ├── aerospike-benchmark ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── playtika │ └── janusgraph │ └── aerospike │ └── benchmark │ ├── Configurations.java │ ├── Graph.java │ └── JanusgraphBenchmarks.java ├── aerospike-container ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── aerospike │ ├── AerospikeContainerUtils.java │ ├── AerospikeProperties.java │ ├── AerospikeWaitStrategy.java │ └── AsadmCommandExecutor.java ├── aerospike-storage-backend ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── playtika │ │ └── janusgraph │ │ ├── aerospike │ │ ├── AerospikeKeyColumnValueStore.java │ │ ├── AerospikePolicyProvider.java │ │ ├── AerospikeStoreManager.java │ │ ├── AerospikeTransaction.java │ │ ├── ConfigOptions.java │ │ ├── DeferredLock.java │ │ ├── operations │ │ │ ├── AerospikeKeyIterator.java │ │ │ ├── AerospikeOperations.java │ │ │ ├── BasicMutateOperations.java │ │ │ ├── BasicOperations.java │ │ │ ├── BasicScanOperations.java │ │ │ ├── ErrorMapper.java │ │ │ ├── IdsCleanupOperations.java │ │ │ ├── MutateOperations.java │ │ │ ├── Operations.java │ │ │ ├── ReadOperations.java │ │ │ ├── ScanOperations.java │ │ │ ├── UnsupportedScanOperations.java │ │ │ └── batch │ │ │ │ ├── BatchExpectedValueOperations.java │ │ │ │ ├── BatchLocks.java │ │ │ │ ├── BatchOperationsUtil.java │ │ │ │ ├── BatchUpdate.java │ │ │ │ ├── BatchUpdateOperations.java │ │ │ │ ├── BatchUpdateSerde.java │ │ │ │ ├── BatchUpdates.java │ │ │ │ ├── ExpectedValue.java │ │ │ │ ├── UpdateValue.java │ │ │ │ └── WalOperations.java │ │ └── util │ │ │ ├── AerospikeUtils.java │ │ │ ├── AsyncUtil.java │ │ │ └── NamedThreadFactory.java │ │ ├── trace │ │ └── DebugJanusGraph.java │ │ └── utility │ │ ├── GhostVertexRemover.java │ │ └── KeyExtractor.java │ └── test │ ├── java │ └── com │ │ └── playtika │ │ └── janusgraph │ │ └── aerospike │ │ ├── AerospikeIDAuthorityCleanupTest.java │ │ ├── AerospikePolicyProviderTest.java │ │ ├── AerospikeStoreManagerTest.java │ │ ├── AerospikeTestUtils.java │ │ ├── GraphOfTheGodsTest.java │ │ ├── InconsistencyTest.java │ │ ├── JmxMetricsTest.java │ │ ├── debuggrapdb │ │ ├── AerospikeDebugGraphCacheTest.java │ │ ├── AerospikeDebugGraphConcurrentTest.java │ │ ├── AerospikeDebugGraphIterativeTest.java │ │ ├── AerospikeDebugGraphPerformanceMemoryTest.java │ │ ├── AerospikeDebugGraphTest.java │ │ ├── AerospikeDebugOLAPTest.java │ │ ├── AerospikeDebugOperationCountingTest.java │ │ ├── AerospikeEventualDebugGraphTest.java │ │ └── AerospikePartitionDebugGraphTest.java │ │ ├── diskstorage │ │ ├── AerospikeIDAuthorityTest.java │ │ ├── AerospikeLockStoreTest.java │ │ ├── AerospikeLogTest.java │ │ ├── AerospikeMultiWriteStoreTest.java │ │ └── AerospikeStoreTest.java │ │ ├── graphdb │ │ ├── AerospikeEventualGraphTest.java │ │ ├── AerospikeGraphCacheTest.java │ │ ├── AerospikeGraphConcurrentTest.java │ │ ├── AerospikeGraphIterativeTest.java │ │ ├── AerospikeGraphPerformanceMemoryTest.java │ │ ├── AerospikeGraphTest.java │ │ ├── AerospikeOLAPTest.java │ │ ├── AerospikeOperationCountingTest.java │ │ └── AerospikePartitionGraphTest.java │ │ └── operations │ │ ├── AerospikeKeyIteratorTest.java │ │ ├── AerospikeTest.java │ │ └── MutateOperationsTest.java │ └── resources │ └── log4j2.xml ├── mvnw ├── mvnw.cmd └── pom.xml /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | "documentation": 2 | - /**/*.adoc -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: $NEXT_PATCH_VERSION 2 | tag-template: $NEXT_PATCH_VERSION 3 | 4 | template: | 5 | # Changes 6 | $CHANGES 7 | 8 | # -------- 9 | # NOTE: When adding new labels please also update required-labels.yml workflow. 10 | # -------- 11 | categories: 12 | - title: 💣️ Breaking changes 13 | label: breaking-change 14 | 15 | - title: 🚀 Features & Enhancements 16 | labels: 17 | - feature 18 | - enhancement 19 | 20 | - title: 🐞 Fixes 21 | label: bug 22 | 23 | - title: 📁 Java Dependencies updates 24 | label: dependencies 25 | 26 | - title: 📁 Docker images updates 27 | label: docker-update-images 28 | 29 | - title: 📖 Documentation 30 | label: documentation 31 | 32 | - title: 🏡 Housekeeping 33 | label: housekeeping 34 | 35 | -------------------------------------------------------------------------------- /.github/workflows/changelog-release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Changelog Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - develop 7 | 8 | jobs: 9 | update_release_draft: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: release-drafter/release-drafter@v5 13 | env: 14 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 15 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "Trivy" 2 | 3 | on: 4 | schedule: 5 | - cron: '24 10 * * 5' 6 | 7 | jobs: 8 | build: 9 | name: Trivy vulnerability scanner 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v4 14 | 15 | - name: Run Trivy vulnerability scanner in repo mode 16 | uses: aquasecurity/trivy-action@master 17 | with: 18 | scan-type: 'fs' 19 | ignore-unfixed: true 20 | format: 'sarif' 21 | output: 'trivy-results.sarif' 22 | severity: 'CRITICAL' 23 | 24 | - name: Upload Trivy scan results to GitHub Security tab 25 | uses: github/codeql-action/upload-sarif@v2 26 | with: 27 | sarif_file: 'trivy-results.sarif' 28 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | name: "Pull Request Auto Labeler" 2 | on: 3 | - pull_request_target 4 | 5 | jobs: 6 | triage: 7 | permissions: 8 | contents: read 9 | pull-requests: write 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/labeler@v4 13 | with: 14 | repo-token: "${{ secrets.GITHUB_TOKEN }}" -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: 9 | - develop 10 | pull_request: 11 | branches: 12 | - develop 13 | 14 | jobs: 15 | build-jdk17: 16 | runs-on: ubuntu-latest 17 | name: Build project 18 | concurrency: 19 | # The commit SHA or the branch name of the pull request. See: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions 20 | group: ${{ github.event_name == 'pull_request' && github.head_ref || github.sha}} 21 | cancel-in-progress: true 22 | steps: 23 | - name: Checkout repository 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | - name: Cache Maven packages 28 | uses: actions/cache@v3 29 | with: 30 | path: ~/.m2 31 | key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} 32 | restore-keys: ${{ runner.os }}-m2 33 | - name: Set up JDK 34 | uses: actions/setup-java@v3 35 | with: 36 | distribution: 'corretto' 37 | java-version: '17' 38 | - name: Build with Maven 39 | run: ./mvnw -version && whoami && umask -S && umask a+rw && umask -S && ./mvnw clean verify -P docker-clean -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 --no-snapshot-updates --batch-mode --no-transfer-progress -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish to the Maven Central Repository 2 | 3 | on: 4 | release: 5 | types: [ published ] 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v4 13 | with: 14 | ref: ${{github.event.release.target_commitish}} 15 | token: ${{ secrets.RELEASE_PERSONAL_ACCESS_TOKEN }} 16 | 17 | - name: Set up JDK 18 | uses: actions/setup-java@v3 19 | with: 20 | distribution: 'corretto' 21 | java-version: '17' 22 | server-id: ossrh 23 | server-username: MAVEN_USERNAME 24 | server-password: MAVEN_PASSWORD 25 | gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} 26 | gpg-passphrase: MAVEN_GPG_PASSPHRASE 27 | cache: 'maven' 28 | 29 | - name: Update version 30 | if: ${{ success() }} 31 | run: ./mvnw --batch-mode --no-transfer-progress versions:set -DnewVersion=${{github.event.release.tag_name}} versions:commit 32 | 33 | - name: Publish to the Maven Central Repository 34 | if: ${{ success() }} 35 | run: ./mvnw --batch-mode --no-transfer-progress -Dgib.disable=true -P ossrh -DskipTests deploy 36 | env: 37 | MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} 38 | MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} 39 | MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} 40 | 41 | - name: Commit & Push changes 42 | if: ${{ success() }} 43 | uses: actions-js/push@master 44 | with: 45 | github_token: ${{ secrets.GITHUB_TOKEN }} 46 | message: 'Release ${{github.event.release.tag_name}}' 47 | branch: ${{ github.event.release.target_commitish }} 48 | -------------------------------------------------------------------------------- /.github/workflows/renovate.yml: -------------------------------------------------------------------------------- 1 | name: Renovate for update docker images 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | dryRun: 7 | description: "Dry-Run" 8 | default: false 9 | required: false 10 | type: boolean 11 | logLevel: 12 | description: "Log-Level" 13 | required: false 14 | default: 'debug' 15 | type: choice 16 | options: 17 | - info 18 | - warn 19 | - debug 20 | - error 21 | - fatal 22 | schedule: 23 | - cron: '0 8 * * *' 24 | 25 | jobs: 26 | renovate: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | 32 | - name: Self-hosted Renovate 33 | uses: renovatebot/github-action@v39.0.5 34 | with: 35 | configurationFile: .github/renovate/renovate.json 36 | token: ${{ secrets.RELEASE_PERSONAL_ACCESS_TOKEN }} 37 | env: 38 | DRY_RUN: ${{ inputs.dryRun || 'false' }} 39 | LOG_LEVEL: ${{ inputs.logLevel || 'debug' }} -------------------------------------------------------------------------------- /.github/workflows/required-labels.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/mheap/github-action-required-labels 2 | name: Pull Request Required Labels 3 | on: 4 | pull_request: 5 | types: [ opened, labeled, unlabeled, synchronize ] 6 | jobs: 7 | label: 8 | if: github.event.pull_request.state == 'open' 9 | runs-on: ubuntu-latest 10 | name: Verify Pull Request has labels 11 | steps: 12 | - uses: mheap/github-action-required-labels@v5 13 | with: 14 | mode: minimum 15 | count: 1 16 | labels: "breaking-change, feature, enhancement, bug, dependencies, docker-update-images, documentation, housekeeping" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | target/ 26 | !.mvn/wrapper/maven-wrapper.jar 27 | 28 | ### STS ### 29 | .apt_generated 30 | .classpath 31 | .factorypath 32 | .project 33 | .settings 34 | .springBeans 35 | 36 | ### IntelliJ IDEA ### 37 | .idea 38 | *.iws 39 | *.iml 40 | *.ipr 41 | 42 | ### NetBeans ### 43 | nbproject/private/ 44 | build/ 45 | nbbuild/ 46 | dist/ 47 | nbdist/ 48 | .nb-gradle/ 49 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlaytikaOSS/aerospike-janusgraph-storage-backend/33e0f78d707ccc6cc306ae016f498b11c5f45f0a/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip 18 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![codecov](https://codecov.io/gh/Playtika/aerospike-janusgraph-storage-backend/branch/develop/graph/badge.svg)](https://codecov.io/gh/Playtika/aerospike-janusgraph-storage-backend) 2 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76d508c67fc04544bc7270140ca8be26)](https://www.codacy.com/app/PlaytikaCodacy/aerospike-janusgraph-storage-backend?utm_source=github.com&utm_medium=referral&utm_content=Playtika/aerospike-janusgraph-storage-backend&utm_campaign=Badge_Grade) 3 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.playtika.janusgraph/aerospike-storage-backend/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.playtika.janusgraph/aerospike-storage-backend) 4 | 5 | # Aerospike storage backend for Janusgraph 6 | 7 | ## Overview 8 | 9 | Aerospike based implementation of Janusgraph storage backend. 10 | When to use: If you need horizontally scalable graph DB backed by Aerospike. 11 | 12 | ## Key features 13 | 14 | ### Emulate transactions via WAL 15 | The main difference with other traditional backends (Canssandra, Berkeley) is that Aerospike does not support transactions. 16 | On each commit Jansugraph writes batch of updates that should be applied to storage backend all together. 17 | In other case graph may become inconsistent. 18 | So we need to emulate transactional behaviour and not surprisingly made it via Write Ahead Log. 19 | We use [No Sql Batch Updater](https://github.com/Playtika/nosql-batch-updater) library to achieve this. 20 | Prior to applying updates we save all batch as one record in Aerospike separate namespace and remove this record after all updates being applied. 21 | This allows [WriteAheadLogCompleter](https://github.com/Playtika/nosql-batch-updater/blob/master/batch-updater/src/main/java/nosql/batch/update/wal/WriteAheadLogCompleter.java) 22 | that runs on each node in separate thread to finish (with configured delay) all needed updates in case of some node had died in the middle of the batch. 23 | 24 | ### Deferred locking 25 | Collects all locks that transaction needs and acquire them just in commit phase. 26 | Allows us to run all lock acquisitions in parallel. This approach caused Aerospike storage backend to be classified 27 | as _optimisticLocking_ In terms of Janusgraph DB. 28 | 29 | ## Known limitations 30 | 31 | ### Record size 32 | Janusgraph keeps vertex and all adjacent edges in one record. 33 | That makes it sensitive to max record value size in key-value storage. 34 | Aerospike record size is limited by 1Mb by default and can be increased up to 8Mb in 35 | namespace configuration. It makes sens to configure WAL namespace to use maximum value (8Mb). 36 | 37 | ### Dirty reads 38 | While emulating eventually consistent batch updates it is still possible to have dirty reads that may lead to some unwanted side effect 39 | like ghost vertices. You should try to avoid concurrent deletion and update of the same vertex. 40 | The best option is to use some external synchronization while doing such thing. 41 | 42 | ## How to run 43 | ### Embedded Mode 44 | In our microservice architecture we run Janusgraph in embedded mode. 45 | This mode uses Janusgraph and Aerospike storage backend just as library to correctly access and persist graphs in Aerospike. 46 | 47 | It allows our services to: 48 | - communicate with Janusgraph in the same JVM with minimal overheads 49 | - scale up/down Janusgraph together with the service 50 | 51 | #### Steps to introduce 52 | * Add dependency to Aerospike storage backend to your project 53 | ``` 54 | 55 | com.playtika.janusgraph 56 | aerospike-storage-backend 57 | 58 | ``` 59 | * Instantiate JanusGraph 60 | ``` 61 | ModifiableConfiguration config = buildGraphConfiguration(); 62 | config.set(STORAGE_HOSTS, new String[]{aerospikeHost}); //Aerospike host 63 | config.set(STORAGE_PORT, container.getMappedPort(aerospikePort)); 64 | config.set(STORAGE_BACKEND, "com.playtika.janusgraph.aerospike.AerospikeStoreManager"); 65 | config.set(NAMESPACE, aerospikeNamespace); 66 | config.set(WAL_NAMESPACE, walNamespace); //Aspike namespace to use for Write Ahead Log 67 | config.set(GRAPH_PREFIX, "test"); //used as prefix for Aspike sets. Allows to run several graphs in one Aspike namespace 68 | //!!! need to prevent small batches mutations as we use deferred locking approach !!! 69 | config.set(BUFFER_SIZE, AEROSPIKE_BUFFER_SIZE); 70 | config.set(TEST_ENVIRONMENT, true); //# whether we should use durable deletes (not available in community version of Aspike) 71 | config.set(ConfigOptions.SCAN_PARALLELISM, 1); //allow tu run scans in single thread only 72 | 73 | JanusGraph graph = JanusGraphFactory.open(config); 74 | ``` 75 | * Run Gremlin queries 76 | ``` 77 | graph.traversal().V().has("name", "jupiter") 78 | ``` 79 | 80 | ### Server Mode 81 | 82 | ## Benchmark 83 | 84 | | Benchmark | Mode | Cnt | Score | Error | Units | 85 | |:--- | :-: | :-: | :-: | :-: | :-: | 86 | |aerospike | thrpt | 30 | 0.106 | ± 0.004 | ops/s | 87 | |cassandra | thrpt | 30 | 0.008 | ± 0.001 | ops/s | 88 | 89 | This benchmark was run using standard 'cassandra:3.11' docker image and custom aerospike image that doesn't keep any data in memory. 90 | https://github.com/kptfh/aerospike-server.docker 91 | 92 | To run benchmarks and test on your local machine you just need to have docker installed. 93 | -------------------------------------------------------------------------------- /aerospike-benchmark/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | com.playtika.janusgraph 7 | aerospike 8 | 1.0.33 9 | 10 | 4.0.0 11 | 12 | aerospike-benchmark 13 | 14 | 15 | 16 | 17 | org.openjdk.jmh 18 | jmh-core 19 | 20 | 21 | org.openjdk.jmh 22 | jmh-generator-annprocess 23 | 24 | 25 | 26 | com.playtika.janusgraph 27 | aerospike-storage-backend 28 | 29 | 30 | 31 | com.playtika.janusgraph 32 | aerospike-container 33 | 34 | 35 | 36 | org.janusgraph 37 | janusgraph-cql 38 | 39 | 40 | 41 | org.testcontainers 42 | cassandra 43 | 44 | 45 | 46 | junit 47 | junit 48 | compile 49 | 50 | 51 | 52 | org.slf4j 53 | slf4j-api 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /aerospike-benchmark/src/main/java/com/playtika/janusgraph/aerospike/benchmark/Configurations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.benchmark; 2 | 3 | import com.aerospike.AerospikeProperties; 4 | import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; 5 | import org.testcontainers.containers.CassandraContainer; 6 | import org.testcontainers.containers.GenericContainer; 7 | 8 | import java.time.Duration; 9 | 10 | import static com.playtika.janusgraph.aerospike.AerospikeStoreManager.AEROSPIKE_BUFFER_SIZE; 11 | import static com.playtika.janusgraph.aerospike.ConfigOptions.*; 12 | import static org.janusgraph.diskstorage.cql.CQLConfigOptions.KEYSPACE; 13 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.*; 14 | 15 | public class Configurations { 16 | 17 | public static final String TEST_NAMESPACE = "TEST"; 18 | 19 | static ModifiableConfiguration getAerospikeConfiguration(GenericContainer container, AerospikeProperties properties) { 20 | 21 | ModifiableConfiguration config = buildGraphConfiguration(); 22 | config.set(STORAGE_HOSTS, new String[]{container.getContainerIpAddress()}); 23 | config.set(STORAGE_PORT, container.getMappedPort(properties.getPort())); 24 | config.set(STORAGE_BACKEND, "com.playtika.janusgraph.aerospike.AerospikeStoreManager"); 25 | config.set(NAMESPACE, properties.getNamespace()); 26 | config.set(WAL_NAMESPACE, properties.getNamespace()); 27 | config.set(GRAPH_PREFIX, "test"); 28 | //!!! need to prevent small batches mutations as we use deferred locking approach !!! 29 | config.set(BUFFER_SIZE, AEROSPIKE_BUFFER_SIZE); 30 | return config; 31 | } 32 | 33 | static ModifiableConfiguration getCQLConfiguration(CassandraContainer cassandraContainer) { 34 | final ModifiableConfiguration config = buildGraphConfiguration(); 35 | config.set(KEYSPACE, "test"); 36 | config.set(PAGE_SIZE, 500); 37 | config.set(CONNECTION_TIMEOUT, Duration.ofSeconds(60L)); 38 | config.set(STORAGE_BACKEND, "cql"); 39 | config.set(STORAGE_HOSTS, new String[]{cassandraContainer.getContainerIpAddress() 40 | +":"+cassandraContainer.getMappedPort(9042)}); 41 | config.set(DROP_ON_CLEAR, false); 42 | return config; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /aerospike-benchmark/src/main/java/com/playtika/janusgraph/aerospike/benchmark/Graph.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.benchmark; 2 | 3 | import org.apache.tinkerpop.gremlin.structure.Vertex; 4 | import org.janusgraph.core.*; 5 | import org.janusgraph.core.schema.ConsistencyModifier; 6 | import org.janusgraph.core.schema.JanusGraphIndex; 7 | import org.janusgraph.core.schema.JanusGraphManagement; 8 | 9 | import java.util.UUID; 10 | 11 | public class Graph { 12 | 13 | public static final String CREDENTIAL_TYPE = "credentialType"; 14 | public static final String CREDENTIAL_VALUE = "credentialValue"; 15 | public static final String PLATFORM_IDENTITY_PARENT_EVENT = "pltfParentEvent"; 16 | public static final String APP_IDENTITY_PARENT_EVENT = "appParentEvent"; 17 | public static final String APP_IDENTITY_ID = "appIdentityId"; 18 | 19 | 20 | static void buildRandomGraph(JanusGraph graph) { 21 | for(int i = 0; i < 1000; i++) { 22 | UUID id = UUID.randomUUID(); 23 | JanusGraphTransaction tx = graph.newTransaction(); 24 | JanusGraphVertex prentEvent = tx.addVertex(); 25 | tx.addVertex(CREDENTIAL_TYPE, "fb_" + id, CREDENTIAL_VALUE, "qd3qeqda3123dwq_" + id) 26 | .addEdge(PLATFORM_IDENTITY_PARENT_EVENT, prentEvent); 27 | tx.addVertex(CREDENTIAL_TYPE, "one_" + id, CREDENTIAL_VALUE, "dwdw@cwd.com_" + id) 28 | .addEdge(PLATFORM_IDENTITY_PARENT_EVENT, prentEvent); 29 | tx.addVertex(APP_IDENTITY_ID, "123:456_" + id).addEdge(APP_IDENTITY_PARENT_EVENT, prentEvent); 30 | tx.commit(); 31 | } 32 | } 33 | 34 | static void defineSchema(JanusGraph graph) { 35 | JanusGraphManagement management = graph.openManagement(); 36 | final PropertyKey credentialType = management.makePropertyKey(CREDENTIAL_TYPE).dataType(String.class).make(); 37 | final PropertyKey credentialValue = management.makePropertyKey(CREDENTIAL_VALUE).dataType(String.class).make(); 38 | final PropertyKey appIdentityId = management.makePropertyKey(APP_IDENTITY_ID).dataType(String.class).make(); 39 | 40 | JanusGraphIndex platformIdentityIndex = management.buildIndex("platformIdentity", Vertex.class) 41 | .addKey(credentialType).addKey(credentialValue) 42 | .unique() 43 | .buildCompositeIndex(); 44 | management.setConsistency(platformIdentityIndex, ConsistencyModifier.LOCK); 45 | 46 | JanusGraphIndex appIdentityIndex = management.buildIndex("appIdentity", Vertex.class) 47 | .addKey(appIdentityId) 48 | .unique() 49 | .buildCompositeIndex(); 50 | management.setConsistency(appIdentityIndex, ConsistencyModifier.LOCK); 51 | 52 | management.makeEdgeLabel(PLATFORM_IDENTITY_PARENT_EVENT).multiplicity(Multiplicity.MANY2ONE).make(); 53 | management.makeEdgeLabel(APP_IDENTITY_PARENT_EVENT).multiplicity(Multiplicity.ONE2ONE).make(); 54 | management.commit(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /aerospike-benchmark/src/main/java/com/playtika/janusgraph/aerospike/benchmark/JanusgraphBenchmarks.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.benchmark; 2 | 3 | import com.aerospike.AerospikeContainerUtils; 4 | import com.aerospike.AerospikeProperties; 5 | import org.janusgraph.core.JanusGraph; 6 | import org.janusgraph.core.JanusGraphFactory; 7 | import org.openjdk.jmh.annotations.Benchmark; 8 | import org.openjdk.jmh.annotations.BenchmarkMode; 9 | import org.openjdk.jmh.annotations.Fork; 10 | import org.openjdk.jmh.annotations.Measurement; 11 | import org.openjdk.jmh.annotations.Mode; 12 | import org.openjdk.jmh.annotations.OutputTimeUnit; 13 | import org.openjdk.jmh.annotations.Scope; 14 | import org.openjdk.jmh.annotations.Setup; 15 | import org.openjdk.jmh.annotations.State; 16 | import org.openjdk.jmh.annotations.TearDown; 17 | import org.openjdk.jmh.annotations.Warmup; 18 | import org.openjdk.jmh.runner.Runner; 19 | import org.openjdk.jmh.runner.RunnerException; 20 | import org.openjdk.jmh.runner.options.Options; 21 | import org.openjdk.jmh.runner.options.OptionsBuilder; 22 | import org.testcontainers.containers.CassandraContainer; 23 | import org.testcontainers.containers.GenericContainer; 24 | 25 | import java.util.concurrent.TimeUnit; 26 | 27 | import static com.playtika.janusgraph.aerospike.benchmark.Configurations.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.benchmark.Configurations.getCQLConfiguration; 29 | import static com.playtika.janusgraph.aerospike.benchmark.Graph.buildRandomGraph; 30 | import static com.playtika.janusgraph.aerospike.benchmark.Graph.defineSchema; 31 | 32 | @Measurement(iterations = 10, time = 1) 33 | @Warmup(iterations = 5, time = 1) 34 | @Fork(3) 35 | @BenchmarkMode(Mode.Throughput) 36 | @OutputTimeUnit(TimeUnit.SECONDS) 37 | @State(Scope.Benchmark) 38 | public class JanusgraphBenchmarks { 39 | 40 | private CassandraContainer cassandra; 41 | private GenericContainer aerospike; 42 | private JanusGraph aerospikeGraph; 43 | private JanusGraph cassandraGraph; 44 | 45 | @Setup 46 | public void setup() { 47 | cassandra = new CassandraContainer("cassandra:3.11"); 48 | cassandra.start(); 49 | cassandraGraph = JanusGraphFactory.open(getCQLConfiguration(cassandra)); 50 | defineSchema(cassandraGraph); 51 | 52 | AerospikeProperties properties = new AerospikeProperties(); 53 | aerospike = AerospikeContainerUtils.startAerospikeContainer(properties); 54 | aerospikeGraph = JanusGraphFactory.open(getAerospikeConfiguration(aerospike, properties)); 55 | defineSchema(aerospikeGraph); 56 | } 57 | 58 | @TearDown 59 | public void tearDown() { 60 | cassandraGraph.close(); 61 | cassandra.close(); 62 | 63 | aerospikeGraph.close(); 64 | aerospike.close(); 65 | } 66 | 67 | @Benchmark 68 | public JanusGraph cassandra() { 69 | buildRandomGraph(cassandraGraph); 70 | return cassandraGraph; 71 | } 72 | 73 | @Benchmark 74 | public JanusGraph aerospike() { 75 | buildRandomGraph(aerospikeGraph); 76 | return aerospikeGraph; 77 | } 78 | 79 | 80 | //used to run from IDE 81 | public static void main(String[] args) throws RunnerException { 82 | Options opt = new OptionsBuilder() 83 | .jvmArgs("-Xms1024m", "-Xmx4024m") 84 | .include(".*" + JanusgraphBenchmarks.class.getSimpleName() + ".*") 85 | //.addProfiler( StackProfiler.class ) 86 | .build(); 87 | 88 | new Runner(opt).run(); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /aerospike-container/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | com.playtika.janusgraph 7 | aerospike 8 | 1.0.33 9 | 10 | 4.0.0 11 | 12 | aerospike-container 13 | 14 | 15 | 16 | com.aerospike 17 | aerospike-client 18 | 19 | 20 | org.testcontainers 21 | testcontainers 22 | 23 | 24 | 25 | junit 26 | junit 27 | compile 28 | 29 | 30 | -------------------------------------------------------------------------------- /aerospike-container/src/main/java/com/aerospike/AerospikeContainerUtils.java: -------------------------------------------------------------------------------- 1 | package com.aerospike; 2 | 3 | import com.github.dockerjava.api.model.Capability; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.testcontainers.containers.GenericContainer; 7 | import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; 8 | import org.testcontainers.containers.wait.strategy.WaitAllStrategy; 9 | import org.testcontainers.containers.wait.strategy.WaitStrategy; 10 | 11 | import java.time.Duration; 12 | 13 | public class AerospikeContainerUtils { 14 | 15 | private static final Logger log = LoggerFactory.getLogger(AerospikeContainerUtils.class); 16 | 17 | public static GenericContainer startAerospikeContainer(AerospikeProperties properties){ 18 | AerospikeWaitStrategy aerospikeWaitStrategy = new AerospikeWaitStrategy(properties); 19 | 20 | log.info("Starting aerospike server. Docker image: {}", properties.dockerImage); 21 | 22 | Duration startupTimeout = Duration.ofSeconds(60); 23 | WaitStrategy waitStrategy = new WaitAllStrategy() 24 | .withStrategy(aerospikeWaitStrategy) 25 | .withStrategy(new HostPortWaitStrategy()) 26 | .withStartupTimeout(startupTimeout); 27 | 28 | GenericContainer aerospike = 29 | new GenericContainer<>(properties.dockerImage) 30 | .withExposedPorts(properties.port) 31 | .withEnv("NAMESPACE", properties.namespace) 32 | .withEnv("SERVICE_PORT", String.valueOf(properties.port)) 33 | .withEnv("MEM_GB", String.valueOf(1)) 34 | .withEnv("STORAGE_GB", String.valueOf(1)) 35 | .withCreateContainerCmdModifier(cmd -> cmd.withCapAdd(Capability.NET_ADMIN)) 36 | .waitingFor(waitStrategy) 37 | .withStartupTimeout(startupTimeout); 38 | 39 | aerospike.start(); 40 | configureEnterpriseServer(properties, aerospike); 41 | return aerospike; 42 | } 43 | 44 | private static void configureEnterpriseServer(AerospikeProperties properties, 45 | GenericContainer aerospikeContainer) { 46 | AsadmCommandExecutor asadmCommandExecutor = new AsadmCommandExecutor(aerospikeContainer); 47 | String namespace = properties.getNamespace(); 48 | /* 49 | By default, the value of this metric is 90%, we set it to 100% to prevent stopping writes for the Aerospike 50 | Enterprise container during high consumption of system memory. For the Aerospike Community Edition, this metric is not used. 51 | Documentation: https://aerospike.com/docs/server/reference/configuration#stop-writes-sys-memory-pct 52 | */ 53 | log.info("Switching off 'stop-writes-sys-memory-pct'... "); 54 | asadmCommandExecutor.execute(String.format("manage config namespace %s param stop-writes-sys-memory-pct to 100", namespace)); 55 | log.info("Success switching off 'stop-writes-sys-memory-pct'"); 56 | 57 | if (properties.isDurableDelete()) { 58 | log.info("Setting up 'disallow-expunge' to true..."); 59 | asadmCommandExecutor.execute(String.format("manage config namespace %s param disallow-expunge to true", namespace)); 60 | log.info("Success setting up 'disallow-expunge' to true"); 61 | } 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /aerospike-container/src/main/java/com/aerospike/AerospikeProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (c) 2018 Playtika 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | package com.aerospike; 25 | 26 | public class AerospikeProperties { 27 | 28 | boolean enabled = true; 29 | String dockerImage = "aerospike/aerospike-server-enterprise:6.3.0.16_1"; 30 | String namespace = "TEST"; 31 | String host = "localhost"; 32 | int port = 3000; 33 | boolean durableDelete = true; 34 | 35 | public boolean isEnabled() { 36 | return enabled; 37 | } 38 | 39 | public void setEnabled(boolean enabled) { 40 | this.enabled = enabled; 41 | } 42 | 43 | public String getDockerImage() { 44 | return dockerImage; 45 | } 46 | 47 | public void setDockerImage(String dockerImage) { 48 | this.dockerImage = dockerImage; 49 | } 50 | 51 | public String getNamespace() { 52 | return namespace; 53 | } 54 | 55 | public void setNamespace(String namespace) { 56 | this.namespace = namespace; 57 | } 58 | 59 | public String getHost() { 60 | return host; 61 | } 62 | 63 | public void setHost(String host) { 64 | this.host = host; 65 | } 66 | 67 | public int getPort() { 68 | return port; 69 | } 70 | 71 | public void setPort(int port) { 72 | this.port = port; 73 | } 74 | 75 | public boolean isDurableDelete() { 76 | return durableDelete; 77 | } 78 | 79 | public void setDurableDelete(boolean durableDelete) { 80 | this.durableDelete = durableDelete; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /aerospike-container/src/main/java/com/aerospike/AerospikeWaitStrategy.java: -------------------------------------------------------------------------------- 1 | package com.aerospike; 2 | 3 | import com.aerospike.client.AerospikeClient; 4 | import com.aerospike.client.AerospikeException; 5 | import com.github.dockerjava.api.command.InspectContainerResponse; 6 | import com.github.dockerjava.api.model.ExposedPort; 7 | import com.github.dockerjava.api.model.NetworkSettings; 8 | import com.github.dockerjava.api.model.Ports; 9 | import org.rnorth.ducttape.TimeoutException; 10 | import org.rnorth.ducttape.unreliables.Unreliables; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.testcontainers.DockerClientFactory; 14 | import org.testcontainers.containers.ContainerLaunchException; 15 | import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; 16 | 17 | import java.util.Map; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | public class AerospikeWaitStrategy extends AbstractWaitStrategy { 21 | 22 | private final Logger log = LoggerFactory.getLogger(getClass()); 23 | 24 | private final AerospikeProperties properties; 25 | 26 | public AerospikeWaitStrategy(AerospikeProperties properties) { 27 | this.properties = properties; 28 | } 29 | 30 | @Override 31 | protected void waitUntilReady() { 32 | long seconds = this.startupTimeout.getSeconds(); 33 | 34 | try { 35 | Unreliables.retryUntilTrue((int)seconds, TimeUnit.SECONDS, () -> { 36 | return (Boolean)this.getRateLimiter().getWhenReady(this::isReady); 37 | }); 38 | } catch (TimeoutException var4) { 39 | throw new ContainerLaunchException(String.format("[%s] notifies that container[%s] is not ready after [%d] seconds, container cannot be started.", this.getContainerType(), this.waitStrategyTarget.getContainerId(), seconds)); 40 | } 41 | } 42 | 43 | protected boolean isReady() { 44 | String containerId = waitStrategyTarget.getContainerId(); 45 | log.debug("Check Aerospike container {} status", containerId); 46 | 47 | InspectContainerResponse containerInfo = waitStrategyTarget.getContainerInfo(); 48 | if (containerInfo == null) { 49 | log.debug("Aerospike container[{}] doesn't contain info. Abnormal situation, should not happen.", containerId); 50 | return false; 51 | } 52 | 53 | int port = getMappedPort(containerInfo.getNetworkSettings(), properties.port); 54 | String host = DockerClientFactory.instance().dockerHostIpAddress(); 55 | 56 | //TODO: Remove dependency to client https://www.aerospike.com/docs/tools/asmonitor/common_tasks.html 57 | try (AerospikeClient client = new AerospikeClient(host, port)) { 58 | return client.isConnected(); 59 | } catch (AerospikeException.Connection e) { 60 | log.debug("Aerospike container: {} not yet started. {}", containerId, e.getMessage()); 61 | } 62 | return false; 63 | } 64 | 65 | private int getMappedPort(NetworkSettings networkSettings, int originalPort) { 66 | ExposedPort exposedPort = new ExposedPort(originalPort); 67 | Ports ports = networkSettings.getPorts(); 68 | Map bindings = ports.getBindings(); 69 | Ports.Binding[] binding = bindings.get(exposedPort); 70 | return Integer.valueOf(binding[0].getHostPortSpec()); 71 | } 72 | 73 | protected String getContainerType() { 74 | return this.getClass().getSimpleName(); 75 | } 76 | } -------------------------------------------------------------------------------- /aerospike-container/src/main/java/com/aerospike/AsadmCommandExecutor.java: -------------------------------------------------------------------------------- 1 | package com.aerospike; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.testcontainers.containers.Container; 6 | import org.testcontainers.containers.GenericContainer; 7 | 8 | public class AsadmCommandExecutor { 9 | 10 | private static final Logger log = LoggerFactory.getLogger(AsadmCommandExecutor.class); 11 | 12 | private final GenericContainer aerospikeContainer; 13 | 14 | public AsadmCommandExecutor(GenericContainer aerospikeContainer) { 15 | this.aerospikeContainer = aerospikeContainer; 16 | } 17 | 18 | public void execute(String command) { 19 | try { 20 | Container.ExecResult result = aerospikeContainer.execInContainer("asadm", "--enable", "-e", command); 21 | logStdout(result); 22 | if (result.getExitCode() != 0 || isBadResponse(result)) { 23 | throw new IllegalStateException(String.format("Failed to execute \"asadm --enable -e '%s'\": \nstdout:\n%s\nstderr:\n%s", 24 | command, result.getStdout(), result.getStderr())); 25 | } 26 | } catch (Exception ex) { 27 | throw new IllegalStateException(String.format("Failed to execute \"asadm\"", ex)); 28 | } 29 | } 30 | 31 | private boolean isBadResponse(Container.ExecResult execResult) { 32 | String stdout = execResult.getStdout(); 33 | /* 34 | Example of the stdout without error: 35 | ~Set Namespace Param stop-writes-sys-memory-pct to 100~ 36 | Node|Response 37 | 728bb242e58c:3000|ok 38 | Number of rows: 1 39 | */ 40 | return !stdout.contains("|ok"); 41 | } 42 | 43 | private static void logStdout(Container.ExecResult result) { 44 | log.debug("Aerospike asadm util stdout: \n{}\n{}", result.getStdout(), result.getStderr()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /aerospike-storage-backend/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 18 | 4.0.0 19 | 20 | 21 | com.playtika.janusgraph 22 | aerospike 23 | 1.0.33 24 | 25 | 26 | aerospike-storage-backend 27 | jar 28 | Aerospike storage backend for Janusgraph graph DB/Decoder 29 | 30 | 31 | 32 | io.dropwizard.metrics 33 | metrics-core 34 | 35 | 36 | io.dropwizard.metrics 37 | metrics-jvm 38 | 39 | 40 | io.dropwizard.metrics 41 | metrics-graphite 42 | 43 | 44 | io.dropwizard.metrics 45 | metrics-jmx 46 | 47 | 48 | 49 | org.janusgraph 50 | janusgraph-core 51 | provided 52 | 53 | 54 | io.dropwizard.metrics 55 | metrics-core 56 | 57 | 58 | io.dropwizard.metrics 59 | metrics-jvm 60 | 61 | 62 | io.dropwizard.metrics 63 | metrics-graphite 64 | 65 | 66 | io.dropwizard.metrics 67 | metrics-jmx 68 | 69 | 70 | io.dropwizard.metrics 71 | metrics-ganglia 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.apache.tinkerpop 79 | gremlin-core 80 | provided 81 | 82 | 83 | org.apache.tinkerpop 84 | gremlin-server 85 | provided 86 | 87 | 88 | 89 | com.aerospike 90 | aerospike-client 91 | 92 | 93 | 94 | org.slf4j 95 | slf4j-api 96 | provided 97 | 98 | 99 | 100 | com.playtika.nosql 101 | aerospike-batch-updater 102 | 103 | 104 | 105 | io.netty 106 | netty-all 107 | 108 | 109 | 110 | 111 | 112 | org.janusgraph 113 | janusgraph-test 114 | test 115 | 116 | 117 | 118 | com.codahale.metrics 119 | metrics-core 120 | 121 | 122 | com.codahale.metrics 123 | metrics-jvm 124 | 125 | 126 | com.codahale.metrics 127 | metrics-graphite 128 | 129 | 130 | com.codahale.metrics 131 | metrics-jmx 132 | 133 | 134 | com.codahale.metrics 135 | metrics-ganglia 136 | 137 | 138 | 139 | 140 | 141 | com.playtika.janusgraph 142 | aerospike-container 143 | ${project.version} 144 | test 145 | 146 | 147 | 148 | junit 149 | junit 150 | test 151 | 152 | 153 | 154 | org.assertj 155 | assertj-core 156 | test 157 | 158 | 159 | 160 | org.apache.logging.log4j 161 | log4j-slf4j-impl 162 | test 163 | 164 | 165 | 166 | org.awaitility 167 | awaitility 168 | test 169 | 170 | 171 | 172 | 173 | io.projectreactor.tools 174 | blockhound 175 | test 176 | 177 | 178 | 179 | com.lmax 180 | disruptor 181 | test 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/AerospikeKeyColumnValueStore.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.aerospike.client.Value; 4 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 5 | import com.playtika.janusgraph.aerospike.operations.ErrorMapper; 6 | import com.playtika.janusgraph.aerospike.operations.IdsCleanupOperations; 7 | import com.playtika.janusgraph.aerospike.operations.MutateOperations; 8 | import com.playtika.janusgraph.aerospike.operations.ReadOperations; 9 | import com.playtika.janusgraph.aerospike.operations.ScanOperations; 10 | import com.playtika.janusgraph.aerospike.operations.batch.BatchLocks; 11 | import com.playtika.janusgraph.aerospike.operations.batch.BatchUpdate; 12 | import com.playtika.janusgraph.aerospike.operations.batch.BatchUpdates; 13 | import nosql.batch.update.BatchUpdater; 14 | import nosql.batch.update.aerospike.lock.AerospikeLock; 15 | import org.janusgraph.diskstorage.BackendException; 16 | import org.janusgraph.diskstorage.Entry; 17 | import org.janusgraph.diskstorage.EntryList; 18 | import org.janusgraph.diskstorage.StaticBuffer; 19 | import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation; 20 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore; 21 | import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; 22 | import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery; 23 | import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery; 24 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 25 | import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.HashMap; 30 | import java.util.List; 31 | import java.util.Map; 32 | 33 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.getValue; 34 | import static java.util.Collections.emptyMap; 35 | import static java.util.Collections.singleton; 36 | import static java.util.Collections.singletonMap; 37 | 38 | public class AerospikeKeyColumnValueStore implements KeyColumnValueStore { 39 | 40 | private static final Logger logger = LoggerFactory.getLogger(AerospikeKeyColumnValueStore.class); 41 | 42 | private final String storeName; 43 | private final ReadOperations readOperations; 44 | private final AerospikeOperations aerospikeOperations; 45 | private final BatchUpdater batchUpdater; 46 | private final MutateOperations mutateOperations; 47 | private final ScanOperations scanOperations; 48 | private final IdsCleanupOperations idsCleanupOperations; 49 | 50 | protected AerospikeKeyColumnValueStore( 51 | String storeName, 52 | ReadOperations readOperations, 53 | AerospikeOperations aerospikeOperations, 54 | BatchUpdater batchUpdater, 55 | MutateOperations mutateOperations, 56 | ScanOperations scanOperations, 57 | IdsCleanupOperations idsCleanupOperations) { 58 | this.storeName = storeName; 59 | this.readOperations = readOperations; 60 | this.aerospikeOperations = aerospikeOperations; 61 | this.batchUpdater = batchUpdater; 62 | this.mutateOperations = mutateOperations; 63 | this.scanOperations = scanOperations; 64 | this.idsCleanupOperations = idsCleanupOperations; 65 | } 66 | 67 | @Override // This method is only supported by stores which keep keys in byte-order. 68 | public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) { 69 | throw new UnsupportedOperationException(); 70 | } 71 | 72 | /** 73 | * Except scan operations may be used by janusgraph to add new index on existing graph 74 | */ 75 | @Override // This method is only supported by stores which do not keep keys in byte-order. 76 | public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) { 77 | logger.trace("getKeys({}, tx:{}, {})", storeName, txh, query); 78 | 79 | return scanOperations.getKeys(storeName, query, txh); 80 | } 81 | 82 | @Override 83 | public Map getSlice(List keys, SliceQuery query, StoreTransaction txh) throws BackendException { 84 | logger.trace("getSlice({}, tx:{}, {}, start:{}, end:{})", 85 | storeName, txh, keys, query.getSliceStart(), query.getSliceEnd()); 86 | 87 | return readOperations.getSlice(storeName, keys, query); 88 | } 89 | 90 | @Override 91 | public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException{ 92 | logger.trace("getSlice({}, tx:{}, {})", storeName, txh, query); 93 | 94 | return readOperations.getSlice(storeName, query); 95 | } 96 | 97 | @Override 98 | public void mutate(StaticBuffer key, List additions, List deletions, StoreTransaction txh) throws BackendException { 99 | logger.trace("mutate({}, tx:{}, {}, {}, {})", storeName, txh, key, additions, deletions); 100 | 101 | AerospikeTransaction transaction = (AerospikeTransaction)txh; 102 | 103 | Map mutationMap = mutationToMap(new KCVMutation(additions, deletions)); 104 | Value keyValue = getValue(key); 105 | 106 | try { 107 | if (transaction.getLocks().isEmpty()) { 108 | //no need in transactional logic 109 | mutateOperations.mutate(storeName, keyValue, mutationMap); 110 | } else { 111 | updateBatch(keyValue, mutationMap, transaction); 112 | } 113 | } finally { 114 | if(idsCleanupOperations != null) { 115 | idsCleanupOperations.cleanUpOldIdsRanges(key); 116 | } 117 | } 118 | } 119 | 120 | private void updateBatch(Value keyValue, Map mutationMap, AerospikeTransaction transaction) throws BackendException { 121 | Map>> locksByStore = transaction.getLocksByStoreKeyColumn(); 122 | if(!singleton(storeName).containsAll(locksByStore.keySet())){ 123 | throw new IllegalArgumentException(); 124 | } 125 | 126 | Map> locks = locksByStore.getOrDefault(storeName, emptyMap()); 127 | 128 | //expect that locks contains key 129 | if(!singleton(keyValue).containsAll(locks.keySet())){ 130 | throw new IllegalArgumentException(); 131 | } 132 | 133 | Map>> mutationsByStore = singletonMap(storeName, 134 | singletonMap(keyValue, mutationMap)); 135 | 136 | try { 137 | batchUpdater.update(new BatchUpdate( 138 | new BatchLocks(locksByStore, aerospikeOperations), 139 | new BatchUpdates(mutationsByStore))); 140 | } catch (Throwable t) { 141 | throw ErrorMapper.INSTANCE.apply(t); 142 | } 143 | 144 | transaction.close(); 145 | } 146 | 147 | public static Map mutationToMap(KCVMutation mutation){ 148 | Map map = new HashMap<>(mutation.getAdditions().size() + mutation.getDeletions().size()); 149 | for(StaticBuffer deletion : mutation.getDeletions()){ 150 | map.put(getValue(deletion), Value.NULL); 151 | } 152 | 153 | for(Entry addition : mutation.getAdditions()){ 154 | map.put(getValue(addition.getColumn()), getValue(addition.getValue())); 155 | } 156 | return map; 157 | } 158 | 159 | @Override 160 | public void acquireLock(final StaticBuffer key, final StaticBuffer column, final StaticBuffer expectedValue, final StoreTransaction txh) { 161 | //deferred locking approach 162 | //just add lock to transaction, actual lock will be acquired at commit phase 163 | ((AerospikeTransaction)txh).addLock(new DeferredLock(storeName, key, column, expectedValue)); 164 | logger.trace("registered lock: {}:{}:{}:{}, tx:{}", storeName, key, column, expectedValue, txh); 165 | } 166 | 167 | @Override 168 | public synchronized void close() {} 169 | 170 | @Override 171 | public String getName() { 172 | return storeName; 173 | } 174 | 175 | 176 | 177 | } 178 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/AerospikePolicyProvider.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.aerospike.client.async.EventLoops; 4 | import com.aerospike.client.async.NettyEventLoops; 5 | import com.aerospike.client.policy.BatchPolicy; 6 | import com.aerospike.client.policy.ClientPolicy; 7 | import com.aerospike.client.policy.Policy; 8 | import com.aerospike.client.policy.QueryPolicy; 9 | import com.aerospike.client.policy.Replica; 10 | import com.aerospike.client.policy.ScanPolicy; 11 | import com.aerospike.client.policy.WritePolicy; 12 | import io.netty.channel.nio.NioEventLoopGroup; 13 | import org.janusgraph.diskstorage.configuration.Configuration; 14 | 15 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_CLIENT_MAX_SOCKET_IDLE; 16 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_CONNECTIONS_PER_NODE; 17 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_MIN_CONNECTIONS_PER_NODE; 18 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_READ_TIMEOUT; 19 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_SOCKET_TIMEOUT; 20 | import static com.playtika.janusgraph.aerospike.ConfigOptions.AEROSPIKE_WRITE_TIMEOUT; 21 | import static com.playtika.janusgraph.aerospike.ConfigOptions.TEST_ENVIRONMENT; 22 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.AUTH_PASSWORD; 23 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.AUTH_USERNAME; 24 | 25 | public class AerospikePolicyProvider { 26 | 27 | public static final int NO_RETRIES = 0; 28 | private final Configuration configuration; 29 | private EventLoops eventLoops; 30 | 31 | public AerospikePolicyProvider(Configuration configuration) { 32 | this.configuration = configuration; 33 | } 34 | 35 | public ClientPolicy clientPolicy() { 36 | ClientPolicy clientPolicy = new ClientPolicy(); 37 | clientPolicy.user = configuration.has(AUTH_USERNAME) ? configuration.get(AUTH_USERNAME) : null; 38 | clientPolicy.password = configuration.has(AUTH_PASSWORD) ? configuration.get(AUTH_PASSWORD) : null; 39 | clientPolicy.minConnsPerNode = configuration.get(AEROSPIKE_MIN_CONNECTIONS_PER_NODE); 40 | clientPolicy.maxConnsPerNode = configuration.get(AEROSPIKE_CONNECTIONS_PER_NODE); 41 | clientPolicy.maxSocketIdle = configuration.has(AEROSPIKE_CLIENT_MAX_SOCKET_IDLE) 42 | ? configuration.get(AEROSPIKE_CLIENT_MAX_SOCKET_IDLE) 43 | : clientPolicy.maxSocketIdle; 44 | clientPolicy.readPolicyDefault = readPolicy(); 45 | clientPolicy.scanPolicyDefault = scanPolicy(); 46 | clientPolicy.queryPolicyDefault = queryPolicy(); 47 | clientPolicy.writePolicyDefault = writePolicy(); 48 | clientPolicy.batchPolicyDefault = batchPolicy(); 49 | clientPolicy.eventLoops = eventLoops(); 50 | return clientPolicy; 51 | } 52 | 53 | public BatchPolicy batchPolicy() { 54 | BatchPolicy batchPolicy = new BatchPolicy(); 55 | batchPolicy.replica = Replica.MASTER; 56 | batchPolicy.totalTimeout = configuration.get(AEROSPIKE_WRITE_TIMEOUT); 57 | batchPolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 58 | batchPolicy.respondAllKeys = true; 59 | return batchPolicy; 60 | } 61 | 62 | public QueryPolicy queryPolicy() { 63 | QueryPolicy queryPolicy = new QueryPolicy(); 64 | queryPolicy.totalTimeout = configuration.get(AEROSPIKE_READ_TIMEOUT); 65 | queryPolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 66 | queryPolicy.maxRetries = NO_RETRIES; 67 | return queryPolicy; 68 | } 69 | 70 | public WritePolicy writePolicy() { 71 | WritePolicy writePolicy = new WritePolicy(); 72 | writePolicy.sendKey = true; 73 | writePolicy.expiration = -1; 74 | writePolicy.totalTimeout = configuration.get(AEROSPIKE_WRITE_TIMEOUT); 75 | writePolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 76 | writePolicy.maxRetries = NO_RETRIES; 77 | writePolicy.durableDelete = !configuration.get(TEST_ENVIRONMENT); 78 | return writePolicy; 79 | } 80 | 81 | public WritePolicy deletePolicy() { 82 | WritePolicy deletePolicy = new WritePolicy(); 83 | deletePolicy.expiration = -1; 84 | deletePolicy.totalTimeout = configuration.get(AEROSPIKE_WRITE_TIMEOUT); 85 | deletePolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 86 | deletePolicy.durableDelete = !configuration.get(TEST_ENVIRONMENT); 87 | deletePolicy.maxRetries = NO_RETRIES; 88 | return deletePolicy; 89 | } 90 | 91 | public Policy readPolicy() { 92 | Policy readPolicy = new Policy(); 93 | readPolicy.replica = Replica.MASTER; 94 | readPolicy.sendKey = true; 95 | readPolicy.totalTimeout = configuration.get(AEROSPIKE_READ_TIMEOUT); 96 | readPolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 97 | readPolicy.maxRetries = NO_RETRIES; 98 | return readPolicy; 99 | } 100 | 101 | public ScanPolicy scanPolicy() { 102 | ScanPolicy scanPolicy = new ScanPolicy(); 103 | scanPolicy.sendKey = true; 104 | scanPolicy.includeBinData = true; 105 | scanPolicy.socketTimeout = configuration.get(AEROSPIKE_SOCKET_TIMEOUT); 106 | scanPolicy.totalTimeout = configuration.get(AEROSPIKE_READ_TIMEOUT); 107 | return scanPolicy; 108 | } 109 | 110 | public EventLoops eventLoops(){ 111 | if(eventLoops ==null){ 112 | eventLoops = new NettyEventLoops(new NioEventLoopGroup()); 113 | } 114 | return eventLoops; 115 | } 116 | 117 | public void close(){ 118 | eventLoops.close(); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/AerospikeTransaction.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.aerospike.client.Value; 4 | import org.janusgraph.diskstorage.BaseTransactionConfig; 5 | import org.janusgraph.diskstorage.common.AbstractStoreTransaction; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.function.BiConsumer; 11 | import java.util.stream.Collectors; 12 | 13 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.getValue; 14 | 15 | final class AerospikeTransaction extends AbstractStoreTransaction { 16 | 17 | private List locks = new ArrayList<>(); 18 | 19 | AerospikeTransaction(final BaseTransactionConfig config) { 20 | super(config); 21 | } 22 | 23 | @Override 24 | public void commit() { 25 | close(); 26 | } 27 | 28 | @Override 29 | public void rollback() { 30 | locks.clear(); 31 | } 32 | 33 | public void close(){ 34 | locks = null; 35 | } 36 | 37 | void addLock(DeferredLock lock){ 38 | locks.add(lock); 39 | } 40 | 41 | List getLocks() { 42 | return locks; 43 | } 44 | 45 | Map>> getLocksByStoreKeyColumn(){ 46 | return locks.stream() 47 | .collect(Collectors.groupingBy(lock -> lock.storeName, 48 | Collectors.groupingBy(lock -> getValue(lock.key), 49 | Collectors.toMap( 50 | lock -> getValue(lock.column), 51 | lock -> lock.expectedValue != null ? getValue(lock.expectedValue) : Value.NULL, 52 | (oldValue, newValue) -> oldValue)))); 53 | } 54 | 55 | @Override 56 | public String toString(){ 57 | return Integer.toHexString(hashCode()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/ConfigOptions.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import org.janusgraph.diskstorage.configuration.ConfigOption; 4 | 5 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_NS; 6 | 7 | public class ConfigOptions { 8 | 9 | public static final ConfigOption GRAPH_PREFIX = new ConfigOption<>(STORAGE_NS, 10 | "graph-prefix", "Graph prefix, allows to keep several graph in one namespace", 11 | ConfigOption.Type.LOCAL, String.class); 12 | 13 | public static final ConfigOption NAMESPACE = new ConfigOption<>(STORAGE_NS, 14 | "namespace", "Aerospike namespace to use", ConfigOption.Type.LOCAL, String.class); 15 | 16 | public static final ConfigOption IDS_NAMESPACE = new ConfigOption<>(STORAGE_NS, 17 | "ids-namespace", "Aerospike namespace to use for Ids", ConfigOption.Type.LOCAL, String.class); 18 | 19 | public static final ConfigOption WAL_NAMESPACE = new ConfigOption<>(STORAGE_NS, 20 | "wal-namespace", "Aerospike namespace to use for write ahead log", 21 | ConfigOption.Type.LOCAL, String.class); 22 | 23 | public static final ConfigOption SCAN_PARALLELISM = new ConfigOption<>(STORAGE_NS, 24 | "scan-parallelism", "How many threads may perform scan operations simultaneously. " + 25 | "Should be greater then zero if you want to enable scan feature", 26 | ConfigOption.Type.LOCAL, 0); 27 | 28 | public static final ConfigOption WAL_STALE_TRANSACTION_LIFETIME_THRESHOLD = new ConfigOption<>(STORAGE_NS, 29 | "wal-threshold", "After this period of time (in ms) transaction in WAL considered to be stale " + 30 | "and can be re-processed", 31 | ConfigOption.Type.LOCAL, 600000L); 32 | 33 | public static final ConfigOption WAL_MAX_BATCH_SIZE = new ConfigOption<>(STORAGE_NS, 34 | "wal-max-batch-size", "Max batch size which completer will process in-memory", 35 | ConfigOption.Type.LOCAL, 5000); 36 | 37 | public static final ConfigOption TEST_ENVIRONMENT = new ConfigOption<>(STORAGE_NS, 38 | "test-environment", "Weather this production or test environment", 39 | ConfigOption.Type.LOCAL, false); 40 | 41 | public static final ConfigOption START_WAL_COMPLETER = new ConfigOption<>(STORAGE_NS, 42 | "start-wal-completer", "Whether WAL Completer should be started inside store manager. " + 43 | "You should not start WAL Completer in passive data center. " + 44 | "You may consider to start WAL Completer externally than you will be able to suspend and resume it manually", 45 | ConfigOption.Type.LOCAL, true); 46 | 47 | public static final ConfigOption AEROSPIKE_MIN_CONNECTIONS_PER_NODE= new ConfigOption<>(STORAGE_NS, 48 | "aerospike-min-connections-per-node", "Minimum number of connections on start-up per server node", 49 | ConfigOption.Type.LOCAL, 10); 50 | 51 | public static final ConfigOption AEROSPIKE_CONNECTIONS_PER_NODE= new ConfigOption<>(STORAGE_NS, 52 | "aerospike-connections-per-node", "Limits how many connections aerospike can hold per node", 53 | ConfigOption.Type.LOCAL, 300); 54 | 55 | public static final ConfigOption AEROSPIKE_EXECUTOR_MAX_THREADS = new ConfigOption<>(STORAGE_NS, 56 | "aerospike-executor-max-threads", "Number of max threads in Aerospike executor", 57 | ConfigOption.Type.LOCAL, 600); 58 | 59 | public static final ConfigOption PARALLEL_READ_THRESHOLD = new ConfigOption<>(STORAGE_NS, 60 | "parallel-read-threshold", "Number of keys when we should start run reads in parallel", 61 | ConfigOption.Type.LOCAL, 1); 62 | 63 | public static final ConfigOption AEROSPIKE_READ_TIMEOUT = new ConfigOption<>(STORAGE_NS, 64 | "aerospike-read-timeout", "Total transaction timeout in milliseconds to aerospike read operations." + 65 | "If timeout is zero, there will be no time limit", 66 | ConfigOption.Type.LOCAL, 0); 67 | 68 | public static final ConfigOption AEROSPIKE_WRITE_TIMEOUT = new ConfigOption<>(STORAGE_NS, 69 | "aerospike-write-timeout", "Total transaction timeout in milliseconds to aerospike write operations." + 70 | "If timeout is zero, there will be no time limit", 71 | ConfigOption.Type.LOCAL, 0); 72 | 73 | public static final ConfigOption AEROSPIKE_SOCKET_TIMEOUT = new ConfigOption<>(STORAGE_NS, 74 | "aerospike-socket-timeout", "Socket idle timeout in milliseconds when processing a database command." + 75 | "If timeout is zero, there will be no time limit", 76 | ConfigOption.Type.LOCAL, 0); 77 | 78 | public static final ConfigOption CHECK_ALL_MUTATIONS_LOCKED = new ConfigOption<>(STORAGE_NS, 79 | "check-all-mutations-locked", "Checks that all mutations are locked", 80 | ConfigOption.Type.LOCAL, false); 81 | 82 | public static final ConfigOption IDS_BLOCK_TTL = new ConfigOption<>(STORAGE_NS, 83 | "ids-block-ttl", "How long keep history of ids block in milliseconds, default to one week", 84 | ConfigOption.Type.LOCAL, 604800000L); 85 | 86 | public static final ConfigOption AEROSPIKE_CLIENT_MAX_SOCKET_IDLE = new ConfigOption<>(STORAGE_NS, 87 | "aerospike-client-max-socket-idle", "Maximum socket idle in seconds. If server's proto-fd-idle-ms is zero (no reap), then maxSocketIdle should also be zero", 88 | ConfigOption.Type.LOCAL, 0); 89 | 90 | } 91 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/DeferredLock.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import org.janusgraph.diskstorage.StaticBuffer; 4 | 5 | final class DeferredLock { 6 | 7 | final String storeName; 8 | final StaticBuffer key; 9 | final StaticBuffer column; 10 | final StaticBuffer expectedValue; 11 | 12 | DeferredLock(String storeName, StaticBuffer key, StaticBuffer column, StaticBuffer expectedValue) { 13 | this.storeName = storeName; 14 | this.key = key; 15 | this.column = column; 16 | this.expectedValue = expectedValue; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/AerospikeKeyIterator.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeException; 4 | import com.aerospike.client.Key; 5 | import com.aerospike.client.Record; 6 | import com.aerospike.client.ScanCallback; 7 | import org.janusgraph.diskstorage.Entry; 8 | import org.janusgraph.diskstorage.StaticBuffer; 9 | import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; 10 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 11 | import org.janusgraph.diskstorage.util.RecordIterator; 12 | import org.janusgraph.diskstorage.util.StaticArrayBuffer; 13 | import org.janusgraph.diskstorage.util.StaticArrayEntry; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.Iterator; 19 | import java.util.Map; 20 | import java.util.NoSuchElementException; 21 | import java.util.concurrent.BlockingQueue; 22 | import java.util.concurrent.LinkedBlockingQueue; 23 | import java.util.concurrent.atomic.AtomicBoolean; 24 | 25 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 26 | 27 | /** 28 | * Should be used for test purposes only 29 | */ 30 | public class AerospikeKeyIterator implements KeyIterator, ScanCallback { 31 | 32 | private static Logger logger = LoggerFactory.getLogger(AerospikeKeyIterator.class); 33 | 34 | private final SliceQuery query; 35 | private final BlockingQueue queue = new LinkedBlockingQueue<>(100); 36 | private KeyRecord next; 37 | private Iterator entriesIt; 38 | private final AtomicBoolean closed = new AtomicBoolean(false); 39 | private Thread thread; 40 | 41 | private static final KeyRecord TERMINATE_VALUE = new KeyRecord(null, null); 42 | 43 | public AerospikeKeyIterator(SliceQuery query) { 44 | this.query = query; 45 | } 46 | 47 | @Override 48 | public RecordIterator getEntries() { 49 | 50 | return new RecordIterator() { 51 | @Override 52 | public boolean hasNext() { 53 | return entriesIt.hasNext(); 54 | } 55 | 56 | @Override 57 | public Entry next() { 58 | return entriesIt.next(); 59 | } 60 | 61 | @Override 62 | public void close() { 63 | } 64 | }; 65 | } 66 | 67 | @Override 68 | public void close() { 69 | closed.set(true); 70 | try { 71 | queue.put(TERMINATE_VALUE); 72 | thread.interrupt(); 73 | } catch (InterruptedException e) { 74 | throw new RuntimeException(); 75 | } 76 | } 77 | 78 | @Override 79 | public boolean hasNext() { 80 | try { 81 | if(next == TERMINATE_VALUE){ 82 | return false; 83 | } 84 | 85 | return next != null || (next = takeNext()) != TERMINATE_VALUE; 86 | } catch (InterruptedException e) { 87 | throw new RuntimeException(e); 88 | } 89 | } 90 | 91 | @Override 92 | public StaticBuffer next() { 93 | if(next == null){ 94 | try { 95 | next = takeNext(); 96 | } catch (InterruptedException e) { 97 | throw new RuntimeException(e); 98 | } 99 | } 100 | if(next == TERMINATE_VALUE){ 101 | throw new NoSuchElementException(); 102 | } 103 | try { 104 | return keyToBuffer(next.key); 105 | } finally { 106 | next = null; 107 | } 108 | } 109 | 110 | private KeyRecord takeNext() throws InterruptedException { 111 | do { 112 | next = queue.take(); 113 | if (next == TERMINATE_VALUE) { 114 | entriesIt = null; 115 | break; 116 | } 117 | entriesIt = entriesIt(next); 118 | } while(!entriesIt.hasNext()); 119 | 120 | return next; 121 | } 122 | 123 | private Iterator entriesIt(KeyRecord keyRecord){ 124 | return keyRecord.record.getMap(ENTRIES_BIN_NAME).entrySet().stream() 125 | .map(o -> { 126 | Map.Entry entry = (Map.Entry)o; 127 | final StaticBuffer column = StaticArrayBuffer.of(entry.getKey()); 128 | final StaticBuffer value = StaticArrayBuffer.of(entry.getValue()); 129 | return StaticArrayEntry.of(column, value); 130 | }) 131 | .filter(entry -> query.contains(entry.getColumn())) 132 | .limit(query.getLimit()) 133 | .iterator(); 134 | } 135 | 136 | @Override 137 | public void scanCallback(Key key, Record record) throws AerospikeException { 138 | if (closed.get()) { 139 | logger.info("AerospikeKeyIterator get closed, terminate scan"); 140 | throw new AerospikeException.ScanTerminated(); 141 | } 142 | try { 143 | queue.put(new KeyRecord(key, record)); 144 | } catch (InterruptedException e) { 145 | throw new RuntimeException(e); 146 | } 147 | } 148 | 149 | private static StaticArrayBuffer keyToBuffer(Key key){ 150 | return new StaticArrayBuffer((byte[]) key.userKey.getObject()); 151 | } 152 | 153 | private static class KeyRecord { 154 | final Key key; 155 | final Record record; 156 | 157 | private KeyRecord(Key key, Record record) { 158 | this.key = key; 159 | this.record = record; 160 | } 161 | } 162 | 163 | public void setThread(Thread thread) { 164 | this.thread = thread; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/AerospikeOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeClient; 4 | import com.aerospike.client.Host; 5 | import com.aerospike.client.IAerospikeClient; 6 | import com.aerospike.client.Key; 7 | import com.aerospike.client.Value; 8 | import com.aerospike.client.policy.ClientPolicy; 9 | import com.playtika.janusgraph.aerospike.AerospikePolicyProvider; 10 | import org.janusgraph.diskstorage.StaticBuffer; 11 | import org.janusgraph.diskstorage.configuration.Configuration; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | import java.util.concurrent.ScheduledExecutorService; 18 | import java.util.concurrent.ScheduledFuture; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.stream.Stream; 21 | 22 | import static nosql.batch.update.util.AsyncUtil.shutdownAndAwaitTermination; 23 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_HOSTS; 24 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_PORT; 25 | 26 | public class AerospikeOperations { 27 | 28 | private static final Logger logger = LoggerFactory.getLogger(AerospikeOperations.class); 29 | 30 | private static final int DEFAULT_PORT = 3000; 31 | 32 | public static final String ENTRIES_BIN_NAME = "entries"; 33 | 34 | private final String namespace; 35 | private final String idsNamespace; 36 | private final String idsStoreName; 37 | private final String graphPrefix; 38 | 39 | private final IAerospikeClient client; 40 | private final ExecutorService aerospikeExecutor; 41 | 42 | private final AerospikePolicyProvider aerospikePolicyProvider; 43 | private final ExecutorService batchExecutor; 44 | 45 | private final ScheduledExecutorService statsLogger; 46 | private final ScheduledFuture statsFuture; 47 | 48 | public AerospikeOperations(String graphPrefix, 49 | String namespace, 50 | String idsNamespace, 51 | String idsStoreName, 52 | IAerospikeClient client, 53 | AerospikePolicyProvider aerospikePolicyProvider, 54 | ExecutorService aerospikeExecutor, 55 | ExecutorService batchExecutor) { 56 | this.graphPrefix = graphPrefix+"."; 57 | this.namespace = namespace; 58 | this.idsNamespace = idsNamespace; 59 | this.idsStoreName = idsStoreName; 60 | this.client = client; 61 | this.aerospikeExecutor = aerospikeExecutor; 62 | this.aerospikePolicyProvider = aerospikePolicyProvider; 63 | this.batchExecutor = batchExecutor; 64 | this.statsLogger = Executors.newScheduledThreadPool(1); 65 | this.statsFuture = statsLogger.scheduleAtFixedRate(() -> 66 | Stream.of(client.getCluster().getNodes()).forEach(node -> 67 | logger.info("node [{}] connections stats: {}", node, node.getConnectionStats())), 68 | 5, 5, TimeUnit.MINUTES); 69 | } 70 | 71 | public IAerospikeClient getClient() { 72 | return client; 73 | } 74 | 75 | public ExecutorService getAerospikeExecutor() { 76 | return aerospikeExecutor; 77 | } 78 | 79 | public ExecutorService getBatchExecutor() { 80 | return batchExecutor; 81 | } 82 | 83 | public String getNamespace() { 84 | return namespace; 85 | } 86 | 87 | Key getKey(String storeName, StaticBuffer staticBuffer) { 88 | return getKey(storeName, getValue(staticBuffer)); 89 | } 90 | 91 | public static Value getValue(StaticBuffer staticBuffer) { 92 | return staticBuffer.as((array, offset, limit) -> Value.get(array, offset, limit - offset)); 93 | } 94 | 95 | public Key getKey(String storeName, Value value) { 96 | String namespace = idsStoreName.equals(storeName) ? this.idsNamespace : this.namespace; 97 | return new Key(namespace, getSetName(storeName), value); 98 | } 99 | 100 | String getSetName(String storeName) { 101 | return graphPrefix + storeName; 102 | } 103 | 104 | public AerospikePolicyProvider getAerospikePolicyProvider() { 105 | return aerospikePolicyProvider; 106 | } 107 | 108 | public String getGraphPrefix() { 109 | return graphPrefix; 110 | } 111 | 112 | public void close() { 113 | statsFuture.cancel(true); 114 | shutdownAndAwaitTermination(statsLogger); 115 | client.close(); 116 | aerospikePolicyProvider.close(); 117 | } 118 | 119 | public static IAerospikeClient buildAerospikeClient(Configuration configuration, ClientPolicy clientPolicy) { 120 | int port = configuration.has(STORAGE_PORT) ? configuration.get(STORAGE_PORT) : DEFAULT_PORT; 121 | 122 | Host[] hosts = Stream.of(configuration.get(STORAGE_HOSTS)) 123 | .map(hostname -> new Host(hostname, port)).toArray(Host[]::new); 124 | 125 | return new AerospikeClient(clientPolicy, hosts); 126 | } 127 | 128 | 129 | } 130 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/BasicMutateOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeException; 4 | import com.aerospike.client.IAerospikeClient; 5 | import com.aerospike.client.Key; 6 | import com.aerospike.client.Operation; 7 | import com.aerospike.client.Record; 8 | import com.aerospike.client.ResultCode; 9 | import com.aerospike.client.Value; 10 | import com.aerospike.client.cdt.MapOperation; 11 | import com.aerospike.client.cdt.MapOrder; 12 | import com.aerospike.client.cdt.MapPolicy; 13 | import com.aerospike.client.cdt.MapReturnType; 14 | import com.aerospike.client.cdt.MapWriteMode; 15 | import com.aerospike.client.policy.WritePolicy; 16 | import com.playtika.janusgraph.aerospike.AerospikePolicyProvider; 17 | import org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 25 | import static com.playtika.janusgraph.aerospike.operations.batch.BatchUpdates.IDS_STORE_NAME; 26 | 27 | public class BasicMutateOperations implements MutateOperations{ 28 | 29 | static final MapPolicy mapPolicy = new MapPolicy(MapOrder.KEY_ORDERED, MapWriteMode.UPDATE); 30 | 31 | private final WritePolicy mutatePolicy; 32 | private final WritePolicy deletePolicy; 33 | private final AerospikeOperations aerospikeOperations; 34 | 35 | public BasicMutateOperations(AerospikeOperations aerospikeOperations) { 36 | this.aerospikeOperations = aerospikeOperations; 37 | 38 | AerospikePolicyProvider aerospikePolicyProvider = aerospikeOperations.getAerospikePolicyProvider(); 39 | this.mutatePolicy = buildMutationPolicy(aerospikePolicyProvider); 40 | this.deletePolicy = aerospikePolicyProvider.deletePolicy(); 41 | } 42 | 43 | @Override 44 | public void mutate(String storeName, Value key, Map mutation) { 45 | 46 | Key aerospikeKey = aerospikeOperations.getKey(storeName, key); 47 | List operations = new ArrayList<>(3); 48 | List keysToRemove = new ArrayList<>(mutation.size()); 49 | Map itemsToAdd = new HashMap<>(mutation.size()); 50 | for(Map.Entry entry : mutation.entrySet()){ 51 | if(entry.getValue() == Value.NULL){ 52 | keysToRemove.add(entry.getKey()); 53 | } else { 54 | itemsToAdd.put(entry.getKey(), entry.getValue()); 55 | } 56 | } 57 | 58 | if(!keysToRemove.isEmpty()) { 59 | operations.add(MapOperation.removeByKeyList(ENTRIES_BIN_NAME, keysToRemove, MapReturnType.NONE)); 60 | } 61 | 62 | if(!itemsToAdd.isEmpty()) { 63 | operations.add(MapOperation.putItems(mapPolicy, ENTRIES_BIN_NAME, itemsToAdd)); 64 | } 65 | 66 | int entriesNoOperationIndex; 67 | if(!keysToRemove.isEmpty()){ 68 | entriesNoOperationIndex = operations.size(); 69 | operations.add(MapOperation.size(ENTRIES_BIN_NAME)); 70 | } else { 71 | entriesNoOperationIndex = -1; 72 | } 73 | 74 | IAerospikeClient client = aerospikeOperations.getClient(); 75 | try { 76 | Record record = client.operate(mutatePolicy, aerospikeKey, operations.toArray(new Operation[0])); 77 | 78 | if(entriesNoOperationIndex != -1){ 79 | long entriesNoAfterMutation = (Long)record.getList(ENTRIES_BIN_NAME).get(entriesNoOperationIndex); 80 | if(entriesNoAfterMutation == 0){ 81 | client.delete(deletePolicy, aerospikeKey); 82 | } 83 | } 84 | 85 | } catch (AerospikeException ae) { 86 | if(ae.getResultCode() == ResultCode.RECORD_TOO_BIG 87 | && storeName.equals(IDS_STORE_NAME)){ 88 | throw new IDPoolExhaustedException(ae); 89 | } else if(ae.getResultCode() != ResultCode.KEY_NOT_FOUND_ERROR){ 90 | throw ae; 91 | } 92 | } 93 | } 94 | 95 | private static WritePolicy buildMutationPolicy(AerospikePolicyProvider policyProvider){ 96 | WritePolicy mutatePolicy = new WritePolicy(policyProvider.writePolicy()); 97 | mutatePolicy.respondAllOps = true; 98 | return mutatePolicy; 99 | } 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/BasicScanOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.policy.ScanPolicy; 4 | import com.playtika.janusgraph.aerospike.util.NamedThreadFactory; 5 | import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; 6 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 7 | import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.stream.Collectors; 12 | import java.util.stream.Stream; 13 | 14 | public class BasicScanOperations implements ScanOperations { 15 | 16 | private static Logger logger = LoggerFactory.getLogger(BasicScanOperations.class); 17 | 18 | private final AerospikeOperations aerospikeOperations; 19 | private final ScanPolicy scanPolicy; 20 | private final NamedThreadFactory namedThreadFactory; 21 | 22 | public BasicScanOperations(AerospikeOperations aerospikeOperations, 23 | NamedThreadFactory namedThreadFactory) { 24 | this.aerospikeOperations = aerospikeOperations; 25 | this.scanPolicy = aerospikeOperations.getAerospikePolicyProvider().scanPolicy(); 26 | this.namedThreadFactory = namedThreadFactory; 27 | } 28 | 29 | @Override 30 | public KeyIterator getKeys(String storeName, SliceQuery query, StoreTransaction txh) { 31 | logger.warn("Starting scan operation storeName=[{}], query=[{}], tx=[{}]", storeName, query, txh); 32 | if (logger.isTraceEnabled()) { 33 | logger.trace("Running scan operation stacktrace:\n{}", 34 | Stream.of(Thread.currentThread().getStackTrace()) 35 | .map(StackTraceElement::toString) 36 | .collect(Collectors.joining("\n"))); 37 | } 38 | 39 | AerospikeKeyIterator keyIterator = new AerospikeKeyIterator(query); 40 | 41 | Thread scanThread = namedThreadFactory.newThread(() -> { 42 | try { 43 | logger.warn("Running scan operation storeName=[{}], query=[{}], tx=[{}]", storeName, query, txh); 44 | aerospikeOperations.getClient().scanAll(scanPolicy, 45 | aerospikeOperations.getNamespace(), aerospikeOperations.getSetName(storeName), keyIterator); 46 | logger.info("Finished scan operation storeName=[{}], query=[{}], tx=[{}]", storeName, query, txh); 47 | } catch (Throwable t) { 48 | logger.error("Error while running scan operation storeName=[{}], query=[{}], tx=[{}]", 49 | storeName, query, txh, t); 50 | throw t; 51 | } finally { 52 | keyIterator.close(); 53 | } 54 | }); 55 | 56 | keyIterator.setThread(scanThread); 57 | 58 | scanThread.start(); 59 | 60 | return keyIterator; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/ErrorMapper.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import nosql.batch.update.lock.TemporaryLockingException; 4 | import org.janusgraph.diskstorage.BackendException; 5 | 6 | import java.util.function.Function; 7 | 8 | public class ErrorMapper { 9 | 10 | private ErrorMapper(){} 11 | 12 | public static final Function INSTANCE = throwable -> { 13 | if(throwable instanceof TemporaryLockingException){ 14 | return new org.janusgraph.diskstorage.locking.TemporaryLockingException(throwable); 15 | } else { 16 | return new org.janusgraph.diskstorage.locking.PermanentLockingException(throwable); 17 | } 18 | }; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/IdsCleanupOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.Value; 4 | import org.janusgraph.diskstorage.BackendException; 5 | import org.janusgraph.diskstorage.Entry; 6 | import org.janusgraph.diskstorage.EntryList; 7 | import org.janusgraph.diskstorage.StaticBuffer; 8 | import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation; 9 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore; 10 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 11 | import org.janusgraph.diskstorage.util.BufferUtil; 12 | import org.janusgraph.diskstorage.util.time.TimestampProvider; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.time.Instant; 18 | import java.util.ArrayList; 19 | import java.util.Comparator; 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.concurrent.ExecutorService; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeKeyColumnValueStore.mutationToMap; 25 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.getValue; 26 | import static java.util.Collections.singletonList; 27 | 28 | //TODO remove after this issue fixed https://github.com/JanusGraph/janusgraph/issues/3007 29 | public class IdsCleanupOperations { 30 | 31 | private static final Logger logger = LoggerFactory.getLogger(IdsCleanupOperations.class); 32 | 33 | public static final StaticBuffer LOWER_SLICE = BufferUtil.zeroBuffer(1); 34 | public static final StaticBuffer UPPER_SLICE = BufferUtil.oneBuffer(17); 35 | 36 | public static final int PROTECTED_BLOCKS_AMOUNT = 10; 37 | 38 | 39 | private final String idsStoreName; 40 | private final long ttl; 41 | private final ReadOperations readOperations; 42 | private final MutateOperations mutateOperations; 43 | private final TimestampProvider timestampProvider; 44 | private final ExecutorService executorService; 45 | 46 | public IdsCleanupOperations(String idsStoreName, 47 | ReadOperations readOperations, MutateOperations mutateOperations, 48 | long ttl, 49 | TimestampProvider timestampProvider, 50 | ExecutorService executorService) { 51 | this.idsStoreName = idsStoreName; 52 | this.ttl = ttl; 53 | this.readOperations = readOperations; 54 | this.mutateOperations = mutateOperations; 55 | this.timestampProvider = timestampProvider; 56 | this.executorService = executorService; 57 | } 58 | 59 | public void cleanUpOldIdsRanges(StaticBuffer key) { 60 | executorService.submit(() -> { 61 | try { 62 | cleanUpOldIdsRangesImpl(key); 63 | } catch (BackendException e) { 64 | logger.error("Error while running cleanup of old ranges for key=[{}]", key, e); 65 | } 66 | }); 67 | } 68 | 69 | public void cleanUpOldIdsRangesImpl(StaticBuffer key) throws BackendException { 70 | if(ttl == Long.MAX_VALUE){ 71 | return; 72 | } 73 | 74 | Map allIdBlocksMap = readOperations.getSlice(idsStoreName, singletonList(key), new SliceQuery(LOWER_SLICE, UPPER_SLICE)); 75 | EntryList blocks = allIdBlocksMap.get(key); 76 | 77 | blocks.sort(TIMESTAMP_COMPARATOR); 78 | List blocksToCheck = blocks.subList(0, Math.max(blocks.size() - PROTECTED_BLOCKS_AMOUNT, 0)); 79 | 80 | List columnsToRemove = new ArrayList<>(); 81 | for (Entry e : blocksToCheck) { 82 | ByteBuffer byteBuffer = e.asByteBuffer(); 83 | long counterVal = byteBuffer.getLong(); 84 | long idBlockTimestamp = byteBuffer.getLong(); 85 | byte[] instanceNameData = new byte[byteBuffer.remaining()]; 86 | byteBuffer.get(instanceNameData); 87 | String instanceName = new String(instanceNameData); 88 | 89 | long currentTimestamp = timestampProvider.getTime(timestampProvider.getTime()); 90 | long ttlTimestamp = timestampProvider.getTime(Instant.ofEpochMilli(ttl)); 91 | if(idBlockTimestamp < currentTimestamp - ttlTimestamp){ 92 | columnsToRemove.add(e.getColumn()); 93 | logger.info("Added for removal id block - key=[{}], value=[{}], timestamp=[{}], instanceName=[{}]", 94 | key, counterVal, idBlockTimestamp, instanceName); 95 | } else { 96 | logger.trace("Will retain id block - key=[{}], value=[{}], timestamp=[{}], instanceName=[{}]", 97 | key, counterVal, idBlockTimestamp, instanceName); 98 | } 99 | } 100 | if(!columnsToRemove.isEmpty()){ 101 | Map mutationMap = mutationToMap(new KCVMutation(KeyColumnValueStore.NO_ADDITIONS, columnsToRemove)); 102 | mutateOperations.mutate(idsStoreName, getValue(key), mutationMap); 103 | logger.info("Removed [{}] old id blocks for - key=[{}]", columnsToRemove.size(), key); 104 | } 105 | } 106 | 107 | public String getIdsStoreName() { 108 | return idsStoreName; 109 | } 110 | 111 | BlockTimestampComparator TIMESTAMP_COMPARATOR = new BlockTimestampComparator(); 112 | private static class BlockTimestampComparator implements Comparator { 113 | 114 | @Override 115 | public int compare(Entry e1, Entry e2) { 116 | return Long.compare(getTimestamp(e1), getTimestamp(e2)); 117 | } 118 | } 119 | 120 | private static long getTimestamp(Entry e){ 121 | ByteBuffer byteBuffer = e.asByteBuffer(); 122 | byteBuffer.getLong(); 123 | return byteBuffer.getLong(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/MutateOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.Value; 4 | 5 | import java.util.Map; 6 | 7 | public interface MutateOperations { 8 | 9 | void mutate(String storeName, Value key, Map mutation); 10 | } 11 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/Operations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.Value; 4 | import com.playtika.janusgraph.aerospike.operations.batch.BatchLocks; 5 | import com.playtika.janusgraph.aerospike.operations.batch.BatchUpdates; 6 | import nosql.batch.update.BatchUpdater; 7 | import nosql.batch.update.aerospike.lock.AerospikeLock; 8 | import nosql.batch.update.wal.WriteAheadLogCompleter; 9 | 10 | public interface Operations { 11 | 12 | AerospikeOperations getAerospikeOperations(); 13 | 14 | BatchUpdater batchUpdater(); 15 | 16 | WriteAheadLogCompleter getWriteAheadLogCompleter(); 17 | 18 | MutateOperations mutateOperations(); 19 | 20 | ReadOperations getReadOperations(); 21 | 22 | ScanOperations getScanOperations(); 23 | 24 | void close(); 25 | 26 | IdsCleanupOperations getIdsCleanupOperations(String storeName); 27 | } 28 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/ReadOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeException; 4 | import com.aerospike.client.Record; 5 | import com.aerospike.client.cdt.MapOperation; 6 | import com.aerospike.client.cdt.MapReturnType; 7 | import com.aerospike.client.policy.WritePolicy; 8 | import org.janusgraph.diskstorage.BackendException; 9 | import org.janusgraph.diskstorage.EntryList; 10 | import org.janusgraph.diskstorage.PermanentBackendException; 11 | import org.janusgraph.diskstorage.StaticBuffer; 12 | import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery; 13 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 14 | import org.janusgraph.diskstorage.util.EntryArrayList; 15 | import org.janusgraph.diskstorage.util.StaticArrayBuffer; 16 | import org.janusgraph.diskstorage.util.StaticArrayEntry; 17 | 18 | import java.util.Collections; 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 24 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.getValue; 25 | import static com.playtika.janusgraph.aerospike.util.AsyncUtil.mapAll; 26 | 27 | public class ReadOperations { 28 | 29 | private final AerospikeOperations aerospikeOperations; 30 | private final WritePolicy getPolicy; 31 | private final int parallelReadThreshold; 32 | 33 | public ReadOperations(AerospikeOperations aerospikeOperations, int parallelReadThreshold) { 34 | this.aerospikeOperations = aerospikeOperations; 35 | this.getPolicy = aerospikeOperations.getAerospikePolicyProvider().writePolicy(); 36 | this.parallelReadThreshold = parallelReadThreshold; 37 | } 38 | 39 | public Map getSlice(String storeName, List keys, SliceQuery query) throws BackendException { 40 | if(keys.size() == 1){ 41 | return getSliceOfOneKey(storeName, keys.get(0), query); 42 | } else if(keys.size() < parallelReadThreshold){ 43 | return getSliceSequentially(storeName, keys, query); 44 | } else { 45 | return getSliceInParallel(storeName, keys, query); 46 | } 47 | } 48 | 49 | private Map getSliceOfOneKey(String storeName, StaticBuffer key, SliceQuery query) throws BackendException { 50 | return Collections.singletonMap(key, 51 | getSlice(storeName, new KeySliceQuery(key, query.getSliceStart(), query.getSliceEnd()))); 52 | } 53 | 54 | private Map getSliceSequentially(String storeName, List keys, SliceQuery query) throws BackendException { 55 | Map resultMap = new HashMap<>(keys.size()); 56 | for(StaticBuffer key : keys){ 57 | resultMap.putAll(getSliceOfOneKey(storeName, key, query)); 58 | } 59 | return resultMap; 60 | } 61 | 62 | private Map getSliceInParallel(String storeName, List keys, SliceQuery query) throws BackendException { 63 | return mapAll(keys, key -> { 64 | try { 65 | return getSlice(storeName, new KeySliceQuery(key, query)); 66 | } catch (BackendException e) { 67 | throw new RuntimeException(e); 68 | } 69 | }, aerospikeOperations.getAerospikeExecutor()); 70 | } 71 | 72 | public EntryList getSlice(String storeName, KeySliceQuery query) throws BackendException { 73 | 74 | try { 75 | Record record = aerospikeOperations.getClient().operate(getPolicy, 76 | aerospikeOperations.getKey(storeName, query.getKey()), 77 | MapOperation.getByKeyRange(ENTRIES_BIN_NAME, 78 | getValue(query.getSliceStart()), getValue(query.getSliceEnd()), MapReturnType.KEY_VALUE) 79 | ); 80 | 81 | return recordToEntries(record, query.getLimit()); 82 | 83 | } catch (AerospikeException e) { 84 | throw new PermanentBackendException(e); 85 | } 86 | } 87 | 88 | private EntryList recordToEntries(Record record, int entriesNo) { 89 | List resultList; 90 | if(record != null 91 | && (resultList = record.getList(ENTRIES_BIN_NAME)) != null 92 | && !resultList.isEmpty()) { 93 | final EntryArrayList result = new EntryArrayList(); 94 | resultList.stream() 95 | .limit(entriesNo) 96 | .forEach(o -> { 97 | Map.Entry entry = (Map.Entry) o; 98 | result.add(StaticArrayEntry.of( 99 | StaticArrayBuffer.of((byte[]) entry.getKey()), 100 | StaticArrayBuffer.of((byte[]) entry.getValue()))); 101 | }); 102 | return result; 103 | } else { 104 | return EntryList.EMPTY_LIST; 105 | } 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/ScanOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; 4 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 5 | import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; 6 | 7 | public interface ScanOperations { 8 | 9 | KeyIterator getKeys(String storeName, SliceQuery query, StoreTransaction txh); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/UnsupportedScanOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; 4 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 5 | import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; 6 | 7 | public class UnsupportedScanOperations implements ScanOperations { 8 | 9 | @Override 10 | public KeyIterator getKeys(String storeName, SliceQuery query, StoreTransaction txh) { 11 | throw new UnsupportedOperationException(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchExpectedValueOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Key; 4 | import com.aerospike.client.Operation; 5 | import com.aerospike.client.Record; 6 | import com.aerospike.client.Value; 7 | import com.aerospike.client.cdt.MapOperation; 8 | import com.aerospike.client.cdt.MapReturnType; 9 | import com.aerospike.client.policy.Replica; 10 | import com.aerospike.client.policy.WritePolicy; 11 | import com.playtika.janusgraph.aerospike.AerospikePolicyProvider; 12 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 13 | import nosql.batch.update.aerospike.lock.AerospikeExpectedValuesOperations; 14 | import nosql.batch.update.aerospike.lock.AerospikeLock; 15 | import nosql.batch.update.lock.PermanentLockingException; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import java.util.Collection; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 24 | import static com.playtika.janusgraph.aerospike.util.AsyncUtil.completeAll; 25 | import static nosql.batch.update.lock.Lock.LockType.SAME_BATCH; 26 | 27 | 28 | public class BatchExpectedValueOperations 29 | implements AerospikeExpectedValuesOperations> { 30 | 31 | private static final Logger logger = LoggerFactory.getLogger(BatchExpectedValueOperations.class); 32 | 33 | private final AerospikeOperations aerospikeOperations; 34 | private final WritePolicy checkValuesPolicy; 35 | 36 | public BatchExpectedValueOperations(AerospikeOperations aerospikeOperations) { 37 | this.aerospikeOperations = aerospikeOperations; 38 | this.checkValuesPolicy = buildCheckValuesPolicy(aerospikeOperations.getAerospikePolicyProvider()); 39 | } 40 | 41 | @Override 42 | public void checkExpectedValues(List locks, Map expectedValues) throws PermanentLockingException { 43 | if(locks.size() != expectedValues.size()){ 44 | throw new IllegalArgumentException("locks.size() != expectedValues.size()"); 45 | } 46 | 47 | completeAll(locks, 48 | aerospikeLock -> aerospikeLock.lockType != SAME_BATCH, 49 | aerospikeLock -> { 50 | ExpectedValue expectedValue = expectedValues.get(aerospikeLock.key); 51 | Key keyToCheck = aerospikeOperations.getKey(expectedValue.storeName, expectedValue.key); 52 | return checkColumnValues(keyToCheck, expectedValue.values); 53 | }, 54 | () -> new PermanentLockingException("Some values don't match expected values"), 55 | aerospikeOperations.getAerospikeExecutor()); 56 | } 57 | 58 | private boolean checkColumnValues(final Key key, final Map valuesForKey) { 59 | if(valuesForKey.isEmpty()){ 60 | throw new IllegalArgumentException(String.format("Empty valuesForKey key=[%s]", key)); 61 | } 62 | 63 | int columnsNo = valuesForKey.size(); 64 | Value[] columns = new Value[columnsNo]; 65 | Operation[] operations = new Operation[columnsNo]; 66 | int i = 0; 67 | for (Value column : valuesForKey.keySet()) { 68 | columns[i] = column; 69 | operations[i] = MapOperation.getByKey(ENTRIES_BIN_NAME, column, MapReturnType.VALUE); 70 | i++; 71 | } 72 | 73 | try { 74 | Record record = aerospikeOperations.getClient().operate(checkValuesPolicy, key, operations); 75 | if (record != null) { 76 | if (columnsNo > 1) { 77 | List resultList = record.getList(ENTRIES_BIN_NAME); 78 | if (resultList != null) { 79 | if(resultList.size() != columnsNo){ 80 | throw new IllegalArgumentException(String.format("Unexpected result size [%s] != [%s]", resultList.size(), columnsNo)); 81 | } 82 | for (int j = 0; j < columnsNo; j++) { 83 | Value column = columns[j]; 84 | if (!checkValue(key, column, valuesForKey.get(column), (byte[]) resultList.get(j))) { 85 | return false; 86 | } 87 | } 88 | return true; 89 | } else { 90 | throw new IllegalArgumentException(); 91 | } 92 | } else { //columnsNo == 1 93 | byte[] actualValueData = (byte[]) record.getValue(ENTRIES_BIN_NAME); 94 | Value column = columns[0]; 95 | return checkValue(key, column, valuesForKey.get(column), actualValueData); 96 | } 97 | } else { 98 | return allNulls(valuesForKey.values()); 99 | } 100 | } catch (Throwable throwable) { 101 | logger.error("Error while checkColumnValues for key={}, values={}", key, valuesForKey, throwable); 102 | throw throwable; 103 | } 104 | } 105 | 106 | private boolean allNulls(Collection values) { 107 | return values.stream().allMatch(value -> value.equals(Value.NULL)); 108 | } 109 | 110 | private boolean checkValue(Key key, Value column, Value expectedValue, byte[] actualValue) { 111 | if(expectedValue.equals(Value.get(actualValue)) 112 | || expectedValue instanceof Value.ByteSegmentValue 113 | && expectedValue.equals(Value.get(actualValue, 0, actualValue != null ? actualValue.length : 0))){ 114 | return true; 115 | } else { 116 | logger.info("Unexpected value for key=[{}], column=[{}], expected=[{}], actual=[{}]", key, column, expectedValue, actualValue); 117 | return false; 118 | } 119 | } 120 | 121 | private static WritePolicy buildCheckValuesPolicy(AerospikePolicyProvider policyProvider){ 122 | WritePolicy checkValuesPolicy = new WritePolicy(policyProvider.writePolicy()); 123 | checkValuesPolicy.replica = Replica.MASTER; 124 | checkValuesPolicy.respondAllOps = true; 125 | return checkValuesPolicy; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchLocks.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Key; 4 | import com.aerospike.client.Value; 5 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 6 | import nosql.batch.update.aerospike.lock.AerospikeBatchLocks; 7 | 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | public class BatchLocks implements AerospikeBatchLocks> { 14 | 15 | private final AerospikeOperations aerospikeOperations; 16 | private final Map>> locksByStore; 17 | private final List keysToLock; 18 | private final Map expectedValues; 19 | 20 | public BatchLocks(Map>> locksByStore, AerospikeOperations aerospikeOperations) { 21 | this.aerospikeOperations = aerospikeOperations; 22 | this.locksByStore = locksByStore; 23 | int locksCount = locksCount(locksByStore); 24 | keysToLock = new ArrayList<>(locksCount); 25 | expectedValues = new HashMap<>(locksCount); 26 | populateData(locksByStore); 27 | } 28 | 29 | private void populateData(Map>> locksByStore){ 30 | for (Map.Entry>> locksForStore : locksByStore.entrySet()) { 31 | String storeName = locksForStore.getKey(); 32 | for (Map.Entry> entry : locksForStore.getValue().entrySet()) { 33 | Key lockKey = getLockKey(storeName, entry.getKey()); 34 | keysToLock.add(lockKey); 35 | expectedValues.put(lockKey, new ExpectedValue(storeName, entry.getKey(), entry.getValue())); 36 | } 37 | } 38 | } 39 | 40 | @Override 41 | public List keysToLock() { 42 | return keysToLock; 43 | } 44 | 45 | private static int locksCount(Map>> locksByStore){ 46 | int count = 0; 47 | for (Map.Entry>> locksForStore : locksByStore.entrySet()) { 48 | count += locksForStore.getValue().size(); 49 | } 50 | return count; 51 | } 52 | 53 | 54 | private Key getLockKey(String storeName, Value value) { 55 | return aerospikeOperations.getKey(storeName + ".lock", value); 56 | } 57 | 58 | @Override 59 | public Map expectedValues() { 60 | return expectedValues; 61 | } 62 | 63 | public Map>> getLocksByStore() { 64 | return locksByStore; 65 | } 66 | 67 | @Override 68 | public String toString(){ 69 | return locksByStore.toString(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchOperationsUtil.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Key; 4 | import com.aerospike.client.Value; 5 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 6 | import com.playtika.janusgraph.aerospike.operations.BasicMutateOperations; 7 | import nosql.batch.update.BatchOperations; 8 | import nosql.batch.update.aerospike.lock.AerospikeLock; 9 | import nosql.batch.update.aerospike.lock.AerospikeLockOperations; 10 | import nosql.batch.update.aerospike.wal.AerospikeWriteAheadLogManager; 11 | 12 | import java.time.Clock; 13 | import java.util.Map; 14 | import java.util.concurrent.ExecutorService; 15 | 16 | public class BatchOperationsUtil { 17 | 18 | public static BatchOperations batchOperations( 19 | AerospikeOperations aerospikeOperations, 20 | String walNamespace, 21 | String walSetName, 22 | Clock clock, 23 | ExecutorService aerospikeExecutorService, 24 | ExecutorService batchExecutorService){ 25 | 26 | AerospikeWriteAheadLogManager> walManager = 27 | walManager(aerospikeOperations, walNamespace, walSetName, clock, aerospikeExecutorService); 28 | 29 | AerospikeLockOperations> lockOperations = 30 | lockOperations(aerospikeOperations, aerospikeExecutorService); 31 | 32 | BatchUpdateOperations updateOperations = updateOperations(aerospikeOperations); 33 | 34 | return new BatchOperations<>(walManager, lockOperations, updateOperations, batchExecutorService); 35 | } 36 | 37 | public static BatchUpdateOperations updateOperations(AerospikeOperations aerospikeOperations) { 38 | return new BatchUpdateOperations(new BasicMutateOperations(aerospikeOperations), aerospikeOperations.getAerospikeExecutor()); 39 | } 40 | 41 | public static AerospikeLockOperations> lockOperations( 42 | AerospikeOperations aerospikeOperations, ExecutorService executorService) { 43 | return new AerospikeLockOperations<>( 44 | aerospikeOperations.getClient(), 45 | new BatchExpectedValueOperations(aerospikeOperations), 46 | executorService); 47 | } 48 | 49 | public static AerospikeWriteAheadLogManager> walManager( 50 | AerospikeOperations aerospikeOperations, 51 | String walNamespace, String walSetName, 52 | Clock clock, ExecutorService executorService) { 53 | return new AerospikeWriteAheadLogManager<>( 54 | aerospikeOperations.getClient(), walNamespace, walSetName, 55 | new BatchUpdateSerde(aerospikeOperations), 56 | clock/*, executorService*/); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchUpdate.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | public class BatchUpdate implements nosql.batch.update.BatchUpdate{ 4 | 5 | private final BatchLocks batchLocks; 6 | private final BatchUpdates batchUpdates; 7 | 8 | public BatchUpdate(BatchLocks batchLocks, BatchUpdates batchUpdates) { 9 | this.batchLocks = batchLocks; 10 | this.batchUpdates = batchUpdates; 11 | } 12 | 13 | @Override 14 | public BatchLocks locks() { 15 | return batchLocks; 16 | } 17 | 18 | @Override 19 | public BatchUpdates updates() { 20 | return batchUpdates; 21 | } 22 | 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchUpdateOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.playtika.janusgraph.aerospike.operations.BasicMutateOperations; 4 | import nosql.batch.update.UpdateOperations; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.stream.Collectors; 10 | import java.util.stream.Stream; 11 | 12 | import static com.playtika.janusgraph.aerospike.util.AsyncUtil.completeAll; 13 | 14 | public class BatchUpdateOperations implements UpdateOperations { 15 | 16 | private final BasicMutateOperations mutateOperations; 17 | private final ExecutorService executorService; 18 | 19 | public BatchUpdateOperations(BasicMutateOperations mutateOperations, ExecutorService executorService) { 20 | this.mutateOperations = mutateOperations; 21 | this.executorService = executorService; 22 | } 23 | 24 | @Override 25 | public void updateMany(BatchUpdates batchUpdates, boolean calledByWal) { 26 | BatchUpdates.UpdatesOrdered updatesOrdered = batchUpdates.getUpdates(); 27 | completeAll(concat(updatesOrdered.indexDeletion, updatesOrdered.edgeCreation), 28 | updateValue -> true, 29 | updateValue -> { 30 | mutateOperations.mutate( 31 | updateValue.storeName, updateValue.key, updateValue.values); 32 | return true; 33 | }, 34 | () -> null, 35 | executorService); 36 | 37 | completeAll(concat(updatesOrdered.indexCreation, updatesOrdered.edgeDeletion), 38 | updateValue -> true, 39 | updateValue -> { 40 | mutateOperations.mutate( 41 | updateValue.storeName, updateValue.key, updateValue.values); 42 | return true; 43 | }, 44 | () -> null, 45 | executorService); 46 | 47 | completeAll(updatesOrdered.other, 48 | updateValue -> true, 49 | updateValue -> { 50 | mutateOperations.mutate( 51 | updateValue.storeName, updateValue.key, updateValue.values); 52 | return true; 53 | }, 54 | () -> null, 55 | executorService); 56 | } 57 | 58 | private static List concat(List list1, List list2){ 59 | if(list1.isEmpty()){ 60 | return list2; 61 | } else if(list2.isEmpty()){ 62 | return list1; 63 | } else { 64 | List result = new ArrayList<>(list1.size() + list2.size()); 65 | result.addAll(list1); 66 | result.addAll(list2); 67 | return result; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchUpdateSerde.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Bin; 4 | import com.aerospike.client.Key; 5 | import com.aerospike.client.Value; 6 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 7 | import nosql.batch.update.BatchUpdate; 8 | import nosql.batch.update.aerospike.wal.AerospikeBatchUpdateSerde; 9 | 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import static java.util.Arrays.asList; 15 | 16 | public class BatchUpdateSerde implements AerospikeBatchUpdateSerde> { 17 | 18 | private static final String LOCKS_BIN = "locks"; 19 | private static final String MUTATIONS_BIN = "mutations"; 20 | 21 | private final AerospikeOperations aerospikeOperations; 22 | 23 | public BatchUpdateSerde(AerospikeOperations aerospikeOperations) { 24 | this.aerospikeOperations = aerospikeOperations; 25 | } 26 | 27 | @Override 28 | public List write(BatchUpdate batch) { 29 | return asList(new Bin(LOCKS_BIN, stringMapToValue(batch.locks().getLocksByStore())), 30 | new Bin(MUTATIONS_BIN, stringMapToValue(batch.updates().getUpdatesByStore()))); 31 | } 32 | 33 | @Override 34 | public BatchUpdate read(Map bins) { 35 | return new com.playtika.janusgraph.aerospike.operations.batch.BatchUpdate( 36 | new BatchLocks(wrapMap((Map>>)bins.get(LOCKS_BIN)), aerospikeOperations), 37 | new BatchUpdates(wrapMap((Map>>)bins.get(MUTATIONS_BIN))) 38 | ); 39 | } 40 | 41 | private static Value stringMapToValue(Map>> map){ 42 | Map>> locksValue = new HashMap<>(map.size()); 43 | for(Map.Entry>> locksEntry : map.entrySet()){ 44 | locksValue.put(Value.get(locksEntry.getKey()), locksEntry.getValue()); 45 | } 46 | return Value.get(locksValue); 47 | } 48 | 49 | private static Map>> wrapMap( 50 | Map>> map){ 51 | Map>> resultMap = new HashMap<>(map.size()); 52 | for(Map.Entry>> mapEntry : map.entrySet()){ 53 | resultMap.put(mapEntry.getKey(), wrapBytesBytesMap(mapEntry.getValue())); 54 | } 55 | return resultMap; 56 | } 57 | 58 | private static Map> wrapBytesBytesMap(Map> map){ 59 | Map> resultMap = new HashMap<>(map.size()); 60 | for(Map.Entry> mapEntry : map.entrySet()){ 61 | resultMap.put(Value.get(mapEntry.getKey()), wrapBytesMap(mapEntry.getValue())); 62 | } 63 | return resultMap; 64 | } 65 | 66 | private static Map wrapBytesMap(Map map){ 67 | Map resultMap = new HashMap<>(map.size()); 68 | for(Map.Entry mapEntry : map.entrySet()){ 69 | resultMap.put(Value.get(mapEntry.getKey()), Value.get(mapEntry.getValue())); 70 | } 71 | return resultMap; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/BatchUpdates.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Value; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.HashSet; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Set; 11 | 12 | import static java.util.Arrays.asList; 13 | import static org.janusgraph.diskstorage.Backend.EDGESTORE_NAME; 14 | import static org.janusgraph.diskstorage.Backend.INDEXSTORE_NAME; 15 | import static org.janusgraph.graphdb.configuration.JanusGraphConstants.JANUSGRAPH_ID_STORE_NAME; 16 | 17 | public class BatchUpdates { 18 | 19 | public static final String INDEX_STORE_NAME = INDEXSTORE_NAME; 20 | public static final String EDGE_STORE_NAME = EDGESTORE_NAME; 21 | public static final String IDS_STORE_NAME = JANUSGRAPH_ID_STORE_NAME; 22 | 23 | public static final Set REGULAR_STORE_NAMES = new HashSet<>(asList(INDEX_STORE_NAME, EDGE_STORE_NAME)); 24 | 25 | private final UpdatesOrdered updatesOrdered; 26 | private final Map>> updatesByStore; 27 | 28 | public BatchUpdates(Map>> updatesByStore){ 29 | this.updatesOrdered = orderUpdates(updatesByStore); 30 | this.updatesByStore = updatesByStore; 31 | } 32 | 33 | /** 34 | * Order updates [indexes deletion, edge deletion, edge creation, index creation] 35 | * @param updatesByStore 36 | * @return 37 | */ 38 | private static UpdatesOrdered orderUpdates(Map>> updatesByStore){ 39 | List indexDeletion = new ArrayList<>(); 40 | List edgeDeletion = new ArrayList<>(); 41 | List edgeCreation = new ArrayList<>(); 42 | List indexCreation = new ArrayList<>(); 43 | List other = new ArrayList<>(); 44 | 45 | for(Map.Entry>> updatesByStoreEntry : updatesByStore.entrySet()){ 46 | String storeName = updatesByStoreEntry.getKey(); 47 | for(Map.Entry> keyUpdate : updatesByStoreEntry.getValue().entrySet()){ 48 | Value key = keyUpdate.getKey(); 49 | 50 | switch (storeName) { 51 | case INDEX_STORE_NAME: { 52 | Split> split = splitToCreationAndDeletion(keyUpdate.getValue()); 53 | if (split.created != null) { 54 | indexCreation.add(new UpdateValue(storeName, key, split.created)); 55 | } 56 | if (split.deleted != null) { 57 | indexDeletion.add(new UpdateValue(storeName, key, split.deleted)); 58 | } 59 | break; 60 | } 61 | case EDGE_STORE_NAME: { 62 | Split> split = splitToCreationAndDeletion(keyUpdate.getValue()); 63 | if (split.created != null) { 64 | edgeCreation.add(new UpdateValue(storeName, key, split.created)); 65 | } 66 | if (split.deleted != null) { 67 | edgeDeletion.add(new UpdateValue(storeName, key, split.deleted)); 68 | } 69 | break; 70 | } 71 | default: 72 | other.add(new UpdateValue(storeName, key, keyUpdate.getValue())); 73 | } 74 | } 75 | } 76 | return new UpdatesOrdered(indexDeletion, edgeDeletion, edgeCreation, indexCreation, other); 77 | } 78 | 79 | static class UpdatesOrdered { 80 | final List indexDeletion; 81 | final List edgeDeletion; 82 | final List edgeCreation; 83 | final List indexCreation; 84 | final List other; 85 | 86 | UpdatesOrdered(List indexDeletion, 87 | List edgeDeletion, 88 | List edgeCreation, 89 | List indexCreation, 90 | List other) { 91 | this.indexDeletion = indexDeletion; 92 | this.edgeDeletion = edgeDeletion; 93 | this.edgeCreation = edgeCreation; 94 | this.indexCreation = indexCreation; 95 | this.other = other; 96 | } 97 | } 98 | 99 | private static Split> splitToCreationAndDeletion(Map updates){ 100 | boolean onlyCreation = true; 101 | boolean onlyDeletion = true; 102 | for(Map.Entry updateEntry : updates.entrySet()){ 103 | if(updateEntry.getValue() == null){ 104 | onlyCreation = false; 105 | } else { 106 | onlyDeletion = false; 107 | } 108 | } 109 | 110 | if(onlyCreation){ 111 | return new Split<>(updates, null); 112 | } else if(onlyDeletion){ 113 | return new Split<>(null, updates); 114 | } else { 115 | Map creation = new HashMap<>(updates.size()); 116 | Map deletion = new HashMap<>(updates.size()); 117 | for(Map.Entry updateEntry : updates.entrySet()){ 118 | if(updateEntry.getValue() == null){ 119 | deletion.put(updateEntry.getKey(), null); 120 | } else { 121 | deletion.put(updateEntry.getKey(), updateEntry.getValue()); 122 | } 123 | } 124 | return new Split<>(creation, deletion); 125 | } 126 | } 127 | 128 | private static class Split{ 129 | 130 | final V created; 131 | final V deleted; 132 | 133 | public Split(V created, V deleted) { 134 | this.created = created; 135 | this.deleted = deleted; 136 | } 137 | } 138 | 139 | public UpdatesOrdered getUpdates() { 140 | return updatesOrdered; 141 | } 142 | 143 | public Map>> getUpdatesByStore() { 144 | return updatesByStore; 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/ExpectedValue.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Value; 4 | 5 | import java.util.Map; 6 | 7 | public class ExpectedValue { 8 | public final String storeName; 9 | public final Value key; 10 | public final Map values; 11 | 12 | public ExpectedValue(String storeName, Value key, Map values) { 13 | this.storeName = storeName; 14 | this.key = key; 15 | this.values = values; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/UpdateValue.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.aerospike.client.Value; 4 | 5 | import java.util.Map; 6 | 7 | public class UpdateValue { 8 | public final String storeName; 9 | public final Value key; 10 | public final Map values; 11 | 12 | public UpdateValue(String storeName, Value key, Map values) { 13 | this.storeName = storeName; 14 | this.key = key; 15 | this.values = values; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/operations/batch/WalOperations.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations.batch; 2 | 3 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 4 | import org.janusgraph.diskstorage.configuration.Configuration; 5 | 6 | import static com.playtika.janusgraph.aerospike.ConfigOptions.WAL_MAX_BATCH_SIZE; 7 | import static com.playtika.janusgraph.aerospike.ConfigOptions.WAL_NAMESPACE; 8 | import static com.playtika.janusgraph.aerospike.ConfigOptions.WAL_STALE_TRANSACTION_LIFETIME_THRESHOLD; 9 | 10 | public class WalOperations { 11 | 12 | private final String walNamespace; 13 | private final Long staleTransactionLifetimeThresholdInMs; 14 | private final Integer maxBatchSize; 15 | private final String walSetName; 16 | private final AerospikeOperations aerospikeOperations; 17 | 18 | public WalOperations(Configuration configuration, AerospikeOperations aerospikeOperations) { 19 | this.walNamespace = configuration.get(WAL_NAMESPACE); 20 | this.staleTransactionLifetimeThresholdInMs = configuration.get(WAL_STALE_TRANSACTION_LIFETIME_THRESHOLD); 21 | this.maxBatchSize = configuration.get(WAL_MAX_BATCH_SIZE); 22 | this.walSetName = aerospikeOperations.getGraphPrefix() + "wal"; 23 | this.aerospikeOperations = aerospikeOperations; 24 | } 25 | 26 | public String getWalNamespace() { 27 | return walNamespace; 28 | } 29 | 30 | public Long getStaleTransactionLifetimeThresholdInMs() { 31 | return staleTransactionLifetimeThresholdInMs; 32 | } 33 | 34 | public Integer getMaxBatchSize() { 35 | return maxBatchSize; 36 | } 37 | 38 | public String getWalSetName() { 39 | return walSetName; 40 | } 41 | 42 | public AerospikeOperations getAerospikeOperations() { 43 | return aerospikeOperations; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/util/AerospikeUtils.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.util; 2 | 3 | import com.aerospike.client.IAerospikeClient; 4 | import com.aerospike.client.Info; 5 | 6 | import java.util.stream.Stream; 7 | 8 | public class AerospikeUtils { 9 | 10 | public static void truncateNamespace(IAerospikeClient client, String namespace) throws InterruptedException { 11 | while(!isEmptyNamespace(client, namespace)){ 12 | client.truncate(null, namespace, null, null); 13 | Thread.sleep(100); 14 | } 15 | } 16 | 17 | public static boolean isEmptyNamespace(IAerospikeClient client, String namespace){ 18 | String answer = Info.request(client.getNodes()[0], "sets/" + namespace); 19 | return answer.isEmpty() 20 | || Stream.of(answer.split(";")) 21 | .allMatch(s -> s.contains("objects=0")); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/util/AsyncUtil.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.util; 2 | 3 | import org.janusgraph.diskstorage.PermanentBackendException; 4 | 5 | import java.util.ArrayList; 6 | import java.util.Collection; 7 | import java.util.Collections; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.concurrent.CompletableFuture; 12 | import java.util.concurrent.ExecutionException; 13 | import java.util.concurrent.Executor; 14 | import java.util.concurrent.atomic.AtomicBoolean; 15 | import java.util.concurrent.atomic.AtomicReference; 16 | import java.util.function.Function; 17 | import java.util.function.Predicate; 18 | import java.util.function.Supplier; 19 | 20 | import static java.util.concurrent.CompletableFuture.runAsync; 21 | import static java.util.concurrent.CompletableFuture.supplyAsync; 22 | 23 | public class AsyncUtil { 24 | 25 | public static final int WAIT_TIMEOUT_IN_SECONDS = 4; 26 | 27 | public static void completeAll(Collection keys, Predicate keyPredicate, 28 | Function resultFunction, 29 | Supplier failedResultErrorSupplier, 30 | Executor executor) throws E { 31 | 32 | if(keys.isEmpty()){ 33 | return; 34 | } 35 | 36 | if(keys.size() == 1){ 37 | K key = keys.iterator().next(); 38 | if(!keyPredicate.test(key)){ 39 | return; 40 | } 41 | boolean result = resultFunction.apply(key); 42 | if(!result && failedResultErrorSupplier != null) { 43 | throw failedResultErrorSupplier.get(); 44 | } 45 | 46 | return; 47 | } 48 | 49 | List> futures = new ArrayList<>(keys.size()); 50 | AtomicReference failed = new AtomicReference<>(); 51 | AtomicBoolean checkFailed = new AtomicBoolean(false); 52 | 53 | for (K key : keys) { 54 | if(!keyPredicate.test(key)){ 55 | continue; 56 | } 57 | 58 | futures.add(runAsync(() -> { 59 | if(checkFailed.get()){ 60 | return; 61 | } 62 | try { 63 | if (!resultFunction.apply(key)) { 64 | checkFailed.set(true); 65 | } 66 | } catch (Throwable t) { 67 | failed.set(t); 68 | checkFailed.set(true); 69 | } 70 | }, executor)); 71 | } 72 | 73 | completeAll(futures); 74 | 75 | if(failed.get() != null){ 76 | throw new RuntimeException(failed.get()); 77 | } 78 | 79 | if(checkFailed.get() && failedResultErrorSupplier != null) { 80 | throw failedResultErrorSupplier.get(); 81 | } 82 | } 83 | 84 | protected static void completeAll(List> futures) { 85 | try { 86 | CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get(); 87 | } catch (InterruptedException | ExecutionException e) { 88 | throw new RuntimeException(e); 89 | } 90 | } 91 | 92 | public static Map mapAll(Collection keys, Function valueFunction, 93 | Executor executor) throws PermanentBackendException { 94 | if(keys.size() == 1){ 95 | K key = keys.iterator().next(); 96 | return Collections.singletonMap(key, valueFunction.apply(key)); 97 | } 98 | 99 | final Map> futures = new HashMap<>(keys.size()); 100 | AtomicReference failed = new AtomicReference<>(); 101 | for (K key : keys) 102 | futures.put(key, supplyAsync(() -> { 103 | try { 104 | if(failed.get() != null){ 105 | return null; 106 | } 107 | return valueFunction.apply(key); 108 | } catch (Throwable t) { 109 | failed.set(t); 110 | return null; 111 | } 112 | }, executor)); 113 | 114 | 115 | 116 | Map result = getAll(futures); 117 | if(failed.get() != null){ 118 | throw new PermanentBackendException(failed.get()); 119 | } else { 120 | return result; 121 | } 122 | } 123 | 124 | public static Map getAll(Map> futureMap) throws PermanentBackendException { 125 | Map resultMap = new HashMap<>(futureMap.size()); 126 | try { 127 | for(Map.Entry> entry : futureMap.entrySet()){ 128 | resultMap.put(entry.getKey(), entry.getValue().get()); 129 | } 130 | } catch (InterruptedException | ExecutionException e) { 131 | throw new RuntimeException(e); 132 | } 133 | return resultMap; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/aerospike/util/NamedThreadFactory.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.util; 2 | 3 | import java.util.concurrent.ThreadFactory; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | /** 7 | * ThreadFactory with the ability to set the thread name prefix. 8 | * This class is exactly similar to 9 | * {@link java.util.concurrent.Executors#defaultThreadFactory()} 10 | * from JDK8, except for the thread naming feature. 11 | * 12 | *

13 | * The factory creates threads that have names on the form 14 | * prefix-N-thread-M, where prefix 15 | * is a string provided in the constructor, N is the sequence number of 16 | * this factory, and M is the sequence number of the thread created 17 | * by this factory. 18 | */ 19 | public class NamedThreadFactory implements ThreadFactory { 20 | 21 | // Note: The source code for this class was based entirely on 22 | // Executors.DefaultThreadFactory class from the JDK8 source. 23 | // The only change made is the ability to configure the thread 24 | // name prefix. 25 | 26 | private final ThreadGroup group; 27 | private final AtomicInteger threadNumber = new AtomicInteger(1); 28 | private final String namePrefix; 29 | 30 | /** 31 | * Creates a new ThreadFactory where threads are created with a name prefix 32 | * of prefix. 33 | * 34 | * @param prefix Thread name prefix. Never use a value of "pool" as in that 35 | * case you might as well have used 36 | * {@link java.util.concurrent.Executors#defaultThreadFactory()}. 37 | */ 38 | public NamedThreadFactory(String groupName, String prefix) { 39 | SecurityManager s = System.getSecurityManager(); 40 | ThreadGroup parent = (s != null) ? s.getThreadGroup() 41 | : Thread.currentThread().getThreadGroup(); 42 | this.group = new ThreadGroup(parent, groupName); 43 | this.namePrefix = groupName+"-"+prefix+"-"; 44 | } 45 | 46 | 47 | @Override 48 | public Thread newThread(Runnable r) { 49 | Thread t = new Thread(group, r, 50 | namePrefix + threadNumber.getAndIncrement(), 51 | 0); 52 | if (t.isDaemon()) { 53 | t.setDaemon(false); 54 | } 55 | if (t.getPriority() != Thread.NORM_PRIORITY) { 56 | t.setPriority(Thread.NORM_PRIORITY); 57 | } 58 | return t; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/utility/GhostVertexRemover.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.utility; 2 | 3 | import com.google.common.base.Preconditions; 4 | import org.janusgraph.core.JanusGraphRelation; 5 | import org.janusgraph.core.JanusGraphVertex; 6 | import org.janusgraph.diskstorage.BackendException; 7 | import org.janusgraph.diskstorage.BaseTransactionConfig; 8 | import org.janusgraph.diskstorage.EntryList; 9 | import org.janusgraph.diskstorage.StaticBuffer; 10 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore; 11 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; 12 | import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery; 13 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 14 | import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; 15 | import org.janusgraph.diskstorage.util.BufferUtil; 16 | import org.janusgraph.graphdb.database.StandardJanusGraph; 17 | import org.janusgraph.graphdb.idmanagement.IDManager; 18 | import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; 19 | import org.janusgraph.graphdb.vertices.CacheVertex; 20 | 21 | import java.util.Collections; 22 | import java.util.Iterator; 23 | import java.util.Map; 24 | 25 | import static org.janusgraph.diskstorage.Backend.EDGESTORE_NAME; 26 | import static org.janusgraph.graphdb.olap.VertexJobConverter.startTransaction; 27 | 28 | public class GhostVertexRemover { 29 | 30 | private static final SliceQuery EVERYTHING_QUERY = new SliceQuery(BufferUtil.zeroBuffer(1),BufferUtil.oneBuffer(4)); 31 | private static final int RELATION_COUNT_LIMIT = 20000; 32 | private static final SliceQuery EVERYTHING_QUERY_LIMIT = EVERYTHING_QUERY.updateLimit(RELATION_COUNT_LIMIT); 33 | 34 | private final IDManager idManager; 35 | private final KeyColumnValueStore edgeStore; 36 | private final StandardJanusGraphTx tx; 37 | 38 | public GhostVertexRemover(StandardJanusGraph standardJanusGraph) throws BackendException { 39 | idManager = standardJanusGraph.getIDManager(); 40 | KeyColumnValueStoreManager storeManager = (KeyColumnValueStoreManager)standardJanusGraph.getBackend().getStoreManager(); 41 | edgeStore = storeManager.openDatabase(EDGESTORE_NAME); 42 | tx = startTransaction(standardJanusGraph); 43 | } 44 | 45 | public void removeGhostVertex(long vertexId) throws BackendException { 46 | StaticBuffer key = idManager.getKey(vertexId); 47 | EntryList entryList = edgeStore.getSlice(new KeySliceQuery(key, EVERYTHING_QUERY), new FakeStoreTransaction()); 48 | Map entries = Collections.singletonMap(EVERYTHING_QUERY, entryList); 49 | 50 | assert entries.get(EVERYTHING_QUERY_LIMIT)!=null; 51 | final EntryList everything = entries.get(EVERYTHING_QUERY_LIMIT); 52 | 53 | JanusGraphVertex vertex = tx.getInternalVertex(vertexId); 54 | Preconditions.checkArgument(vertex instanceof CacheVertex, 55 | "The bounding transaction is not configured correctly"); 56 | CacheVertex v = (CacheVertex)vertex; 57 | v.loadRelations(EVERYTHING_QUERY, input -> everything); 58 | 59 | int removedRelations = 0; 60 | Iterator iterator = v.query().noPartitionRestriction().relations().iterator(); 61 | while (iterator.hasNext()) { 62 | iterator.next(); 63 | iterator.remove(); 64 | removedRelations++; 65 | } 66 | v.remove(); 67 | 68 | tx.commit(); 69 | 70 | } 71 | 72 | private static class FakeStoreTransaction implements StoreTransaction { 73 | @Override 74 | public void commit() { 75 | } 76 | 77 | @Override 78 | public void rollback() { 79 | } 80 | 81 | @Override 82 | public BaseTransactionConfig getConfiguration() { 83 | return null; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/main/java/com/playtika/janusgraph/utility/KeyExtractor.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.utility; 2 | 3 | import org.janusgraph.diskstorage.StaticBuffer; 4 | import org.janusgraph.graphdb.idmanagement.IDManager; 5 | 6 | import static com.playtika.janusgraph.aerospike.operations.batch.BatchUpdates.EDGE_STORE_NAME; 7 | 8 | public class KeyExtractor { 9 | 10 | private final IDManager idManager; 11 | 12 | public KeyExtractor(IDManager idManager) { 13 | this.idManager = idManager; 14 | } 15 | 16 | public long getVertexId(String storeName, StaticBuffer key) { 17 | long keyID; 18 | if(storeName.equals(EDGE_STORE_NAME)){ 19 | keyID = idManager.getKeyID(key); 20 | } else { 21 | keyID = -1; 22 | } 23 | return keyID; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/AerospikeIDAuthorityCleanupTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike; 16 | 17 | import com.aerospike.client.AerospikeClient; 18 | import com.aerospike.client.async.NioEventLoops; 19 | import com.playtika.janusgraph.aerospike.operations.AerospikeOperations; 20 | import com.playtika.janusgraph.aerospike.operations.ReadOperations; 21 | import org.janusgraph.diskstorage.EntryList; 22 | import org.janusgraph.diskstorage.IDAuthorityTest; 23 | import org.janusgraph.diskstorage.StaticBuffer; 24 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 25 | import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery; 26 | import org.janusgraph.diskstorage.util.StaticArrayBuffer; 27 | import org.junit.ClassRule; 28 | import org.junit.Ignore; 29 | import org.junit.jupiter.api.Test; 30 | import org.junit.jupiter.params.ParameterizedTest; 31 | import org.junit.jupiter.params.provider.MethodSource; 32 | import org.testcontainers.containers.GenericContainer; 33 | 34 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.AEROSPIKE_PROPERTIES; 35 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeClient; 36 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 37 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 38 | import static com.playtika.janusgraph.aerospike.ConfigOptions.IDS_BLOCK_TTL; 39 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.AEROSPIKE_PREFIX; 40 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.BATCH_PREFIX; 41 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.executorService; 42 | import static com.playtika.janusgraph.aerospike.operations.IdsCleanupOperations.LOWER_SLICE; 43 | import static com.playtika.janusgraph.aerospike.operations.IdsCleanupOperations.PROTECTED_BLOCKS_AMOUNT; 44 | import static com.playtika.janusgraph.aerospike.operations.IdsCleanupOperations.UPPER_SLICE; 45 | import static org.assertj.core.api.Assertions.assertThat; 46 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.IDS_STORE_NAME; 47 | import static org.janusgraph.graphdb.configuration.JanusGraphConstants.JANUSGRAPH_ID_STORE_NAME; 48 | 49 | 50 | public class AerospikeIDAuthorityCleanupTest extends IDAuthorityTest { 51 | @ClassRule 52 | public static final GenericContainer container = getAerospikeContainer(); 53 | public static final String IDS_STORE = "ids"; 54 | 55 | @Test 56 | public void testSimpleIDAcquisition() throws Exception { 57 | super.testSimpleIDAcquisition((WriteConfiguration) configs().findFirst().get().get()[0]); 58 | 59 | ReadOperations readOperations = getReadOperations(); 60 | 61 | StaticBuffer partitionKey = new StaticArrayBuffer(new byte[]{0, 0, 0, 0, 0, 0, 0, 0}); 62 | EntryList entries = readOperations.getSlice(IDS_STORE, new KeySliceQuery(partitionKey, LOWER_SLICE, UPPER_SLICE)); 63 | assertThat(entries).hasSize(PROTECTED_BLOCKS_AMOUNT); 64 | } 65 | 66 | @Override 67 | public AerospikeStoreManager openStorageManager() { 68 | return new AerospikeStoreManager( 69 | getAerospikeConfiguration(container) 70 | .set(IDS_STORE_NAME, IDS_STORE) 71 | .set(IDS_BLOCK_TTL, 1L)); 72 | } 73 | 74 | public ReadOperations getReadOperations() { 75 | NioEventLoops eventLoops = new NioEventLoops(); 76 | AerospikeClient client = getAerospikeClient(container, eventLoops); 77 | return new ReadOperations( 78 | new AerospikeOperations("test", 79 | AEROSPIKE_PROPERTIES.getNamespace(), AEROSPIKE_PROPERTIES.getNamespace(), 80 | JANUSGRAPH_ID_STORE_NAME, 81 | client, 82 | new AerospikePolicyProvider(getAerospikeConfiguration(container)), 83 | executorService(4, AEROSPIKE_PREFIX), 84 | executorService(4, 4, BATCH_PREFIX)), 85 | 1); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/AerospikePolicyProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.aerospike.client.policy.ClientPolicy; 4 | import org.junit.ClassRule; 5 | import org.junit.Test; 6 | import org.testcontainers.containers.GenericContainer; 7 | 8 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 9 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 10 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 11 | 12 | public class AerospikePolicyProviderTest { 13 | 14 | @ClassRule 15 | public static GenericContainer container = getAerospikeContainer(); 16 | 17 | @Test 18 | public void shouldReturnDefaultClientPolicyMaxSocketIdle() { 19 | var aerospikePolicyProvider = new AerospikePolicyProvider( 20 | getAerospikeConfiguration(container) 21 | ); 22 | 23 | var clientPolicy = aerospikePolicyProvider.clientPolicy(); 24 | 25 | assertThat(clientPolicy.maxSocketIdle) 26 | .isEqualTo(new ClientPolicy().maxSocketIdle); 27 | } 28 | 29 | @Test 30 | public void shouldOverrideClientPolicyMaxSocketIdle() { 31 | var clientMaxSocketIdle = 55; 32 | var aerospikePolicyProvider = new AerospikePolicyProvider( 33 | getAerospikeConfiguration(container) 34 | .set(ConfigOptions.AEROSPIKE_CLIENT_MAX_SOCKET_IDLE, clientMaxSocketIdle) 35 | ); 36 | 37 | var clientPolicy = aerospikePolicyProvider.clientPolicy(); 38 | 39 | assertThat(clientPolicy.maxSocketIdle).isEqualTo(clientMaxSocketIdle); 40 | } 41 | } -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/AerospikeStoreManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import org.janusgraph.diskstorage.BackendException; 4 | import org.junit.ClassRule; 5 | import org.junit.Test; 6 | import org.testcontainers.containers.GenericContainer; 7 | 8 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 9 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 10 | 11 | public class AerospikeStoreManagerTest { 12 | 13 | @ClassRule 14 | public static GenericContainer container = getAerospikeContainer(); 15 | 16 | @Test 17 | public void shouldNotFailIfUdfsAlreadyRegistered() throws BackendException { 18 | AerospikeStoreManager manager = new AerospikeStoreManager(getAerospikeConfiguration(container)); 19 | manager.close(); 20 | manager = new AerospikeStoreManager(getAerospikeConfiguration(container)); 21 | manager.close(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/AerospikeTestUtils.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.aerospike.AerospikeContainerUtils; 4 | import com.aerospike.AerospikeProperties; 5 | import com.aerospike.client.AerospikeClient; 6 | import com.aerospike.client.async.EventLoops; 7 | import com.aerospike.client.policy.ClientPolicy; 8 | import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; 9 | import org.testcontainers.containers.GenericContainer; 10 | 11 | import static com.playtika.janusgraph.aerospike.AerospikeStoreManager.AEROSPIKE_BUFFER_SIZE; 12 | import static com.playtika.janusgraph.aerospike.ConfigOptions.GRAPH_PREFIX; 13 | import static com.playtika.janusgraph.aerospike.ConfigOptions.IDS_NAMESPACE; 14 | import static com.playtika.janusgraph.aerospike.ConfigOptions.NAMESPACE; 15 | import static com.playtika.janusgraph.aerospike.ConfigOptions.SCAN_PARALLELISM; 16 | import static com.playtika.janusgraph.aerospike.ConfigOptions.WAL_NAMESPACE; 17 | import static com.playtika.janusgraph.aerospike.util.AerospikeUtils.isEmptyNamespace; 18 | import static com.playtika.janusgraph.aerospike.util.AerospikeUtils.truncateNamespace; 19 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.BUFFER_SIZE; 20 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_BACKEND; 21 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_HOSTS; 22 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_PORT; 23 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.buildGraphConfiguration; 24 | 25 | public class AerospikeTestUtils { 26 | 27 | public static AerospikeProperties AEROSPIKE_PROPERTIES = new AerospikeProperties(); 28 | 29 | public static GenericContainer getAerospikeContainer() { 30 | return AerospikeContainerUtils.startAerospikeContainer(AEROSPIKE_PROPERTIES); 31 | } 32 | 33 | public static AerospikeClient getAerospikeClient(GenericContainer aerospike, EventLoops eventLoops) { 34 | ClientPolicy clientPolicy = new ClientPolicy(); 35 | clientPolicy.eventLoops = eventLoops; 36 | return new AerospikeClient(clientPolicy, aerospike.getContainerIpAddress(), 37 | aerospike.getMappedPort(AEROSPIKE_PROPERTIES.getPort())); 38 | } 39 | 40 | public static ModifiableConfiguration getAerospikeConfiguration(GenericContainer container) { 41 | 42 | ModifiableConfiguration config = buildGraphConfiguration(); 43 | config.set(STORAGE_HOSTS, new String[]{container.getContainerIpAddress()}); 44 | config.set(STORAGE_PORT, container.getMappedPort(AEROSPIKE_PROPERTIES.getPort())); 45 | config.set(STORAGE_BACKEND, "com.playtika.janusgraph.aerospike.AerospikeStoreManager"); 46 | config.set(NAMESPACE, AEROSPIKE_PROPERTIES.getNamespace()); 47 | config.set(IDS_NAMESPACE, AEROSPIKE_PROPERTIES.getNamespace()); 48 | config.set(WAL_NAMESPACE, AEROSPIKE_PROPERTIES.getNamespace()); 49 | config.set(GRAPH_PREFIX, "test"); 50 | //!!! need to prevent small batches mutations as we use deferred locking approach !!! 51 | config.set(BUFFER_SIZE, AEROSPIKE_BUFFER_SIZE); 52 | config.set(SCAN_PARALLELISM, 100); 53 | return config; 54 | } 55 | 56 | static void deleteAllRecords(GenericContainer container) throws InterruptedException { 57 | try(AerospikeClient client = new AerospikeClient(container.getContainerIpAddress(), 58 | container.getMappedPort(AEROSPIKE_PROPERTIES.getPort()))) { 59 | while(!isEmptyNamespace(client, AEROSPIKE_PROPERTIES.getNamespace())){ 60 | truncateNamespace(client, AEROSPIKE_PROPERTIES.getNamespace()); 61 | Thread.sleep(100); 62 | } 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/GraphOfTheGodsTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike; 2 | 3 | import com.playtika.janusgraph.trace.DebugJanusGraph; 4 | import org.apache.tinkerpop.gremlin.structure.Vertex; 5 | import org.janusgraph.core.JanusGraph; 6 | import org.janusgraph.core.JanusGraphFactory; 7 | import org.janusgraph.example.GraphOfTheGodsFactory; 8 | import org.junit.After; 9 | import org.junit.Before; 10 | import org.junit.Rule; 11 | import org.junit.Test; 12 | import org.testcontainers.containers.GenericContainer; 13 | 14 | import java.util.ArrayList; 15 | import java.util.Iterator; 16 | import java.util.List; 17 | 18 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 19 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 20 | import static org.junit.Assert.assertEquals; 21 | import static org.junit.Assert.assertFalse; 22 | import static org.junit.Assert.assertNotNull; 23 | import static org.junit.Assert.assertTrue; 24 | 25 | public class GraphOfTheGodsTest { 26 | 27 | @Rule 28 | public GenericContainer container = getAerospikeContainer(); 29 | 30 | JanusGraph graph; 31 | 32 | @Before 33 | public void buildGraph() { 34 | // graph = JanusGraphFactory.open(getAerospikeConfiguration(container)); 35 | graph = DebugJanusGraph.open(getAerospikeConfiguration(container).getConfiguration()); 36 | 37 | GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true); 38 | } 39 | 40 | @After 41 | public void tearDownGraph() { 42 | graph.close(); 43 | } 44 | 45 | @Test 46 | public void testQueryByName() { 47 | final Iterator results = graph.traversal().V().has("name", "jupiter"); 48 | assertTrue("Query should return a result", results.hasNext()); 49 | final Vertex jupiter = results.next(); 50 | assertNotNull("Query result should be non null", jupiter); 51 | 52 | jupiter.remove(); 53 | graph.tx().commit(); 54 | 55 | final Iterator resultsNew = graph.traversal().V().has("name", "jupiter"); 56 | assertFalse("Query should not return a result", resultsNew.hasNext()); 57 | } 58 | 59 | @Test 60 | public void testQueryAllVertices() { 61 | assertEquals("Expected the correct number of VERTICES", 62 | 12, graph.traversal().V().count().tryNext().get().longValue()); 63 | } 64 | 65 | @Test 66 | public void testQueryAllEdges() { 67 | assertEquals("Expected the correct number of EDGES", 68 | 17, graph.traversal().E().count().tryNext().get().longValue()); 69 | } 70 | 71 | @Test 72 | public void testRemoveAllVertices() { 73 | List vertexIds = new ArrayList<>(); 74 | graph.traversal().V().forEachRemaining(vertex -> vertexIds.add(vertex.id())); 75 | graph.traversal().V(vertexIds).drop().iterate(); 76 | graph.tx().commit(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugGraphCacheTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.StorageSetup; 19 | import org.janusgraph.core.JanusGraphException; 20 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 21 | import org.janusgraph.graphdb.JanusGraphTest; 22 | import org.janusgraph.graphdb.database.StandardJanusGraph; 23 | import org.junit.ClassRule; 24 | import org.junit.Ignore; 25 | import org.junit.jupiter.api.Disabled; 26 | import org.junit.jupiter.api.Test; 27 | import org.testcontainers.containers.GenericContainer; 28 | 29 | import java.util.concurrent.ExecutionException; 30 | 31 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 32 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 33 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 34 | 35 | @Disabled 36 | public class AerospikeDebugGraphCacheTest extends JanusGraphTest { 37 | 38 | @ClassRule 39 | public static final GenericContainer container = getAerospikeContainer(); 40 | 41 | @Override 42 | public void open(WriteConfiguration config) { 43 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 44 | this.features = this.graph.getConfiguration().getStoreFeatures(); 45 | this.tx = this.graph.newTransaction(); 46 | this.mgmt = this.graph.openManagement(); 47 | } 48 | 49 | @Override 50 | public WriteConfiguration getConfiguration() { 51 | return StorageSetup.addPermanentCache(getAerospikeConfiguration(container)); 52 | } 53 | 54 | //TODO Investigate 55 | @Ignore 56 | @Test 57 | @Override 58 | public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException { 59 | } 60 | 61 | @Test 62 | @Override 63 | public void testLargeJointIndexRetrieval() { 64 | assertThatThrownBy(super::testLargeJointIndexRetrieval) 65 | .isInstanceOf(JanusGraphException.class) //Record too big 66 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 67 | } 68 | 69 | @Test 70 | @Override 71 | public void testVertexCentricQuery() { 72 | assertThatThrownBy(super::testVertexCentricQuery) 73 | .isInstanceOf(JanusGraphException.class) //Record too big 74 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugGraphConcurrentTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.TestCategory; 19 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 20 | import org.janusgraph.graphdb.JanusGraphConcurrentTest; 21 | import org.janusgraph.graphdb.database.StandardJanusGraph; 22 | import org.junit.ClassRule; 23 | import org.junit.jupiter.api.Disabled; 24 | import org.junit.jupiter.api.Tag; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | 30 | @Disabled 31 | @Tag(TestCategory.PERFORMANCE_TESTS) 32 | public class AerospikeDebugGraphConcurrentTest extends JanusGraphConcurrentTest { 33 | 34 | @Override 35 | public void open(WriteConfiguration config) { 36 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 37 | this.features = this.graph.getConfiguration().getStoreFeatures(); 38 | this.tx = this.graph.newTransaction(); 39 | this.mgmt = this.graph.openManagement(); 40 | } 41 | 42 | @ClassRule 43 | public static final GenericContainer container = getAerospikeContainer(); 44 | 45 | @Override 46 | public WriteConfiguration getConfiguration() { 47 | return getAerospikeConfiguration(container).getConfiguration(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugGraphIterativeTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import com.playtika.janusgraph.trace.DebugJanusGraph; 19 | import org.janusgraph.diskstorage.BackendException; 20 | import org.janusgraph.diskstorage.configuration.BasicConfiguration; 21 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 22 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; 23 | import org.janusgraph.graphdb.JanusGraphIterativeBenchmark; 24 | import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; 25 | import org.janusgraph.graphdb.database.StandardJanusGraph; 26 | import org.junit.ClassRule; 27 | import org.junit.jupiter.api.Disabled; 28 | import org.testcontainers.containers.GenericContainer; 29 | 30 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 31 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 32 | 33 | 34 | @Disabled 35 | public class AerospikeDebugGraphIterativeTest extends JanusGraphIterativeBenchmark { 36 | 37 | @ClassRule 38 | public static final GenericContainer container = getAerospikeContainer(); 39 | 40 | @Override 41 | public void open(WriteConfiguration config) { 42 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 43 | this.features = this.graph.getConfiguration().getStoreFeatures(); 44 | this.tx = this.graph.newTransaction(); 45 | this.mgmt = this.graph.openManagement(); 46 | } 47 | 48 | @Override 49 | public WriteConfiguration getConfiguration() { 50 | return getAerospikeConfiguration(container).getConfiguration(); 51 | } 52 | 53 | @Override 54 | public KeyColumnValueStoreManager openStorageManager() throws BackendException { 55 | return new AerospikeStoreManager(new BasicConfiguration( 56 | GraphDatabaseConfiguration.ROOT_NS, 57 | getConfiguration(), 58 | BasicConfiguration.Restriction.NONE)); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugGraphPerformanceMemoryTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphPerformanceMemoryTest; 20 | import org.janusgraph.graphdb.database.StandardJanusGraph; 21 | import org.junit.ClassRule; 22 | import org.junit.jupiter.api.Disabled; 23 | import org.testcontainers.containers.GenericContainer; 24 | 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 26 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 27 | 28 | 29 | @Disabled 30 | public class AerospikeDebugGraphPerformanceMemoryTest extends JanusGraphPerformanceMemoryTest { 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public void open(WriteConfiguration config) { 36 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 37 | this.features = this.graph.getConfiguration().getStoreFeatures(); 38 | this.tx = this.graph.newTransaction(); 39 | this.mgmt = this.graph.openManagement(); 40 | } 41 | 42 | @Override 43 | public WriteConfiguration getConfiguration() { 44 | return getAerospikeConfiguration(container).getConfiguration(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.core.JanusGraphException; 19 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 20 | import org.janusgraph.graphdb.JanusGraphTest; 21 | import org.janusgraph.graphdb.database.StandardJanusGraph; 22 | import org.junit.ClassRule; 23 | import org.junit.Ignore; 24 | import org.junit.jupiter.api.Disabled; 25 | import org.junit.jupiter.api.Test; 26 | import org.testcontainers.containers.GenericContainer; 27 | 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 29 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 30 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 31 | 32 | @Disabled 33 | public class AerospikeDebugGraphTest extends JanusGraphTest { 34 | 35 | @ClassRule 36 | public static final GenericContainer container = getAerospikeContainer(); 37 | 38 | @Override 39 | public void open(WriteConfiguration config) { 40 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 41 | this.features = this.graph.getConfiguration().getStoreFeatures(); 42 | this.tx = this.graph.newTransaction(); 43 | this.mgmt = this.graph.openManagement(); 44 | } 45 | 46 | @Override 47 | public WriteConfiguration getConfiguration() { 48 | return getAerospikeConfiguration(container).getConfiguration(); 49 | } 50 | 51 | //TODO Investigate 52 | @Ignore 53 | @Test 54 | @Override 55 | public void testIndexUpdatesWithReindexAndRemove() { 56 | } 57 | 58 | @Test 59 | @Override 60 | public void testLargeJointIndexRetrieval() { 61 | assertThatThrownBy(super::testLargeJointIndexRetrieval) 62 | .isInstanceOf(JanusGraphException.class) //Record too big 63 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 64 | } 65 | 66 | @Test 67 | @Override 68 | public void testVertexCentricQuery() { 69 | assertThatThrownBy(super::testVertexCentricQuery) 70 | .isInstanceOf(JanusGraphException.class) //Record too big 71 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 72 | } 73 | 74 | } -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugOLAPTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.database.StandardJanusGraph; 20 | import org.janusgraph.olap.OLAPTest; 21 | import org.junit.ClassRule; 22 | import org.junit.Ignore; 23 | import org.junit.jupiter.api.Disabled; 24 | import org.junit.jupiter.api.Test; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | 30 | 31 | //TODO https://github.com/JanusGraph/janusgraph/issues/1527 32 | //TODO wait for https://github.com/JanusGraph/janusgraph/issues/1524 33 | @Disabled 34 | public class AerospikeDebugOLAPTest extends OLAPTest { 35 | @ClassRule 36 | public static final GenericContainer container = getAerospikeContainer(); 37 | 38 | @Override 39 | public void open(WriteConfiguration config) { 40 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 41 | this.features = this.graph.getConfiguration().getStoreFeatures(); 42 | this.tx = this.graph.newTransaction(); 43 | this.mgmt = this.graph.openManagement(); 44 | } 45 | 46 | @Override 47 | public WriteConfiguration getConfiguration() { 48 | return getAerospikeConfiguration(container).getConfiguration(); 49 | } 50 | 51 | //TODO 52 | @Ignore 53 | @Override 54 | @Test 55 | public void degreeCounting() { 56 | } 57 | 58 | //Throws java.lang.OutOfMemoryError: Java heap space 59 | @Ignore 60 | @Override 61 | @Test 62 | public void degreeCountingDistance() { 63 | } 64 | 65 | //TODO 66 | @Ignore 67 | @Override 68 | @Test 69 | public void testVertexScan() { 70 | } 71 | 72 | //TODO 73 | @Ignore 74 | @Override 75 | @Test 76 | public void testShortestDistance() { 77 | } 78 | 79 | //TODO 80 | @Ignore 81 | @Override 82 | @Test 83 | public void testPageRank() { 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeDebugOperationCountingTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphOperationCountingTest; 20 | import org.janusgraph.graphdb.database.StandardJanusGraph; 21 | import org.janusgraph.util.stats.MetricManager; 22 | import org.junit.ClassRule; 23 | import org.junit.jupiter.api.Disabled; 24 | import org.testcontainers.containers.GenericContainer; 25 | 26 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 28 | 29 | 30 | @Disabled 31 | public class AerospikeDebugOperationCountingTest extends JanusGraphOperationCountingTest { 32 | @ClassRule 33 | public static final GenericContainer container = getAerospikeContainer(); 34 | 35 | @Override 36 | public void open(WriteConfiguration config) { 37 | this.metric = MetricManager.INSTANCE; 38 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 39 | this.features = this.graph.getConfiguration().getStoreFeatures(); 40 | this.tx = this.graph.newTransaction(); 41 | this.mgmt = this.graph.openManagement(); 42 | } 43 | 44 | @Override 45 | public WriteConfiguration getBaseConfiguration() { 46 | return getAerospikeConfiguration(container).getConfiguration(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikeEventualDebugGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphEventualGraphTest; 20 | import org.janusgraph.graphdb.database.StandardJanusGraph; 21 | import org.junit.ClassRule; 22 | import org.junit.Ignore; 23 | import org.junit.jupiter.api.Disabled; 24 | import org.junit.jupiter.api.Test; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | 30 | @Disabled 31 | public class AerospikeEventualDebugGraphTest extends JanusGraphEventualGraphTest { 32 | 33 | @ClassRule 34 | public static final GenericContainer container = getAerospikeContainer(); 35 | 36 | @Override 37 | public void open(WriteConfiguration config) { 38 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 39 | this.features = this.graph.getConfiguration().getStoreFeatures(); 40 | this.tx = this.graph.newTransaction(); 41 | this.mgmt = this.graph.openManagement(); 42 | } 43 | 44 | @Override 45 | public WriteConfiguration getConfiguration() { 46 | return getAerospikeConfiguration(container).getConfiguration(); 47 | } 48 | 49 | //TODO can't get a glue why it should fail 50 | @Ignore 51 | @Override 52 | @Test 53 | public void testLockException() { 54 | } 55 | 56 | //TODO TTL not supported currently. think about 57 | @Ignore 58 | @Override 59 | @Test 60 | public void testTimestampSetting() { 61 | 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/debuggrapdb/AerospikePartitionDebugGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.debuggrapdb; 16 | 17 | import com.playtika.janusgraph.trace.DebugJanusGraph; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphPartitionGraphTest; 20 | import org.janusgraph.graphdb.database.StandardJanusGraph; 21 | import org.junit.ClassRule; 22 | import org.junit.jupiter.api.Disabled; 23 | import org.testcontainers.containers.GenericContainer; 24 | 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 26 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 27 | 28 | 29 | @Disabled 30 | public class AerospikePartitionDebugGraphTest extends JanusGraphPartitionGraphTest { 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public void open(WriteConfiguration config) { 36 | this.graph = (StandardJanusGraph) DebugJanusGraph.open(config); 37 | this.features = this.graph.getConfiguration().getStoreFeatures(); 38 | this.tx = this.graph.newTransaction(); 39 | this.mgmt = this.graph.openManagement(); 40 | } 41 | 42 | @Override 43 | public WriteConfiguration getBaseConfiguration() { 44 | return getAerospikeConfiguration(container).getConfiguration(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/diskstorage/AerospikeIDAuthorityTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.diskstorage; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.diskstorage.IDAuthorityTest; 19 | import org.junit.ClassRule; 20 | import org.testcontainers.containers.GenericContainer; 21 | 22 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 23 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 24 | import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.IDS_STORE_NAME; 25 | 26 | 27 | public class AerospikeIDAuthorityTest extends IDAuthorityTest { 28 | 29 | @ClassRule 30 | public static final GenericContainer container = getAerospikeContainer(); 31 | 32 | @Override 33 | public AerospikeStoreManager openStorageManager() { 34 | return new AerospikeStoreManager( 35 | getAerospikeConfiguration(container) 36 | .set(IDS_STORE_NAME, "ids")); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/diskstorage/AerospikeLockStoreTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.diskstorage; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.diskstorage.BackendException; 19 | import org.janusgraph.diskstorage.LockKeyColumnValueStoreTest; 20 | import org.janusgraph.diskstorage.configuration.Configuration; 21 | import org.junit.ClassRule; 22 | import org.junit.Ignore; 23 | import org.junit.jupiter.api.Tag; 24 | import org.junit.jupiter.api.Test; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | import static com.playtika.janusgraph.aerospike.diskstorage.AerospikeStoreTest.DUE_TO_DEFERRED_LOCKING_AND_MUTATION; 30 | 31 | 32 | public class AerospikeLockStoreTest extends LockKeyColumnValueStoreTest { 33 | 34 | @ClassRule 35 | public static final GenericContainer container = getAerospikeContainer(); 36 | 37 | @Override 38 | public AerospikeStoreManager openStorageManager(int id, Configuration configuration) { 39 | return new AerospikeStoreManager(getAerospikeConfiguration(container)); 40 | } 41 | 42 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 43 | @Ignore 44 | @Override 45 | @Test 46 | public void expiredLocalLockIsIgnored() {} 47 | 48 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 49 | @Ignore 50 | @Override 51 | @Test 52 | public void singleTransactionWithMultipleLocks() {} 53 | 54 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 55 | @Ignore 56 | @Override 57 | @Test 58 | public void testLocalLockContention() throws BackendException {} 59 | 60 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 61 | @Ignore 62 | @Override 63 | @Test 64 | public void testRemoteLockContention(){} 65 | } 66 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/diskstorage/AerospikeLogTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.diskstorage; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.TestCategory; 19 | import org.janusgraph.diskstorage.log.KCVSLogTest; 20 | import org.junit.ClassRule; 21 | import org.junit.jupiter.api.Tag; 22 | import org.testcontainers.containers.GenericContainer; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 26 | 27 | @Tag(TestCategory.SERIAL_TESTS) 28 | 29 | public class AerospikeLogTest extends KCVSLogTest { 30 | 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public AerospikeStoreManager openStorageManager() { 36 | return new AerospikeStoreManager(getAerospikeConfiguration(container)); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/diskstorage/AerospikeMultiWriteStoreTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.diskstorage; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.diskstorage.MultiWriteKeyColumnValueStoreTest; 19 | import org.junit.ClassRule; 20 | import org.junit.Ignore; 21 | import org.junit.jupiter.api.Tag; 22 | import org.junit.jupiter.api.Test; 23 | import org.testcontainers.containers.GenericContainer; 24 | 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 26 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 27 | import static com.playtika.janusgraph.aerospike.diskstorage.AerospikeStoreTest.DUE_TO_DEFERRED_LOCKING_AND_MUTATION; 28 | 29 | public class AerospikeMultiWriteStoreTest extends MultiWriteKeyColumnValueStoreTest { 30 | 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public AerospikeStoreManager openStorageManager() { 36 | return new AerospikeStoreManager(getAerospikeConfiguration(container)); 37 | } 38 | 39 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 40 | @Ignore 41 | @Override 42 | @Test 43 | public void mutateManyStressTest() { 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/diskstorage/AerospikeStoreTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.diskstorage; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.diskstorage.KeyColumnValueStoreTest; 19 | import org.janusgraph.testutil.FeatureFlag; 20 | import org.janusgraph.testutil.JanusGraphFeature; 21 | import org.junit.ClassRule; 22 | import org.junit.Ignore; 23 | import org.junit.jupiter.api.Tag; 24 | import org.junit.jupiter.api.Test; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | 30 | public class AerospikeStoreTest extends KeyColumnValueStoreTest { 31 | 32 | public static final String DUE_TO_DEFERRED_LOCKING_AND_MUTATION = "Due to deferred locking and mutation approach"; 33 | 34 | @ClassRule 35 | public static final GenericContainer container = getAerospikeContainer(); 36 | 37 | @Override 38 | public AerospikeStoreManager openStorageManager() { 39 | return new AerospikeStoreManager(getAerospikeConfiguration(container)); 40 | } 41 | 42 | //TODO check after janusgraph 0.6.0 43 | @Ignore 44 | @Override 45 | @Test 46 | public void scanTestWithSimpleJob() { 47 | } 48 | 49 | //TODO investigate 50 | @Ignore 51 | @Test 52 | @FeatureFlag( 53 | feature = JanusGraphFeature.Scan 54 | ) 55 | public void testGetKeysColumnSlicesSimple() { 56 | } 57 | 58 | @Tag(DUE_TO_DEFERRED_LOCKING_AND_MUTATION) 59 | @Ignore 60 | @Override 61 | @Test 62 | public void testConcurrentGetSliceAndMutate(){ 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeEventualGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 18 | import org.janusgraph.graphdb.JanusGraphEventualGraphTest; 19 | import org.junit.ClassRule; 20 | import org.junit.Ignore; 21 | import org.junit.jupiter.api.Test; 22 | import org.testcontainers.containers.GenericContainer; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 26 | 27 | public class AerospikeEventualGraphTest extends JanusGraphEventualGraphTest { 28 | 29 | @ClassRule 30 | public static final GenericContainer container = getAerospikeContainer(); 31 | 32 | @Override 33 | public WriteConfiguration getConfiguration() { 34 | return getAerospikeConfiguration(container).getConfiguration(); 35 | } 36 | 37 | //TODO can't get a glue why it should fail 38 | @Ignore 39 | @Override 40 | @Test 41 | public void testLockException() {} 42 | 43 | //TODO TTL not supported currently. think about 44 | @Ignore 45 | @Override 46 | @Test 47 | public void testTimestampSetting() { 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeGraphCacheTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.StorageSetup; 18 | import org.janusgraph.core.JanusGraphException; 19 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 20 | import org.janusgraph.graphdb.JanusGraphTest; 21 | import org.junit.ClassRule; 22 | import org.junit.Ignore; 23 | import org.junit.jupiter.api.Test; 24 | import org.testcontainers.containers.GenericContainer; 25 | 26 | import java.util.concurrent.ExecutionException; 27 | 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 29 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 30 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 31 | 32 | public class AerospikeGraphCacheTest extends JanusGraphTest { 33 | 34 | @ClassRule 35 | public static final GenericContainer container = getAerospikeContainer(); 36 | 37 | @Override 38 | public WriteConfiguration getConfiguration() { 39 | return StorageSetup.addPermanentCache(getAerospikeConfiguration(container)); 40 | } 41 | 42 | //TODO Investigate 43 | @Ignore 44 | @Test 45 | @Override 46 | public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException { 47 | } 48 | 49 | @Test 50 | @Override 51 | public void testLargeJointIndexRetrieval() { 52 | assertThatThrownBy(super::testLargeJointIndexRetrieval) 53 | .isInstanceOf(JanusGraphException.class) //Record too big 54 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 55 | } 56 | 57 | @Test 58 | @Override 59 | public void testVertexCentricQuery() { 60 | assertThatThrownBy(super::testVertexCentricQuery) 61 | .isInstanceOf(JanusGraphException.class) //Record too big 62 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeGraphConcurrentTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.TestCategory; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphConcurrentTest; 20 | import org.junit.ClassRule; 21 | import org.junit.jupiter.api.Tag; 22 | import org.testcontainers.containers.GenericContainer; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 26 | 27 | @Tag(TestCategory.PERFORMANCE_TESTS) 28 | public class AerospikeGraphConcurrentTest extends JanusGraphConcurrentTest { 29 | 30 | @ClassRule 31 | public static final GenericContainer container = getAerospikeContainer(); 32 | 33 | @Override 34 | public WriteConfiguration getConfiguration() { 35 | return getAerospikeConfiguration(container).getConfiguration(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeGraphIterativeTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import com.playtika.janusgraph.aerospike.AerospikeStoreManager; 18 | import org.janusgraph.diskstorage.BackendException; 19 | import org.janusgraph.diskstorage.configuration.BasicConfiguration; 20 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 21 | import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; 22 | import org.janusgraph.graphdb.JanusGraphIterativeBenchmark; 23 | import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; 24 | import org.junit.ClassRule; 25 | import org.testcontainers.containers.GenericContainer; 26 | 27 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 28 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 29 | 30 | 31 | public class AerospikeGraphIterativeTest extends JanusGraphIterativeBenchmark { 32 | 33 | @ClassRule 34 | public static final GenericContainer container = getAerospikeContainer(); 35 | 36 | @Override 37 | public WriteConfiguration getConfiguration() { 38 | return getAerospikeConfiguration(container).getConfiguration(); 39 | } 40 | 41 | @Override 42 | public KeyColumnValueStoreManager openStorageManager() throws BackendException { 43 | return new AerospikeStoreManager(new BasicConfiguration( 44 | GraphDatabaseConfiguration.ROOT_NS, 45 | getConfiguration(), 46 | BasicConfiguration.Restriction.NONE)); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeGraphPerformanceMemoryTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 18 | import org.janusgraph.graphdb.JanusGraphPerformanceMemoryTest; 19 | import org.junit.ClassRule; 20 | import org.testcontainers.containers.GenericContainer; 21 | 22 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 23 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 24 | 25 | 26 | public class AerospikeGraphPerformanceMemoryTest extends JanusGraphPerformanceMemoryTest { 27 | @ClassRule 28 | public static final GenericContainer container = getAerospikeContainer(); 29 | 30 | @Override 31 | public WriteConfiguration getConfiguration() { 32 | return getAerospikeConfiguration(container).getConfiguration(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.core.JanusGraphException; 18 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 19 | import org.janusgraph.graphdb.JanusGraphTest; 20 | import org.junit.ClassRule; 21 | import org.junit.Ignore; 22 | import org.junit.jupiter.api.Test; 23 | import org.testcontainers.containers.GenericContainer; 24 | 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 26 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 27 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 28 | 29 | public class AerospikeGraphTest extends JanusGraphTest { 30 | 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public WriteConfiguration getConfiguration() { 36 | return getAerospikeConfiguration(container).getConfiguration(); 37 | } 38 | 39 | //TODO Investigate 40 | @Ignore 41 | @Test 42 | @Override 43 | public void testIndexUpdatesWithReindexAndRemove() { 44 | } 45 | 46 | @Test 47 | @Override 48 | public void testLargeJointIndexRetrieval() { 49 | assertThatThrownBy(super::testLargeJointIndexRetrieval) 50 | .isInstanceOf(JanusGraphException.class) //Record too big 51 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 52 | } 53 | 54 | @Test 55 | @Override 56 | public void testVertexCentricQuery() { 57 | assertThatThrownBy(super::testVertexCentricQuery) 58 | .isInstanceOf(JanusGraphException.class) //Record too big 59 | .hasMessageContaining("Could not commit transaction due to exception during persistence"); 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeOLAPTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 18 | import org.janusgraph.olap.OLAPTest; 19 | import org.junit.ClassRule; 20 | import org.junit.Ignore; 21 | import org.junit.jupiter.api.Test; 22 | import org.testcontainers.containers.GenericContainer; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 26 | 27 | 28 | //TODO https://github.com/JanusGraph/janusgraph/issues/1527 29 | //TODO wait for https://github.com/JanusGraph/janusgraph/issues/1524 30 | public class AerospikeOLAPTest extends OLAPTest { 31 | @ClassRule 32 | public static final GenericContainer container = getAerospikeContainer(); 33 | 34 | @Override 35 | public WriteConfiguration getConfiguration() { 36 | return getAerospikeConfiguration(container).getConfiguration(); 37 | } 38 | 39 | //TODO 40 | @Ignore 41 | @Override 42 | @Test 43 | public void degreeCounting() { 44 | } 45 | 46 | //Throws java.lang.OutOfMemoryError: Java heap space 47 | @Ignore 48 | @Override 49 | @Test 50 | public void degreeCountingDistance() { 51 | } 52 | 53 | //TODO 54 | @Ignore 55 | @Override 56 | @Test 57 | public void testVertexScan() { 58 | } 59 | 60 | //TODO 61 | @Ignore 62 | @Override 63 | @Test 64 | public void testShortestDistance() { 65 | } 66 | 67 | //TODO unstable scan operations after upgrade to new Aerospike Server:ce-6.2.0.2 + Client:6.2.0 68 | @Ignore 69 | @Override 70 | @Test 71 | public void testShortestPath() { 72 | } 73 | 74 | //TODO 75 | @Ignore 76 | @Override 77 | @Test 78 | public void testPageRank(){ 79 | } 80 | 81 | //TODO unstable scan operations after upgrade to new Aerospike Server:ce-6.2.0.2 + Client:6.2.0 82 | @Ignore 83 | @Override 84 | @Test 85 | public void testConnectedComponent() { 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikeOperationCountingTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 18 | import org.janusgraph.graphdb.JanusGraphOperationCountingTest; 19 | import org.junit.ClassRule; 20 | import org.testcontainers.containers.GenericContainer; 21 | 22 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 23 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 24 | 25 | 26 | public class AerospikeOperationCountingTest extends JanusGraphOperationCountingTest { 27 | @ClassRule 28 | public static final GenericContainer container = getAerospikeContainer(); 29 | 30 | @Override 31 | public WriteConfiguration getBaseConfiguration() { 32 | return getAerospikeConfiguration(container).getConfiguration(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/graphdb/AerospikePartitionGraphTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 JanusGraph Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package com.playtika.janusgraph.aerospike.graphdb; 16 | 17 | import org.janusgraph.diskstorage.configuration.WriteConfiguration; 18 | import org.janusgraph.graphdb.JanusGraphPartitionGraphTest; 19 | import org.junit.ClassRule; 20 | import org.junit.Ignore; 21 | import org.junit.jupiter.api.Test; 22 | import org.testcontainers.containers.GenericContainer; 23 | 24 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 25 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 26 | 27 | 28 | public class AerospikePartitionGraphTest extends JanusGraphPartitionGraphTest { 29 | @ClassRule 30 | public static final GenericContainer container = getAerospikeContainer(); 31 | 32 | @Override 33 | public WriteConfiguration getBaseConfiguration() { 34 | return getAerospikeConfiguration(container).getConfiguration(); 35 | } 36 | 37 | //TODO unstable scan operations after upgrade to new Aerospike Server:ce-6.2.0.2 + Client:6.2.0 38 | @Ignore 39 | @Override 40 | @Test 41 | public void testVertexPartitionOlapCluster(){ 42 | } 43 | 44 | //TODO unstable scan operations after upgrade to new Aerospike Server:ce-6.2.0.2 + Client:6.2.0 45 | @Ignore 46 | @Override 47 | @Test 48 | public void testVertexPartitionOlapBatch(){ 49 | } 50 | 51 | //TODO unstable scan operations after upgrade to new Aerospike Server:ce-6.2.0.2 + Client:6.2.0 52 | @Ignore 53 | @Override 54 | @Test 55 | public void testVertexPartitionOlapIndividual(){ 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/operations/AerospikeKeyIteratorTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.Key; 4 | import com.aerospike.client.Record; 5 | import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; 6 | import org.janusgraph.diskstorage.util.StaticArrayBuffer; 7 | import org.junit.Test; 8 | 9 | import java.nio.ByteBuffer; 10 | import java.util.NoSuchElementException; 11 | 12 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 13 | import static java.util.Collections.singletonMap; 14 | import static org.assertj.core.api.Assertions.assertThat; 15 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 16 | 17 | public class AerospikeKeyIteratorTest { 18 | 19 | @Test 20 | public void shouldNotFailOnHasNextAfterClose(){ 21 | AerospikeKeyIterator keyIterator = new AerospikeKeyIterator(new SliceQuery( 22 | new StaticArrayBuffer(new byte[]{1}), new StaticArrayBuffer(new byte[]{2}) 23 | )); 24 | keyIterator.setThread(new Thread()); 25 | 26 | keyIterator.scanCallback( 27 | new Key("ns", "set", new byte[]{7}), 28 | new Record(singletonMap(ENTRIES_BIN_NAME, 29 | singletonMap(ByteBuffer.wrap(new byte[]{1}), new byte[]{3})), 0, 100)); 30 | 31 | assertThat(keyIterator.hasNext()).isTrue(); 32 | assertThat(keyIterator.next()).isNotNull(); 33 | 34 | keyIterator.close(); 35 | 36 | assertThat(keyIterator.hasNext()).isFalse(); 37 | assertThatThrownBy(keyIterator::next) 38 | .isInstanceOf(NoSuchElementException.class); 39 | 40 | assertThat(keyIterator.hasNext()).isFalse(); 41 | assertThatThrownBy(keyIterator::next) 42 | .isInstanceOf(NoSuchElementException.class); 43 | 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/operations/AerospikeTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeClient; 4 | import com.aerospike.client.Key; 5 | import com.aerospike.client.Record; 6 | import com.aerospike.client.Value; 7 | import com.aerospike.client.cdt.MapOperation; 8 | import com.aerospike.client.cdt.MapReturnType; 9 | import org.junit.ClassRule; 10 | import org.junit.Test; 11 | import org.testcontainers.containers.GenericContainer; 12 | 13 | import java.util.Collections; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.AEROSPIKE_PROPERTIES; 18 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 19 | import static com.playtika.janusgraph.aerospike.operations.AerospikeOperations.ENTRIES_BIN_NAME; 20 | import static com.playtika.janusgraph.aerospike.operations.BasicMutateOperations.mapPolicy; 21 | import static org.assertj.core.api.Assertions.assertThat; 22 | 23 | public class AerospikeTest { 24 | 25 | @ClassRule 26 | public static GenericContainer container = getAerospikeContainer(); 27 | 28 | private AerospikeClient client = new AerospikeClient(null, container.getContainerIpAddress(), 29 | container.getMappedPort(AEROSPIKE_PROPERTIES.getPort())); 30 | 31 | @Test 32 | public void shouldNotRemoveRecordOnEmptyMap(){ 33 | Key aerospikeKey = new Key(AEROSPIKE_PROPERTIES.getNamespace(), "test_set", Value.get(111)); 34 | 35 | Value key = Value.get(222); 36 | Map itemsToAdd = new HashMap(){{ 37 | put(key, Value.get(333)); 38 | }}; 39 | 40 | client.operate(null, aerospikeKey, 41 | MapOperation.putItems(mapPolicy, ENTRIES_BIN_NAME, itemsToAdd)); 42 | 43 | assertThat(client.exists(null, aerospikeKey)).isTrue(); 44 | 45 | Record record = client.operate(null, aerospikeKey, 46 | MapOperation.removeByKeyList(ENTRIES_BIN_NAME, Collections.singletonList(key), MapReturnType.NONE)); 47 | 48 | assertThat(client.exists(null, aerospikeKey)).isTrue(); 49 | assertThat(record.getMap(ENTRIES_BIN_NAME)).isNull(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/java/com/playtika/janusgraph/aerospike/operations/MutateOperationsTest.java: -------------------------------------------------------------------------------- 1 | package com.playtika.janusgraph.aerospike.operations; 2 | 3 | import com.aerospike.client.AerospikeClient; 4 | import com.aerospike.client.Value; 5 | import com.aerospike.client.async.NioEventLoops; 6 | import com.playtika.janusgraph.aerospike.AerospikePolicyProvider; 7 | import org.janusgraph.diskstorage.BackendException; 8 | import org.junit.ClassRule; 9 | import org.junit.Test; 10 | import org.testcontainers.containers.GenericContainer; 11 | 12 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.AEROSPIKE_PROPERTIES; 13 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeClient; 14 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeConfiguration; 15 | import static com.playtika.janusgraph.aerospike.AerospikeTestUtils.getAerospikeContainer; 16 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.AEROSPIKE_PREFIX; 17 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.BATCH_PREFIX; 18 | import static com.playtika.janusgraph.aerospike.operations.BasicOperations.executorService; 19 | import static java.util.Collections.singletonMap; 20 | import static org.janusgraph.graphdb.configuration.JanusGraphConstants.JANUSGRAPH_ID_STORE_NAME; 21 | 22 | public class MutateOperationsTest { 23 | 24 | @ClassRule 25 | public static GenericContainer container = getAerospikeContainer(); 26 | 27 | public static final String STORE_NAME = "testStore"; 28 | public static final Value KEY = Value.get("testKey"); 29 | public static final Value COLUMN_NAME = Value.get("column_name"); 30 | public static final Value COLUMN_VALUE = Value.get(new byte[]{1, 2, 3}); 31 | 32 | private NioEventLoops eventLoops = new NioEventLoops(); 33 | private AerospikeClient client = getAerospikeClient(getAerospikeContainer(), eventLoops); 34 | 35 | private MutateOperations mutateOperations = new BasicMutateOperations( 36 | new AerospikeOperations("test", 37 | AEROSPIKE_PROPERTIES.getNamespace(), AEROSPIKE_PROPERTIES.getNamespace(), 38 | JANUSGRAPH_ID_STORE_NAME, 39 | client, 40 | new AerospikePolicyProvider(getAerospikeConfiguration(container)), 41 | executorService(4, AEROSPIKE_PREFIX), 42 | executorService(4, 4, BATCH_PREFIX))); 43 | 44 | 45 | @Test 46 | public void shouldDeleteKeyIdempotently() throws BackendException { 47 | //when 48 | mutateOperations.mutate(STORE_NAME, KEY, 49 | singletonMap(COLUMN_NAME, COLUMN_VALUE)); 50 | 51 | //then 52 | mutateOperations.mutate(STORE_NAME, KEY, 53 | singletonMap(COLUMN_NAME, Value.NULL)); 54 | 55 | //expect 56 | mutateOperations.mutate(STORE_NAME, KEY, 57 | singletonMap(COLUMN_NAME, Value.NULL)); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /aerospike-storage-backend/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | --------------------------------------------------------------------------------