├── VERSION ├── h3lib ├── docs │ └── .gitignore ├── h3lib_config.h.in ├── url_parser.h ├── README.rst ├── cmake │ └── Modules │ │ ├── Findhiredis.cmake │ │ ├── Findkreon.cmake │ │ ├── Findkreonrdma.cmake │ │ └── Findrocksdb.cmake ├── util.h ├── util.c ├── kv_interface.h ├── common.h ├── CMakeLists.txt ├── url_parser.c └── kv_kreon.c ├── docs ├── requirements.txt ├── README.rst ├── configuration.rst ├── api.rst ├── Makefile ├── make.bat ├── index.rst └── conf.py ├── Jclouds-H3 ├── config │ ├── keystore.jks │ ├── s3proxy.conf │ └── run-docker-container.sh ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── jclouds │ │ │ └── h3 │ │ │ ├── predicates │ │ │ └── validators │ │ │ │ ├── H3ContainerNameValidator.java │ │ │ │ ├── internal │ │ │ │ ├── H3BlobKeyValidatorImpl.java │ │ │ │ └── H3ContainerNameValidatorImpl.java │ │ │ │ └── H3BlobKeyValidator.java │ │ │ ├── reference │ │ │ └── H3Constants.java │ │ │ ├── util │ │ │ ├── Utils.java │ │ │ └── internal │ │ │ │ └── H3BlobUtilsImpl.java │ │ │ ├── config │ │ │ └── H3ContextModule.java │ │ │ └── H3ApiMetadata.java │ └── test │ │ └── resources │ │ └── logging.properties ├── README.md └── pom.xml ├── h3fuse ├── h3fuse_config.h.in ├── README.rst └── CMakeLists.txt ├── JH3lib ├── config.ini ├── JH3 │ ├── config.ini │ ├── src │ │ ├── main │ │ │ └── java │ │ │ │ └── gr │ │ │ │ └── forth │ │ │ │ └── ics │ │ │ │ └── JH3lib │ │ │ │ ├── JH3Exception.java │ │ │ │ ├── JH3Auth.java │ │ │ │ ├── JH3MultipartId.java │ │ │ │ ├── JH3PartInfo.java │ │ │ │ ├── JH3Timespec.java │ │ │ │ ├── JH3Attribute.java │ │ │ │ ├── JH3AttributeType.java │ │ │ │ ├── JH3BucketInfo.java │ │ │ │ ├── JH3Object.java │ │ │ │ ├── JH3Status.java │ │ │ │ ├── JH3BucketStats.java │ │ │ │ └── JH3ObjectInfo.java │ │ └── test │ │ │ └── java │ │ │ └── gr │ │ │ └── forth │ │ │ └── ics │ │ │ └── JH3lib │ │ │ ├── JH3MultipartTest.java │ │ │ └── JH3BucketTest.java │ └── pom.xml ├── JH3HadoopFS │ ├── spark-examples │ │ ├── dataframes.spark │ │ ├── binaryFile.spark │ │ └── simpleWordCount.spark │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── gr │ │ │ └── forth │ │ │ └── ics │ │ │ └── JH3lib │ │ │ └── JH3HadoopAbstractFS.java │ └── pom.xml ├── pom.xml ├── JH3Interface │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── gr │ │ │ └── forth │ │ │ └── ics │ │ │ └── JH3lib │ │ │ ├── NativeAuth.java │ │ │ ├── NativeTimespec.java │ │ │ ├── NativePartInfo.java │ │ │ ├── NativeBucketInfo.java │ │ │ ├── NativeBucketStats.java │ │ │ ├── NativeObjectInfo.java │ │ │ └── NativeAttribute.java │ └── pom.xml └── README.md ├── CREDITS ├── h3cp ├── Makefile ├── README.rst ├── h3wrapper.h ├── h3wrapper.c └── main.go ├── .dockerignore ├── .gitignore ├── h3controllers ├── README.rst ├── expiresAtController.py ├── readOnlyAfterController.py └── cacheController.py ├── pyh3lib ├── pyh3lib │ ├── __init__.py │ └── version.py ├── tests │ ├── conftest.py │ ├── test_multipart.py │ ├── test_bucket.py │ ├── test_file.py │ └── test_metadata.py ├── README.rst └── setup.py ├── Makefile ├── .github └── workflows │ └── build-and-test.yaml ├── README.rst └── Dockerfile /VERSION: -------------------------------------------------------------------------------- 1 | 1.2 2 | -------------------------------------------------------------------------------- /h3lib/docs/.gitignore: -------------------------------------------------------------------------------- 1 | /html/ 2 | /latex/ 3 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==2.3.1 2 | sphinx-rtd-theme==0.4.3 3 | javasphinx 4 | -------------------------------------------------------------------------------- /Jclouds-H3/config/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CARV-ICS-FORTH/H3/HEAD/Jclouds-H3/config/keystore.jks -------------------------------------------------------------------------------- /h3fuse/h3fuse_config.h.in: -------------------------------------------------------------------------------- 1 | // the configured options and settings for h3fuse 2 | #define H3FS_VERSION_MAJOR @h3fuse_VERSION_MAJOR@ 3 | #define H3FS_VERSION_MINOR @h3fuse_VERSION_MINOR@ -------------------------------------------------------------------------------- /h3lib/h3lib_config.h.in: -------------------------------------------------------------------------------- 1 | // the configured options and settings for h3lib 2 | #define H3LIB_VERSION_MAJOR @h3lib_VERSION_MAJOR@ 3 | #define H3LIB_VERSION_MINOR @h3lib_VERSION_MINOR@ -------------------------------------------------------------------------------- /JH3lib/config.ini: -------------------------------------------------------------------------------- 1 | [H3] 2 | store = filesystem 3 | 4 | [FILESYSTEM] 5 | root = /tmp/h3/ 6 | 7 | [KREON] 8 | 9 | [REDISCLUSTER] 10 | 11 | [REDIS] 12 | 13 | [ROCKSDB] 14 | 15 | -------------------------------------------------------------------------------- /CREDITS: -------------------------------------------------------------------------------- 1 | Efstratios Politis 2 | Antony Chazapis 3 | Giorgos Kalaentzis 4 | Giorgos Saloustros 5 | -------------------------------------------------------------------------------- /JH3lib/JH3/config.ini: -------------------------------------------------------------------------------- 1 | [H3] 2 | store = filesystem 3 | 4 | [FILESYSTEM] 5 | root = /tmp/h3/ 6 | 7 | [KREON] 8 | 9 | [REDISCLUSTER] 10 | 11 | [REDIS] 12 | 13 | [ROCKSDB] 14 | 15 | -------------------------------------------------------------------------------- /h3cp/Makefile: -------------------------------------------------------------------------------- 1 | default: all 2 | 3 | h3wrapper: h3wrapper.h h3wrapper.c 4 | gcc -c h3wrapper.c 5 | 6 | all: h3wrapper main.go 7 | go build 8 | 9 | clean: 10 | rm -f h3wrapper.o h3cp 11 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | **/.DS_Store 3 | **/*.pyc 4 | **/__pycache__/ 5 | docs/ 6 | h3lib/build/ 7 | h3fuse/build/ 8 | pyh3lib/build/ 9 | pyh3lib/dist/ 10 | pyh3lib/*.egg-info/ 11 | pyh3lib/venv/ 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | venv/ 3 | *.egg-info 4 | build/ 5 | dist/ 6 | *.o 7 | *.so* 8 | *.bin 9 | .cproject 10 | .project 11 | .settings 12 | __pychache__ 13 | _build/ 14 | *.pyc 15 | Jclouds-H3/target/ 16 | Jclouds-H3/.idea/ 17 | Jclouds-H3/*.iml 18 | h3cp/h3cp 19 | docs/packages.rst 20 | docs/gr/ 21 | -------------------------------------------------------------------------------- /h3cp/README.rst: -------------------------------------------------------------------------------- 1 | h3cp utility 2 | ============ 3 | 4 | ``h3cp`` is simple example in `Go `_ that uses ``h3lib`` to copy files to and from H3:: 5 | 6 | Usage: h3cp -s 7 | 8 | **Note:** Use the ``h3:///`` notation to reference H3 objects. 9 | 10 | To build, compile and install ``h3lib``, then run ``make``. 11 | -------------------------------------------------------------------------------- /Jclouds-H3/config/s3proxy.conf: -------------------------------------------------------------------------------- 1 | s3proxy.secure-endpoint=https://0.0.0.0:8080 2 | s3proxy.keystore-path=keystore.jks 3 | s3proxy.keystore-password=CARVICS 4 | s3proxy.authorization=aws-v2-or-v4 5 | 6 | s3proxy.identity=test:tester 7 | s3proxy.credential=testing 8 | jclouds.identity=test:tester 9 | jclouds.credential=testing 10 | jclouds.provider=h3 11 | jclouds.h3.basedir=file:///tmp/demo_h3via_jclouds 12 | jclouds.regions=us-east-1 13 | jclouds.region= -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/predicates/validators/H3ContainerNameValidator.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.predicates.validators; 2 | 3 | import org.jclouds.predicates.Validator; 4 | 5 | /** 6 | * Validates container name for filesystem provider 7 | * 8 | * @see org.jclouds.rest.InputParamValidator 9 | * @see org.jclouds.predicates.Validator 10 | */ 11 | public abstract class H3ContainerNameValidator extends Validator { 12 | } 13 | -------------------------------------------------------------------------------- /h3controllers/README.rst: -------------------------------------------------------------------------------- 1 | H3Controllers in python3 2 | =========================== 3 | 4 | Build-in controllers to manage objects according to metadata (in this case ``ExpiresAt`` and ``ReadOnlyAfter``). 5 | 6 | Run 7 | ------------ 8 | 9 | python3 expiresAtController --storage "file:///tmp/h3" 10 | 11 | python3 readOnlyAfterController --storage "file:///tmp/h3" 12 | 13 | python3 cacheController --hot_storage="{hot_storage_url}" --cold_storage="{cold_storage_url}" 14 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/reference/H3Constants.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.reference; 2 | 3 | /** 4 | * Common constants used in h3 provider 5 | */ 6 | public final class H3Constants { 7 | 8 | /** Specify the base directory where provider starts its file operations - must exists */ 9 | public static final String PROPERTY_BASEDIR = "jclouds.h3.basedir"; 10 | private H3Constants() { 11 | throw new AssertionError("intentionally unimplemented"); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- 1 | H3 documentation 2 | ================ 3 | 4 | This includes design details and APIs. 5 | 6 | Installation 7 | ------------ 8 | 9 | Install dependencies in a virtual environment:: 10 | 11 | python3 -m venv venv 12 | source venv/bin/activate 13 | pip install -r requirements.txt 14 | 15 | Install Python library:: 16 | 17 | cd ../pyh3lib && ./setup.py install 18 | 19 | Compile docs:: 20 | 21 | make html 22 | 23 | Then open ``_build/html/index.html`` in a browser. 24 | -------------------------------------------------------------------------------- /pyh3lib/pyh3lib/__init__.py: -------------------------------------------------------------------------------- 1 | from .version import __version__ 2 | 3 | from .h3 import H3List, H3Bytes, H3 4 | from .h3_cache import H3Cache 5 | 6 | from .h3lib import FailureError as H3FailureError 7 | from .h3lib import InvalidArgsError as H3InvalidArgsError 8 | from .h3lib import StoreError as H3StoreError 9 | from .h3lib import ExistsError as H3ExistsError 10 | from .h3lib import NotExistsError as H3NotExistsError 11 | from .h3lib import NameTooLongError as H3NameTooLongError 12 | from .h3lib import NotEmptyError as H3NotEmptyError 13 | -------------------------------------------------------------------------------- /pyh3lib/pyh3lib/version.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = '1.2' 16 | -------------------------------------------------------------------------------- /docs/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | When initializing H3, through the ``H3_Init()`` function (or similar in Python and Java), you need to specify a storage URI to select the storage backend type and options. 5 | 6 | Example storage URIs include (defaults for each type shown): 7 | 8 | * ``file:///tmp/h3`` for local filesystem 9 | * ``kreon:///tmp/h3/kreon.dat`` for Kreon, where the file can also be block device 10 | * ``kreon-rdma://127.0.0.1:2181`` for distributed Kreon with RDMA, where the network location refers to the ZooKeeper host and port 11 | * ``rocksdb:///tmp/h3/rocksdb`` for `RocksDB `_ 12 | * ``redis://127.0.0.1:6379`` for `Redis `_ 13 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Exception.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * Generic JH3 Exception. Used when the client receives an unknown status from native h3lib 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3Exception extends Exception { 9 | 10 | public JH3Exception(String message) { 11 | super(message); 12 | } 13 | 14 | public JH3Exception(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | 18 | public JH3Exception(Throwable cause) { 19 | super(cause); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return super.toString(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/util/Utils.java: -------------------------------------------------------------------------------- 1 | 2 | package org.jclouds.h3.util; 3 | /** 4 | * Utilities for the filesystem blobstore. 5 | */ 6 | public class Utils { 7 | /** 8 | * Private constructor for utility class. 9 | */ 10 | private Utils() { 11 | // Do nothing 12 | } 13 | 14 | /** 15 | * Determine if Java is running on a Mac OS 16 | */ 17 | public static boolean isMacOSX() { 18 | String osName = System.getProperty("os.name"); 19 | return osName.contains("OS X"); 20 | } 21 | 22 | 23 | /** 24 | * Determine if Java is running on a windows OS 25 | */ 26 | public static boolean isWindows() { 27 | return System.getProperty("os.name", "").toLowerCase().contains("windows"); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /JH3lib/JH3HadoopFS/spark-examples/dataframes.spark: -------------------------------------------------------------------------------- 1 | // Write Dataframe to H3 as text 2 | val someDF = Seq("word1", "word2", "word4", "this is").toDF("word") 3 | someDF.write.text("h3://mybucket/dir1/df.txt") 4 | 5 | // Read text from H3 6 | spark.read.text("h3://mybucket/dir1/df.txt").take(4) 7 | 8 | // Create DataFrame 9 | val someDF = Seq((8, "word1"), (64, "word2"), (-27, "word4"), (104, "this is")).toDF("number", "word") 10 | 11 | // Write DataFrame to H3 as CSV 12 | someDF.write.csv("h3://mybucket/dir1/df.csv") 13 | // Read CSV File from H3 14 | spark.read.csv("h3://mybucket/dir1/df.csv").take(4) 15 | // Write DataFrame to H3 as json 16 | someDF.write.json("h3://mybucket/dir1/df.json") 17 | // Read Json from H3 18 | spark.read.json("h3://mybucket/dir1/df.json").take(4) 19 | 20 | -------------------------------------------------------------------------------- /JH3lib/JH3HadoopFS/spark-examples/binaryFile.spark: -------------------------------------------------------------------------------- 1 | import org.apache.hadoop.fs.{FileSystem, Path} 2 | import java.io.BufferedOutputStream 3 | import org.apache.hadoop.conf.Configuration 4 | 5 | 6 | // Write binary file "mybucket/test" to H3 7 | val path = new Path("h3://mybucket/test") 8 | val conf = new Configuration(sc.hadoopConfiguration) 9 | val fs = path.getFileSystem(conf) 10 | val output = fs.create(path) 11 | val os = new BufferedOutputStream(output) 12 | val txt = "This is a text from Spark" 13 | os.write(txt.getBytes("UTF-8")) 14 | os.close() 15 | 16 | 17 | // Read binary file from "mybucket/test" 18 | val data = sc.binaryFiles("h3://mybucket/test").map{case (fileName, pds) => {scala.io.Source.fromInputStream(pds.open()).getLines().toArray}} 19 | data.take(1)(0).foreach(println) 20 | 21 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | Programming APIs 2 | ================ 3 | 4 | Python 5 | ------ 6 | 7 | .. autoclass:: pyh3lib.H3 8 | :members: 9 | :undoc-members: 10 | :member-order: bysource 11 | 12 | .. autoattribute:: VERSION 13 | :annotation: 14 | 15 | Class property, which returns the current ``h3lib`` version. 16 | 17 | .. autoclass:: pyh3lib.H3List 18 | 19 | .. exception:: pyh3lib.H3FailureError 20 | 21 | .. exception:: pyh3lib.H3InvalidArgsError 22 | 23 | .. exception:: pyh3lib.H3StoreError 24 | 25 | .. exception:: pyh3lib.H3ExistsError 26 | 27 | .. exception:: pyh3lib.H3NotExistsError 28 | 29 | .. exception:: pyh3lib.H3NameTooLongError 30 | 31 | .. exception:: pyh3lib.H3NotEmptyError 32 | 33 | Java 34 | ---- 35 | 36 | .. toctree:: 37 | :maxdepth: 2 38 | 39 | packages 40 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Auth.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * User authentication info. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3Auth { 9 | 10 | private int userId; 11 | 12 | /** 13 | * Create user authentication info. 14 | * @param userId The id of the user 15 | */ 16 | public JH3Auth(int userId) { 17 | this.userId = userId; 18 | } 19 | 20 | /** 21 | * Get the ID of the user. 22 | * @return The ID of the user. 23 | */ 24 | public int getUserId() { 25 | return userId; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return "JH3Auth{" + 31 | "userId=" + userId + 32 | '}'; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3MultipartId.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * Information about multipart identification. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3MultipartId { 9 | private String multipartId; 10 | 11 | public JH3MultipartId(String multipartId) { 12 | this.multipartId = multipartId; 13 | } 14 | 15 | 16 | /** 17 | * Get the ID of the multipart object. 18 | * @return the ID of the multipart object. 19 | */ 20 | public String getMultipartId() { 21 | return multipartId; 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | return "JH3MultipartId{" + 27 | "multipartId='" + multipartId + '\'' + 28 | '}'; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /h3fuse/README.rst: -------------------------------------------------------------------------------- 1 | H3 FUSE filesystem 2 | ================== 3 | 4 | ``h3fuse`` is a FUSE filesystem implementation that uses H3 for storage. 5 | 6 | Installation 7 | ------------ 8 | 9 | To compile ``h3fuse`` you need CMake v3.10 or latter. 10 | 11 | **Note:** For CentOS 7, use the ``cmake3`` package available in EPEL. Replace ``cmake`` in the following commands with ``cmake3``. 12 | 13 | Install ``h3lib`` and ``libfuse-dev`` (you can use ``apt-get install libfuse-dev`` in Ubuntu or ``yum install fuse3 fuse3-devel`` in CentOS). 14 | 15 | **Note:** For `Kreon `_, use ``cmake -DCMAKE_EXE_LINKER_FLAGS="-lm" ..``. 16 | 17 | To build and install:: 18 | 19 | mkdir -p build && cd build 20 | cmake .. 21 | make 22 | make install 23 | 24 | To package (generates RPM file):: 25 | 26 | make package 27 | -------------------------------------------------------------------------------- /JH3lib/JH3HadoopFS/spark-examples/simpleWordCount.spark: -------------------------------------------------------------------------------- 1 | // First method (Datasets) 2 | case class InData(line: String) 3 | case class OutData(word: String) 4 | val dataDF = sc.textFile("dataset.json").toDF("line") 5 | dataDF.write.text("h3://mybucket/dir1/data.txt") 6 | val readDF = spark.read.text("h3://mybucket/dir1/data.txt").toDF("line") 7 | val wordsDF = readDF.as[InData].flatMap(l => l.line.split(' ').map(OutData)) 8 | val wordCountDF = wordsDF.groupBy("word").count() 9 | wordCountDF.show() 10 | 11 | // alternative (explode is deprecated) 12 | val dataDF = sc.textFile("dataset.json").toDF("line") 13 | dataDF.write.text("h3://mybucket/dir1/data2.txt") 14 | val readDF = spark.read.text("h3://mybucket/dir1/data2.txt").toDF("line") 15 | val wordsDF = readDF.explode("line", "word")((line: String) => line.split(" ")) 16 | val wordCountDF = wordsDF.groupBy("word").count() 17 | wordCountDF.show() 18 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | clean: 18 | @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 19 | @rm -rf gr/ # Remove directory created by javasphinx 20 | 21 | # Catch-all target: route all unknown targets to Sphinx using the new 22 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 23 | %: Makefile 24 | @javasphinx-apidoc -o . --title='JH3lib' ../JH3lib -f --no-member-headers 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /h3cp/h3wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright [2020] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef H3WRAPPER_H_ 16 | #define H3WRAPPER_H_ 17 | 18 | #include 19 | 20 | int h3lib_write_object_from_file(H3_Handle handle, char *bucketName, char *objectName, char *filename); 21 | int h3lib_read_object_to_file(H3_Handle handle, char *bucketName, char *objectName, char *filename); 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/predicates/validators/internal/H3BlobKeyValidatorImpl.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.predicates.validators.internal; 2 | 3 | import com.google.inject.Singleton; 4 | import org.jclouds.h3.predicates.validators.H3BlobKeyValidator; 5 | 6 | /** 7 | * Validates name for filesystem container blob keys implementation 8 | * 9 | * @see org.jclouds.rest.InputParamValidator 10 | * @see org.jclouds.predicates.Validator 11 | */ 12 | @Singleton 13 | public class H3BlobKeyValidatorImpl extends H3BlobKeyValidator { 14 | 15 | @Override 16 | public void validate(String name) throws IllegalArgumentException { 17 | //blob key cannot be null or empty 18 | if (name == null || name.length() < 1) 19 | throw new IllegalArgumentException("Blob key can't be null or empty"); 20 | 21 | //blobkey cannot start with / (or \ in Windows) character 22 | if (name.startsWith("\\") || name.startsWith("/")) 23 | throw new IllegalArgumentException("Blob key '" + name + "' cannot start with \\ or /"); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /pyh3lib/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from pyh3lib import H3 18 | 19 | def pytest_addoption(parser): 20 | parser.addoption('--storage', action='store', required=True, help="H3 storage URI") 21 | 22 | @pytest.fixture(scope='module') 23 | def h3(request): 24 | storage_uri = request.config.getoption('--storage') 25 | 26 | assert storage_uri, 'You need to specify a storage URI with "--storage"' 27 | return H3(storage_uri) 28 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/predicates/validators/internal/H3ContainerNameValidatorImpl.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.predicates.validators.internal; 2 | 3 | 4 | import com.google.inject.Singleton; 5 | import org.jclouds.h3.predicates.validators.H3ContainerNameValidator; 6 | 7 | /** 8 | * Validates container name for filesystem provider implementation 9 | * 10 | * @see org.jclouds.rest.InputParamValidator 11 | * @see org.jclouds.predicates.Validator 12 | */ 13 | @Singleton 14 | public class H3ContainerNameValidatorImpl extends H3ContainerNameValidator { 15 | 16 | @Override 17 | public void validate(String name) throws IllegalArgumentException { 18 | //container name cannot be null or empty 19 | if (name == null || name.length() < 1) 20 | throw new IllegalArgumentException("Container name can't be null or empty"); 21 | 22 | //container name cannot contains / (or \ in Windows) character 23 | if (name.contains("\\") || name.contains("/")) 24 | throw new IllegalArgumentException("Container name '" + name + "' cannot contain \\ or /"); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /JH3lib/JH3HadoopFS/src/main/java/gr/forth/ics/JH3lib/JH3HadoopAbstractFS.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.classification.InterfaceAudience; 5 | import org.apache.hadoop.classification.InterfaceStability; 6 | import org.apache.hadoop.fs.DelegateToFileSystem; 7 | 8 | import java.io.IOException; 9 | import java.net.URI; 10 | import java.net.URISyntaxException; 11 | 12 | /** 13 | * H3 implementation of AbstractFileSystem. 14 | * This implementation delegates to the H3FileSystem 15 | * @author Giorgos Kalaentzis 16 | * @version 0.1-beta 17 | */ 18 | 19 | @InterfaceAudience.Public 20 | @InterfaceStability.Evolving 21 | public class JH3HadoopAbstractFS extends DelegateToFileSystem { 22 | 23 | public JH3HadoopAbstractFS(URI theUri, Configuration conf) throws IOException, URISyntaxException { 24 | super(theUri, new JH3HadoopFS(), conf, "h3", false); 25 | } 26 | 27 | @Override 28 | public int getUriDefaultPort(){ 29 | // TODO Change to some predefined constant(e.g. Constants.H3_DEFAULT_PORT) 30 | return 8000; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pyh3lib/README.rst: -------------------------------------------------------------------------------- 1 | H3 Python wrapper and tests 2 | =========================== 3 | 4 | ``pyh3lib`` includes a Python interface for ``h3lib`` (``H3`` class) and tests for H3. 5 | 6 | Installation 7 | ------------ 8 | 9 | With ``h3lib`` and the ``python3-devel`` package installed (for CentOS), compile and install ``pyh3lib``, by running:: 10 | 11 | python3 setup.py install 12 | 13 | Or, to do it in a virtual environment:: 14 | 15 | python3 -m venv venv 16 | source venv/bin/activate 17 | python3 setup.py install 18 | 19 | If you have installed ``h3lib`` in some non-standard path and ``setup.py install`` fails to find it, run the following and retry (the example assumes ``h3lib`` is installed in ``/usr/local/``):: 20 | 21 | export C_INCLUDE_PATH=/usr/local/include 22 | export LIBRARY_PATH=/usr/local/lib 23 | 24 | To package, install ``python3-wheel`` (for CentOS) and run:: 25 | 26 | python3 setup.py bdist_wheel 27 | 28 | This will create a ``.whl`` file in ``dist``, which you can ``pip3 install``. 29 | 30 | To run tests:: 31 | 32 | mkdir /tmp/h3 33 | pytest -v -s --storage "file:///tmp/h3" tests 34 | -------------------------------------------------------------------------------- /Jclouds-H3/src/test/resources/logging.properties: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | # 21 | 22 | # Set the default logging level for all loggers to WARNING 23 | .level = INFO 24 | 25 | handlers = java.util.logging.ConsoleHandler 26 | java.util.logging.ConsoleHandler.level = ALL 27 | java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter 28 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. H3 documentation master file, created by 2 | sphinx-quickstart on Thu Nov 28 11:35:06 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to H3 7 | ============= 8 | 9 | H3 is an embedded High speed, High volume, and High availability object store, backed by a high-performance key-value store. 10 | H3 is implemented in the ``h3lib`` library, which provides a cloud-friendly API, similar to Amazon's S3. On the backend, H3 supports `RocksDB `_ for single-node runs, `Redis `_ for over-the-network storage, Kreon for local or distributed storage deployments, as well as plain files for layering on top of any filesystem (and easy testing). 11 | 12 | ``h3lib`` is written in C. Python and Java wrappers are also available. An H3 FUSE-based filesystem allows object access using file semantics. 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | :caption: Contents: 17 | 18 | design 19 | configuration 20 | api 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /pyh3lib/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from setuptools import setup, Extension 4 | 5 | version = {} 6 | with open("pyh3lib/version.py") as fp: 7 | exec(fp.read(), version) 8 | 9 | setup(name='pyh3lib', 10 | version=version['__version__'], 11 | description='Python interface to H3: A High speed, High Volume and Highly available object storage', 12 | url='https://www.ics.forth.gr/carv/', 13 | author='FORTH-ICS', 14 | license='Apache-2.0', 15 | packages=['pyh3lib'], 16 | ext_modules=[Extension('pyh3lib.h3lib', 17 | sources=['pyh3lib/h3lib.c'], 18 | libraries=['h3lib'])], 19 | entry_points={'console_scripts': ['h3cli = pyh3lib.cli:main']}, 20 | python_requires='>=3.6', 21 | classifiers=['Development Status :: 4 - Beta', 22 | 'Environment :: Console', 23 | 'Programming Language :: Python :: 3.6', 24 | 'Programming Language :: C' 25 | 'Operating System :: OS Independent', 26 | 'Topic :: Software Development :: Libraries :: Python Modules' 27 | 'License :: OSI Approved :: Apache Software License']) 28 | -------------------------------------------------------------------------------- /JH3lib/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | gr.forth.ics.JH3lib 6 | JH3lib 7 | pom 8 | 0.1-beta 9 | JH3lib 10 | http://maven.apache.org 11 | 12 | 13 | UTF-8 14 | 15 | 16 | 17 | 18 | 19 | org.apache.maven.plugins 20 | maven-compiler-plugin 21 | 3.8.1 22 | 23 | 8 24 | 8 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | JH3Interface 34 | JH3 35 | JH3HadoopFS 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3PartInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * Brief information on individual parts of a multipart object. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3PartInfo { 9 | private int partNumber; 10 | private long size; 11 | 12 | /** 13 | * 14 | * @param partNumber The part number of the multipart 15 | * @param size The size of the part 16 | */ 17 | public JH3PartInfo(int partNumber, long size) { 18 | this.partNumber = partNumber; 19 | this.size = size; 20 | } 21 | 22 | /** 23 | * Get the assigned part number inside of the multipart object. 24 | * @return the part number. 25 | */ 26 | public int getPartNumber() { 27 | return partNumber; 28 | } 29 | 30 | /** 31 | * Get the size of the part object. 32 | * @return the size of the part object. 33 | */ 34 | public long getSize() { 35 | return size; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return "JH3PartInfo{" + 41 | "partNumber=" + partNumber + 42 | ", size=" + size + 43 | '}'; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | REGISTRY_NAME?=carvicsforth 16 | H3_VERSION=$(shell cat VERSION) 17 | 18 | .PHONY: containers containers-push 19 | 20 | containers: 21 | docker build -f Dockerfile --target h3 -t $(REGISTRY_NAME)/h3:$(H3_VERSION) . 22 | docker build -f Dockerfile --target h3-s3proxy -t $(REGISTRY_NAME)/h3-s3proxy:$(H3_VERSION) . 23 | docker build -f Dockerfile --build-arg BUILD_TYPE=Debug --target h3-builder -t $(REGISTRY_NAME)/h3:$(H3_VERSION)-dev . 24 | 25 | containers-push: 26 | docker push $(REGISTRY_NAME)/h3:$(H3_VERSION) 27 | docker push $(REGISTRY_NAME)/h3-s3proxy:$(H3_VERSION) 28 | docker push $(REGISTRY_NAME)/h3:$(H3_VERSION)-dev 29 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeAuth.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.Pointer; 4 | import com.sun.jna.Structure; 5 | 6 | import java.util.Arrays; 7 | import java.util.List; 8 | 9 | /** 10 | * JNA representation of H3_Auth 11 | * @author Giorgos Kalaentzis 12 | * @version 0.1-beta 13 | */ 14 | public class NativeAuth extends Structure { 15 | public int userId; 16 | 17 | public NativeAuth() { super(); } 18 | 19 | @Override 20 | protected List getFieldOrder() { 21 | return Arrays.asList("userId"); 22 | } 23 | 24 | /** 25 | * Create a native authentication token. 26 | * @param userId The id of the user 27 | */ 28 | public NativeAuth(int userId){ 29 | super(); 30 | this.userId = userId; 31 | } 32 | 33 | /** 34 | * Create a native authentication token from the real native pointer. 35 | * @param peer Pointer value of the real native pointer 36 | */ 37 | public NativeAuth(Pointer peer) { super(peer); } 38 | 39 | public static class ByReference extends NativeAuth implements Structure.ByReference{ 40 | 41 | }; 42 | 43 | public static class ByValue extends NativeAuth implements Structure.ByValue{ 44 | 45 | }; 46 | 47 | } 48 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/predicates/validators/H3BlobKeyValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.jclouds.h3.predicates.validators; 18 | 19 | import org.jclouds.predicates.Validator; 20 | 21 | /** 22 | * Validates name for h3 container blob keys 23 | * 24 | * @see org.jclouds.rest.InputParamValidator 25 | * @see org.jclouds.predicates.Validator 26 | */ 27 | public abstract class H3BlobKeyValidator extends Validator { 28 | } 29 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Timespec.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * A time interval broken down in seconds and nanoseconds. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3Timespec { 9 | private long seconds; 10 | private long nanoseconds; 11 | 12 | /** 13 | * Create a Timespec object holding an interval. 14 | * @param seconds The seconds of the interval 15 | * @param nanoseconds The nanoseconds of the interval 16 | */ 17 | public JH3Timespec(long seconds, long nanoseconds) { 18 | this.seconds = seconds; 19 | this.nanoseconds = nanoseconds; 20 | } 21 | 22 | /** 23 | * Create a Timespec object without nanosecond resolution. 24 | * @param seconds The seconds of the interval 25 | */ 26 | public JH3Timespec(long seconds) { 27 | this.seconds = seconds; 28 | } 29 | 30 | public long getSeconds() { 31 | return this.seconds; 32 | } 33 | 34 | public long getNanoseconds() { 35 | return this.nanoseconds; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return "JH3Timespec{" + 41 | " seconds='" + seconds + "'" + 42 | ", nanoseconds='" + nanoseconds + "'" + 43 | "}"; 44 | } 45 | 46 | } -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Attribute.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | // TODO Add javadoc of usage etc 4 | public class JH3Attribute { 5 | 6 | private JH3AttributeType type; 7 | private int mode; 8 | private int uid; 9 | private int gid; 10 | 11 | public JH3Attribute(JH3AttributeType type, int mode) { 12 | this.type = type; 13 | this.mode = mode; 14 | this.uid = -1; 15 | this.gid = -1; 16 | } 17 | 18 | public JH3Attribute(JH3AttributeType type, int uid, int gid){ 19 | this.type = type; 20 | this.mode = -1; 21 | this.uid = -1; 22 | this.gid = -1; 23 | } 24 | 25 | // TODO only show mode / uid+gid based on Attribute type; 26 | @Override 27 | public String toString() { 28 | return "JH3Attribute{" + 29 | " type='" + type + "'" + 30 | ", mode='" + mode + "'" + 31 | ", uid='" + uid + "'" + 32 | ", gid='" + gid + "'" + 33 | "}"; 34 | } 35 | 36 | 37 | public JH3AttributeType getType() { 38 | return this.type; 39 | } 40 | 41 | public int getMode() { 42 | return this.mode; 43 | } 44 | 45 | public int getUid() { 46 | return this.uid; 47 | } 48 | 49 | public int getGid() { 50 | return this.gid; 51 | } 52 | 53 | 54 | } -------------------------------------------------------------------------------- /h3lib/url_parser.h: -------------------------------------------------------------------------------- 1 | /*_ 2 | * Copyright 2010 Scyphus Solutions Co. Ltd. All rights reserved. 3 | * 4 | * Authors: 5 | * Hirochika Asai 6 | */ 7 | 8 | /* 9 | * Original code found at: http://draft.scyphus.co.jp/lang/c/url_parser.html. 10 | * Changes marked with "FIXED". 11 | * 12 | * Copyright 2020 FORTH-ICS 13 | * 14 | * Authors: 15 | * Antony Chazapis 16 | */ 17 | 18 | #ifndef _URL_PARSER_H 19 | #define _URL_PARSER_H 20 | 21 | /* 22 | * URL storage 23 | */ 24 | struct parsed_url { 25 | char *scheme; /* mandatory */ 26 | char *host; /* FIXED: optional */ 27 | char *port; /* optional */ 28 | char *path; /* optional */ 29 | char *query; /* optional */ 30 | char *fragment; /* optional */ 31 | char *username; /* optional */ 32 | char *password; /* optional */ 33 | }; 34 | 35 | #ifdef __cplusplus 36 | extern "C" { 37 | #endif 38 | 39 | /* 40 | * Declaration of function prototypes 41 | */ 42 | struct parsed_url * parse_url(const char *); 43 | void parsed_url_free(struct parsed_url *); 44 | 45 | #ifdef __cplusplus 46 | } 47 | #endif 48 | 49 | #endif /* _URL_PARSER_H */ 50 | 51 | /* 52 | * Local variables: 53 | * tab-width: 4 54 | * c-basic-offset: 4 55 | * End: 56 | * vim600: sw=4 ts=4 fdm=marker 57 | * vim<600: sw=4 ts=4 58 | */ 59 | -------------------------------------------------------------------------------- /Jclouds-H3/config/run-docker-container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec java \ 4 | -DLOG_LEVEL=${LOG_LEVEL} \ 5 | -Ds3proxy.secure-endpoint=${S3PROXY_ENDPOINT} \ 6 | -Ds3proxy.virtual-host=${S3PROXY_VIRTUALHOST} \ 7 | -Ds3proxy.authorization=${S3PROXY_AUTHORIZATION} \ 8 | -Ds3proxy.identity=${S3PROXY_IDENTITY} \ 9 | -Ds3proxy.keystore-path=${S3PROXY_KEYSTORE_PATH} \ 10 | -Ds3proxy.keystore-password=${S3PROXY_KEYSTORE_PASSWORD} \ 11 | -Ds3proxy.credential=${S3PROXY_CREDENTIAL} \ 12 | -Ds3proxy.cors-allow-all=${S3PROXY_CORS_ALLOW_ALL} \ 13 | -Ds3proxy.cors-allow-origins="${S3PROXY_CORS_ALLOW_ORIGINS}" \ 14 | -Ds3proxy.cors-allow-methods="${S3PROXY_CORS_ALLOW_METHODS}" \ 15 | -Ds3proxy.cors-allow-headers="${S3PROXY_CORS_ALLOW_HEADERS}" \ 16 | -Ds3proxy.ignore-unknown-headers=${S3PROXY_IGNORE_UNKNOWN_HEADERS} \ 17 | -Djclouds.provider=${JCLOUDS_PROVIDER} \ 18 | -Djclouds.identity=${JCLOUDS_IDENTITY} \ 19 | -Djclouds.credential=${JCLOUDS_CREDENTIAL} \ 20 | -Djclouds.region=${JCLOUDS_REGION} \ 21 | -Djclouds.regions=${JCLOUDS_REGIONS} \ 22 | -Djclouds.keystone.version=${JCLOUDS_KEYSTONE_VERSION} \ 23 | -Djclouds.keystone.scope=${JCLOUDS_KEYSTONE_SCOPE} \ 24 | -Djclouds.keystone.project-domain-name=${JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME} \ 25 | -Djclouds.h3.basedir=${JCLOUDS_BASEDIR} \ 26 | -jar /opt/s3proxy/s3proxy \ 27 | --properties /dev/null 28 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3AttributeType.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import java.util.EnumSet; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * The attribute type of a bucket / object. 9 | * 10 | * @author Giorgos Kalaentzis 11 | * @version 0.1-beta 12 | */ 13 | public enum JH3AttributeType { 14 | 15 | /** Owner attributes. */ 16 | JH3_ATTRIBUTE_OWNER(JH3Interface.AttributeType.H3_ATTRIBUTE_OWNER), 17 | /** Permission attribute. */ 18 | JH3_ATTRIBUTE_PERMISSIONS(JH3Interface.AttributeType.H3_ATTRIBUTE_PERMISSIONS), 19 | /** Not an option, used for iteration purposes. */ 20 | JH3_NumOfAttributes(JH3Interface.AttributeType.H3_NumOfAttributes); 21 | 22 | private final int type; 23 | private static final Map lookup = new HashMap<>(); 24 | 25 | // Map int -> JH3AttributeType enum for easy lookup 26 | static { 27 | for(JH3AttributeType t : EnumSet.allOf(JH3AttributeType.class)){ 28 | lookup.put(t.getAttributeType(), t); 29 | } 30 | } 31 | JH3AttributeType(int type){ 32 | this.type = type; 33 | } 34 | 35 | public int getAttributeType(){ 36 | return type; 37 | } 38 | 39 | /** 40 | * Get a JH3AttributeType from its integer representation 41 | * @param id Integer representation of the attribute type 42 | * @return The attribute type as enum 43 | */ 44 | public static JH3AttributeType fromInt(int id){ 45 | return lookup.get(id); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeTimespec.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.NativeLong; 4 | import com.sun.jna.Structure; 5 | import com.sun.jna.Pointer; 6 | 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | /** 11 | * JNA representation of timespec struct. 12 | * This struct holds an interval broken down into seconds and nanoseconds. 13 | */ 14 | public class NativeTimespec extends Structure { 15 | 16 | public NativeLong tv_sec; 17 | public NativeLong tv_nsec; 18 | 19 | public NativeTimespec(){ super();} 20 | 21 | /** 22 | * Create a Timespec object. 23 | * @param tv_sec The seconds of the interval. 24 | * @param tv_nsec The nanoseconds of the interval. 25 | */ 26 | public NativeTimespec(NativeLong tv_sec, NativeLong tv_nsec){ 27 | super(); 28 | this.tv_sec = tv_sec; 29 | this.tv_nsec = tv_nsec; 30 | } 31 | 32 | /** 33 | * Create native object info from the real native pointer. 34 | * @param peer Pointer value of the real native pointer 35 | */ 36 | public NativeTimespec(Pointer peer){ 37 | super(peer); 38 | } 39 | 40 | @Override 41 | protected List getFieldOrder() { 42 | return Arrays.asList("tv_sec", "tv_nsec"); 43 | } 44 | 45 | public static class ByReference extends NativeTimespec implements Structure.ByReference { 46 | 47 | }; 48 | public static class ByValue extends NativeTimespec implements Structure.ByValue { 49 | 50 | }; 51 | } 52 | -------------------------------------------------------------------------------- /h3lib/README.rst: -------------------------------------------------------------------------------- 1 | H3 library 2 | ========== 3 | 4 | ``h3lib`` is an embedded object store in C. 5 | 6 | Installation 7 | ------------ 8 | 9 | To compile ``h3lib`` you need CMake v3.10 or latter. H3 uses the ``glib2`` and ``uuid`` libraries, so make sure you have them installed, along with their header files. You also need the ``cppcheck`` utility. 10 | 11 | **Note:** For CentOS 7, use the ``cmake3`` package available in EPEL. Replace ``cmake`` in the following commands with ``cmake3``. 12 | 13 | **Note:** For macOS, install ``cmake``, ``pkgconfig``, ``glib2``, and ``libuuid`` via MacPorts. ``h3lib`` requires GCC to build, so you also need to install ``gcc9`` and replace the following ``cmake`` command with: ``export CC=; cmake -DCMAKE_C_COMPILER=$CC `` 14 | 15 | For key-value store plugins install the appropriate library dependencies. ``h3lib`` will build and link with any such libraries available: 16 | 17 | * For `Kreon `_, use ``cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON ..``. 18 | * For `RocksDB `_, instal all dependencies as per https://github.com/facebook/rocksdb/blob/master/INSTALL.md (``make shared_lib && make install-shared``). 19 | * For `Redis `_, install the ``hiredis`` client library. 20 | 21 | To enable compression, add the ``-DH3LIB_USE_COMPRESSION`` flag to the ``cmake`` command. 22 | 23 | To build and install:: 24 | 25 | mkdir -p build && cd build 26 | cmake .. 27 | make 28 | make install 29 | 30 | To package (generates RPM file):: 31 | 32 | make package 33 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3BucketInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | 4 | /** 5 | * Bucket statistics 6 | * @author Giorgos Kalaentzis 7 | * @version 0.1-beta 8 | */ 9 | public class JH3BucketInfo { 10 | 11 | private JH3Timespec creation; 12 | private JH3BucketStats stats; 13 | 14 | /** 15 | * Create a BucketInfo object with statistics. 16 | * @param creation Creation timestamp 17 | * @param stats Aggregate object statistics 18 | */ 19 | public JH3BucketInfo(JH3Timespec creation, JH3BucketStats stats) { 20 | this.creation = creation; 21 | this.stats = stats; 22 | } 23 | 24 | /** 25 | * Create a BucketInfo object without statistics. Stats are set to null. 26 | * @param creation Creation timestamp 27 | */ 28 | public JH3BucketInfo(JH3Timespec creation) { 29 | this.creation = creation; 30 | this.stats = null; 31 | } 32 | 33 | /** 34 | * Get the creation timestamp. 35 | * @return The creation timestamp. 36 | */ 37 | public JH3Timespec getCreation() { 38 | return creation; 39 | } 40 | 41 | /** 42 | * Get the aggregate bucket stats. 43 | * @return The aggregate bucket stats if they were collected, null otherwise; 44 | */ 45 | public JH3BucketStats getStats() { 46 | return stats; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "JH3BucketInfo{" + 52 | "creation=" + creation + 53 | ", stats=" + stats + 54 | '}'; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /h3lib/cmake/Modules/Findhiredis.cmake: -------------------------------------------------------------------------------- 1 | #https://gitlab.kitware.com/cmake/community/-/wikis/doc/tutorials/How-To-Find-Libraries 2 | 3 | # - Try to find Redis client (HIREDIS) 4 | # Once done this will define 5 | # HIREDIS_FOUND - System has Redis client 6 | # HIREDIS_INCLUDE_DIRS - The client's include directories 7 | # HIREDIS_LIBRARIES - The libraries needed to use the client 8 | # HIREDIS_DEFINITIONS - Compiler switches required for using the client 9 | 10 | 11 | # Use pkg-config to detect include/library paths hiredis 12 | find_package(PkgConfig) 13 | pkg_check_modules(PC_HIREDIS QUIET hiredis) 14 | set(HIREDIS_DEFINITIONS ${PC_HIREDIS_CFLAGS_OTHER}) 15 | 16 | 17 | # Dependencies use plural forms, the package itself uses the singular forms defined by find_path and find_library 18 | find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h 19 | HINTS ${PC_HIREDIS_INCLUDEDIR} ${PC_HIREDIS_INCLUDE_DIRS} 20 | PATH_SUFFIXES include ) 21 | 22 | find_library(HIREDIS_LIBRARIES hiredis 23 | HINTS ${PC_HIREDIS_LIBDIR} ${PC_HIREDIS_LIBRARY_DIRS} ) 24 | 25 | 26 | 27 | 28 | # Call the find_package_handle_standard_args() macro to set the _FOUND variable and print a success or failure message 29 | include(FindPackageHandleStandardArgs) 30 | find_package_handle_standard_args(hiredis DEFAULT_MSG HIREDIS_LIBRARIES HIREDIS_INCLUDE_DIR) 31 | 32 | 33 | # https://cmake.org/cmake/help/latest/command/mark_as_advanced.html?highlight=s 34 | # Don't show vars in CMakeGUI unless the "show advanced" option is set 35 | #mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARIES ) 36 | 37 | 38 | if(HIREDIS_FOUND) 39 | message(STATUS "Hiredis found") 40 | else() 41 | message(STATUS "Hiredis not found") 42 | endif() 43 | 44 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Object.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import java.util.Arrays; 4 | 5 | /** 6 | * A basic JH3 object. Holds the data as a byte array and its size. 7 | * @author Giorgos Kalaentzis 8 | * @version 0.1-beta 9 | */ 10 | public class JH3Object { 11 | private byte[] data; 12 | private long size; 13 | 14 | /** 15 | * Create an JH3Object. 16 | * @param data The data in bytes to be written / were read 17 | * @param size The size of the data 18 | */ 19 | public JH3Object(byte[] data, long size) { 20 | this.data = data; 21 | this.size = size; 22 | } 23 | 24 | public JH3Object() { 25 | this.data = new byte[0]; 26 | this.size = 0; 27 | } 28 | 29 | /** 30 | * Sets the data of the JH3Object. 31 | * @param data the data of the object. 32 | */ 33 | public void setData(byte[] data) { 34 | this.data = data; 35 | } 36 | 37 | /** 38 | * Sets the size of the JH3Object. 39 | * @param size the size of the object. 40 | */ 41 | public void setSize(long size) { 42 | this.size = size; 43 | } 44 | 45 | /** 46 | * Get the data of the JH3Object. 47 | * @return the data of the object. 48 | */ 49 | public byte[] getData() { 50 | return data; 51 | } 52 | 53 | /** 54 | * Get the size of the JH3Object. 55 | * @return the size of the object. 56 | */ 57 | public long getSize() { 58 | return size; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "JH3Object{" + 64 | "data=" + Arrays.toString(data) + 65 | ", size=" + size + 66 | '}'; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /h3lib/cmake/Modules/Findkreon.cmake: -------------------------------------------------------------------------------- 1 | #https://gitlab.kitware.com/cmake/community/-/wikis/doc/tutorials/How-To-Find-Libraries 2 | 3 | # - Try to find Kreon 4 | # Once done this will define 5 | # KREON_FOUND - System has Kreon 6 | # KREON_INCLUDE_DIRS - The Kreon include directories 7 | # KREON_LIBRARIES - The libraries needed to use Kreon 8 | # KREON_DEFINITIONS - Compiler switches required for using Kreon 9 | 10 | 11 | # Use pkg-config to detect include/library paths kreon_client kreon_rdma kreonr 12 | find_package(PkgConfig) 13 | pkg_check_modules(PC_KREON QUIET kreon) 14 | set(KREON_DEFINITIONS ${PC_KREON_CFLAGS_OTHER}) 15 | 16 | 17 | # Dependencies use plural forms, the package itself uses the singular forms defined by find_path and find_library 18 | find_path(KREON_INCLUDE_DIR kreon.h 19 | HINTS ${PC_KREON_INCLUDEDIR} ${PC_KREON_INCLUDE_DIRS} 20 | PATH_SUFFIXES include ) 21 | 22 | find_library(KREON_LIBRARY kreon 23 | HINTS ${PC_KREON_LIBDIR} ${PC_KREON_LIBRARY_DIRS} ) 24 | 25 | # Call the find_package_handle_standard_args() macro to set the _FOUND variable and print a success or failure message 26 | include(FindPackageHandleStandardArgs) 27 | 28 | 29 | find_package_handle_standard_args(kreon DEFAULT_MSG 30 | KREON_LIBRARY KREON_INCLUDE_DIR) 31 | 32 | 33 | # https://cmake.org/cmake/help/latest/command/mark_as_advanced.html?highlight=s 34 | # Don't show vars in CMakeGUI unless the "show advanced" option is set 35 | #mark_as_advanced(KREON_INCLUDE_DIR KREON_LIBRARY ) 36 | 37 | 38 | if(KREON_FOUND) 39 | set(KREON_LIBRARIES ${KREON_LIBRARY}) 40 | set(KREON_DEFINITIONS ${KREON_INCLUDE_DIR}) 41 | message(STATUS "Kreon found") 42 | else() 43 | message(STATUS "Kreon not found") 44 | endif() 45 | 46 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativePartInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.NativeLong; 4 | import com.sun.jna.Pointer; 5 | import com.sun.jna.Structure; 6 | 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | /** 11 | * JNA representation of H3_PartInfo 12 | * @author Giorgos Kalaentzis 13 | * @version 0.1-beta 14 | */ 15 | public class NativePartInfo extends Structure { 16 | public int partNumber; //!< Part number 17 | public NativeLong size; //!< Part size 18 | 19 | public NativePartInfo() { 20 | super(); 21 | } 22 | 23 | @Override 24 | protected List getFieldOrder() { 25 | return Arrays.asList("partNumber", "size"); 26 | } 27 | 28 | /** 29 | * Create native part info. 30 | * @param partNumber Part number 31 | * @param size Part size 32 | */ 33 | public NativePartInfo(int partNumber, NativeLong size) { 34 | super(); 35 | this.partNumber = partNumber; 36 | this.size = size; 37 | } 38 | 39 | /** 40 | * Create native part info from the real native pointer. 41 | * @param peer Pointer value of the real native pointer 42 | */ 43 | public NativePartInfo(Pointer peer) { 44 | super(peer); 45 | } 46 | 47 | public static class ByReference extends NativePartInfo implements Structure.ByReference { 48 | 49 | }; 50 | public static class ByValue extends NativePartInfo implements Structure.ByValue { 51 | 52 | }; 53 | 54 | @Override 55 | public String toString() { 56 | return "NativePartInfo{" + 57 | "partNumber=" + partNumber + 58 | ", size=" + size + 59 | '}'; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeBucketInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.Pointer; 4 | import com.sun.jna.Structure; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | /** 9 | * Brief bucket information 10 | * @author Giorgos Kalaentzis 11 | * @version 0.1-beta 12 | */ 13 | public class NativeBucketInfo extends Structure { 14 | 15 | public NativeTimespec creation; 16 | public NativeBucketStats stats; 17 | 18 | public NativeBucketInfo(){ super(); } 19 | 20 | @Override 21 | protected List getFieldOrder() { 22 | return Arrays.asList("creation", "stats"); 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "NativeBucketInfo{" + 28 | "creation=" + creation + 29 | ", stats=" + stats + 30 | '}'; 31 | } 32 | 33 | /** 34 | * Create native bucket info 35 | * @param creation Creation timestamp 36 | * @param mode Bucket type and mode (used by h3fuse) 37 | * @param stats Aggregate object statistics 38 | */ 39 | public NativeBucketInfo(NativeTimespec creation, NativeBucketStats stats) { 40 | super(); 41 | this.creation = creation; 42 | this.stats = stats; 43 | } 44 | 45 | /** 46 | * Create native bucket info from the real native pointer 47 | * @param peer Pointer value of the real native pointer 48 | */ 49 | public NativeBucketInfo(Pointer peer) { super(peer); } 50 | 51 | public static class ByReference extends NativeBucketInfo implements Structure.ByReference { 52 | 53 | }; 54 | 55 | public static class ByValue extends NativeBucketInfo implements Structure.ByValue { 56 | 57 | }; 58 | 59 | } 60 | -------------------------------------------------------------------------------- /JH3lib/JH3/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | gr.forth.ics.JH3lib 8 | JH3lib 9 | 0.1-beta 10 | 11 | 12 | JH3 13 | 0.1-beta 14 | JH3 15 | http://maven.apache.org 16 | 17 | 18 | 19 | 20 | maven-assembly-plugin 21 | 3.1.1 22 | 23 | 24 | jar-with-dependencies 25 | 26 | 27 | 28 | 29 | make-assembly 30 | package 31 | 32 | single 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | gr.forth.ics.JH3lib 44 | JH3Interface 45 | 0.1-beta 46 | compile 47 | 48 | 49 | 50 | junit 51 | junit 52 | 4.13.1 53 | test 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /.github/workflows/build-and-test.yaml: -------------------------------------------------------------------------------- 1 | name: Build and test 2 | on: push 3 | jobs: 4 | build-and-test: 5 | name: Build and test 6 | runs-on: ubuntu-latest 7 | container: centos:7.7.1908 8 | steps: 9 | - name: Checkout code 10 | uses: actions/checkout@v2 11 | - name: Install dependencies 12 | run: | 13 | yum groupinstall -y "Development Tools" 14 | yum install -y epel-release 15 | yum install -y cmake3 glib2-devel libuuid-devel hiredis-devel cppcheck fuse3 fuse3-devel python3-devel python3-wheel 16 | - name: Build and install h3lib 17 | working-directory: h3lib 18 | env: 19 | BUILD_TYPE: Release 20 | run: | 21 | mkdir build 22 | (cd build && cmake3 -DCMAKE_INSTALL_PREFIX="/usr" -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" .. && make package && make install) 23 | - name: Build and install h3fuse 24 | working-directory: h3fuse 25 | env: 26 | BUILD_TYPE: Release 27 | run: | 28 | mkdir build 29 | (cd build && cmake3 -DCMAKE_INSTALL_PREFIX="/usr" -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" .. && make package && make install) 30 | - name: Build and install pyh3lib 31 | working-directory: pyh3lib 32 | run: | 33 | ./setup.py bdist_wheel 34 | pip3 install dist/pyh3lib-1.2-cp36-cp36m-linux_x86_64.whl 35 | - name: Upload packages 36 | uses: actions/upload-artifact@v2 37 | with: 38 | name: packages 39 | path: | 40 | h3lib/build/h3lib*.rpm 41 | h3fuse/build/h3fuse*.rpm 42 | pyh3lib/dist/pyh3lib*.whl 43 | - name: Run tests 44 | working-directory: pyh3lib 45 | run: | 46 | pip3 install pytest 47 | mkdir /tmp/h3 48 | pytest -v -s --storage "file:///tmp/h3" tests 49 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 4.0.0 6 | 7 | 8 | gr.forth.ics.JH3lib 9 | JH3lib 10 | 0.1-beta 11 | 12 | 13 | JH3Interface 14 | 0.1-beta 15 | JH3Interface 16 | http://maven.apache.org 17 | 18 | 19 | 20 | 21 | maven-assembly-plugin 22 | 3.1.1 23 | 24 | 25 | jar-with-dependencies 26 | 27 | 28 | 29 | 30 | make-assembly 31 | package 32 | 33 | single 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | net.java.dev.jna 45 | jna 46 | 5.2.0 47 | 48 | 49 | 50 | junit 51 | junit 52 | 4.13.1 53 | test 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /h3lib/util.h: -------------------------------------------------------------------------------- 1 | // Copyright [2019] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef UTIL_H_ 16 | #define UTIL_H_ 17 | 18 | #include 19 | #include 20 | 21 | // Use typeof to make sure each argument is evaluated only once 22 | // https://gcc.gnu.org/onlinedocs/gcc-4.9.2/gcc/Typeof.html#Typeof 23 | #define max(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a > _b ? _a : _b; }) 24 | #define min(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) 25 | 26 | #define H3_MSG_SIZE 1024 27 | #define H3_HEADER_SIZE 80 28 | 29 | #ifdef DEBUG 30 | #define LogActivity(level, ...) _LogActivity(level, __func__, __LINE__, __VA_ARGS__) 31 | #else 32 | #define LogActivity(level, ...) 33 | #endif 34 | 35 | #define __1KByte 1024 36 | #define __1MByte (1024 * __1KByte) 37 | #define __1GByte (1024 * __1MByte) 38 | 39 | typedef enum{ 40 | H3_INFO_MSG = 0, 41 | H3_DEBUG_MSG, 42 | H3_ERROR_MSG, 43 | H3_NumberOfLevels 44 | }H3_MsgLevel; 45 | 46 | 47 | void _LogActivity(H3_MsgLevel level, const char* function, int lineNumber, const char *format, ...); 48 | int64_t Compare(struct timespec* a, struct timespec* b); 49 | struct timespec Posterior(struct timespec* a, struct timespec* b); 50 | struct timespec Anterior(struct timespec* a, struct timespec* b); 51 | void* ReAllocFreeOnFail(void* buffer, size_t size); 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/config/H3ContextModule.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.config; 2 | 3 | import com.google.inject.AbstractModule; 4 | import org.jclouds.blobstore.BlobRequestSigner; 5 | import org.jclouds.blobstore.BlobStore; 6 | import org.jclouds.blobstore.LocalBlobRequestSigner; 7 | // import org.jclouds.blobstore.LocalStorageStrategy; 8 | // import org.jclouds.blobstore.attr.ConsistencyModel; 9 | import org.jclouds.blobstore.LocalStorageStrategy; 10 | import org.jclouds.blobstore.attr.ConsistencyModel; 11 | import org.jclouds.blobstore.config.BlobStoreObjectModule; 12 | import org.jclouds.blobstore.util.BlobUtils; 13 | import org.jclouds.h3.blobStore.H3BlobStore; 14 | import org.jclouds.h3.predicates.validators.H3BlobKeyValidator; 15 | import org.jclouds.h3.predicates.validators.H3ContainerNameValidator; 16 | import org.jclouds.h3.predicates.validators.internal.H3BlobKeyValidatorImpl; 17 | import org.jclouds.h3.predicates.validators.internal.H3ContainerNameValidatorImpl; 18 | import org.jclouds.h3.strategy.internal.H3StorageStrategyImpl; 19 | import org.jclouds.h3.util.internal.H3BlobUtilsImpl; 20 | 21 | import static org.jclouds.h3.util.Utils.isWindows; 22 | 23 | public class H3ContextModule extends AbstractModule { 24 | @Override 25 | protected void configure() { 26 | bind(BlobStore.class).to(H3BlobStore.class); 27 | install(new BlobStoreObjectModule()); 28 | if (isWindows()) { 29 | bind(ConsistencyModel.class).toInstance(ConsistencyModel.EVENTUAL); 30 | } else { 31 | bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT); 32 | } 33 | bind(LocalStorageStrategy.class).to(H3StorageStrategyImpl.class); 34 | bind(BlobUtils.class).to(H3BlobUtilsImpl.class); 35 | bind(H3BlobKeyValidator.class).to(H3BlobKeyValidatorImpl.class); 36 | bind(H3ContainerNameValidator.class).to(H3ContainerNameValidatorImpl.class); 37 | bind(BlobRequestSigner.class).to(LocalBlobRequestSigner.class); 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | H3 2 | === 3 | 4 | H3 is an embedded High speed, High volume, and High availability object store, backed by a high-performance key-value store (Kreon, RocksDB, Redis, etc.) or a filesystem. H3 is implemented in the h3lib library, which provides a C-based cloud-friendly API, similar to Amazon's S3. Python and Java wrappers are also available. The H3 FUSE filesystem allows object access using file semantics. 5 | 6 | For design details and APIs, consult the `documentation `_. 7 | 8 | H3 is developed by the `Computer Architecture and VLSI Systems (CARV) `_ lab of the `Institute of Computer Science (ICS) `_ at the `Foundation for Research and Technology Hellas (FORTH) `_ and is available under the Apache License, Version 2.0. There is also a `list of contributors `_. 9 | 10 | Installation 11 | ------------ 12 | 13 | To install locally, check the ``README`` file in each folder. 14 | 15 | A Docker image is also `available `_. 16 | 17 | Docker images are built with `make containers` and uploaded with `make containers-push`. 18 | 19 | Extensions 20 | ---------- 21 | 22 | Use the `h3-benchmark `_ to measure H3 performance. 23 | 24 | The `CSI H3 mount plugin `_ (``csi-h3`` for short), allows you to use H3 FUSE for implementing persistent volumes in Kubernetes. 25 | 26 | The `h3-support branch `_ in our Argo fork adds support for using H3 as a workflow artifact repository. `Argo `_ is a workflow engine for Kubernetes. 27 | 28 | Acknowledgements 29 | ---------------- 30 | This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 825061 (EVOLVE - `website `_, `CORDIS `_). 31 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3Status.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import java.util.EnumSet; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * The status of a h3lib operation. 9 | * @author Giorgos Kalaentzis 10 | * @version 0.1-beta 11 | */ 12 | public enum JH3Status { 13 | 14 | /** Operation failed. */ 15 | JH3_FAILURE(JH3Interface.Status.H3_FAILURE), 16 | /** Arguments are missing or malformed. */ 17 | JH3_INVALID_ARGS(JH3Interface.Status.H3_INVALID_ARGS), 18 | /** External (store provider) error. */ 19 | JH3_STORE_ERROR(JH3Interface.Status.H3_STORE_ERROR), 20 | /** Bucket or object already exists. */ 21 | JH3_EXISTS(JH3Interface.Status.H3_EXISTS), 22 | /** Bucket or object does not exist. */ 23 | JH3_NOT_EXISTS(JH3Interface.Status.H3_NOT_EXISTS), 24 | /** Bucker or object name is too long */ 25 | JH3_NAME_TOO_LONG(JH3Interface.Status.H3_NAME_TOO_LONG), 26 | /** Bucket is not empty. */ 27 | JH3_NOT_EMPTY(JH3Interface.Status.H3_NOT_EMPTY), 28 | /** Operation succeeded. */ 29 | JH3_SUCCESS(JH3Interface.Status.H3_SUCCESS), 30 | /** Operation succeeded though there are more data to retrieve. */ 31 | JH3_CONTINUE(JH3Interface.Status.H3_CONTINUE); 32 | 33 | private final int status; 34 | private static final Map lookup = new HashMap<>(); 35 | 36 | 37 | // Map int -> JH3Status enum for easy lookup later 38 | static { 39 | for( JH3Status s : EnumSet.allOf(JH3Status.class)){ 40 | lookup.put(s.getStatus(), s); 41 | } 42 | } 43 | 44 | JH3Status(int status){ 45 | this.status = status; 46 | } 47 | 48 | public int getStatus() { 49 | return status; 50 | } 51 | 52 | /** 53 | * Get a JH3Status enum from its integer representation. 54 | * @param id Integer representation of the status 55 | * @return The status as enum 56 | */ 57 | public static JH3Status fromInt(int id) { 58 | return lookup.get(id); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/H3ApiMetadata.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3; 2 | 3 | //import static org.jclouds.filesystem.reference.FilesystemConstants.PROPERTY_AUTO_DETECT_CONTENT_TYPE; 4 | import java.net.URI; 5 | import java.util.Properties; 6 | import org.jclouds.apis.ApiMetadata; 7 | import org.jclouds.apis.internal.BaseApiMetadata; 8 | import org.jclouds.blobstore.BlobStoreContext; 9 | import com.google.auto.service.AutoService; 10 | import org.jclouds.h3.config.H3ContextModule; 11 | import org.jclouds.rest.internal.BaseHttpApiMetadata; 12 | 13 | /** 14 | * Implementation of {@link ApiMetadata} for jclouds H3 15 | */ 16 | @AutoService(ApiMetadata.class) 17 | public class H3ApiMetadata extends BaseApiMetadata { 18 | 19 | @Override 20 | public Builder toBuilder() { 21 | return new Builder().fromApiMetadata(this); 22 | } 23 | 24 | public H3ApiMetadata() { 25 | super(new Builder()); 26 | } 27 | 28 | protected H3ApiMetadata(Builder builder) { 29 | super(builder); 30 | } 31 | 32 | @Override 33 | public Properties getDefaultProperties() { 34 | Properties properties = BaseHttpApiMetadata.defaultProperties(); 35 | // properties.setProperty(PROPERTY_AUTO_DETECT_CONTENT_TYPE, "false"); 36 | return properties; 37 | } 38 | 39 | public static class Builder extends BaseApiMetadata.Builder { 40 | 41 | protected Builder() { 42 | id("h3") 43 | .name("h3 api") 44 | .identityName("Unused") 45 | .defaultEndpoint("http://localhost/transient") 46 | .defaultIdentity(System.getProperty("user.name")) 47 | .defaultCredential("bar") 48 | .version("1") 49 | .documentation(URI.create("http://www.jclouds.org/documentation/userguide/blobstore-guide")) 50 | .defaultProperties(H3ApiMetadata.defaultProperties()) 51 | .view(BlobStoreContext.class) 52 | .defaultModule(H3ContextModule.class); 53 | } 54 | 55 | @Override 56 | public H3ApiMetadata build() { 57 | return new H3ApiMetadata(this); 58 | } 59 | 60 | @Override 61 | protected Builder self() { 62 | return this; 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'H3' 21 | copyright = '2019, FORTH-ICS' 22 | author = 'FORTH-ICS' 23 | 24 | 25 | # -- General configuration --------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = ['sphinx.ext.autodoc', 'javasphinx'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # List of patterns, relative to source directory, that match files and 36 | # directories to ignore when looking for source files. 37 | # This pattern also affects html_static_path and html_extra_path. 38 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README.rst', 'venv'] 39 | 40 | 41 | # -- Options for HTML output ------------------------------------------------- 42 | 43 | # The theme to use for HTML and HTML Help pages. See the documentation for 44 | # a list of builtin themes. 45 | # 46 | #html_theme = 'alabaster' 47 | html_theme = 'sphinx_rtd_theme' 48 | 49 | # Add any paths that contain custom static files (such as style sheets) here, 50 | # relative to this directory. They are copied after the builtin static files, 51 | # so a file named "default.css" will overwrite the builtin "default.css". 52 | # html_static_path = ['_static'] 53 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3BucketStats.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * Brief bucket statistics. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3BucketStats { 9 | private long size; 10 | private long nObjects; 11 | private JH3Timespec lastAccess; 12 | private JH3Timespec lastModification; 13 | 14 | /** 15 | * Create a BucketStats object. 16 | * @param size The size of all objects contained in the bucket 17 | * @param nObjects Number of objects contained in the bucket 18 | * @param lastAccess Last time an object was accessed 19 | * @param lastModification Last time an object was modified 20 | */ 21 | public JH3BucketStats(long size, long nObjects, JH3Timespec lastAccess, JH3Timespec lastModification) { 22 | this.size = size; 23 | this.nObjects = nObjects; 24 | this.lastAccess = lastAccess; 25 | this.lastModification = lastModification; 26 | } 27 | 28 | /** 29 | * Get the bucket size. 30 | * @return The bucket size. 31 | */ 32 | public long getSize() { 33 | return size; 34 | } 35 | 36 | /** 37 | * Get the number of objects that are hosted in the bucket. 38 | * @return The number of objects. 39 | */ 40 | public long getNumObjects() { 41 | return nObjects; 42 | } 43 | 44 | /** 45 | * Get the timestamp when an object was last accessed. 46 | * @return the last access timestamp. 47 | */ 48 | public JH3Timespec getLastAccess() { 49 | return lastAccess; 50 | } 51 | 52 | /** 53 | * Get the timestamp when an object was last modified. 54 | * @return the last modified timestamp. 55 | */ 56 | public JH3Timespec getLastModification() { 57 | return lastModification; 58 | } 59 | 60 | @Override 61 | public String toString() { 62 | return "JH3BucketStats{" + 63 | "size=" + size + 64 | ", nObjects=" + nObjects + 65 | ", lastAccess=" + lastAccess + 66 | ", lastModification=" + lastModification + 67 | '}'; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /h3cp/h3wrapper.c: -------------------------------------------------------------------------------- 1 | // Copyright [2020] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "h3wrapper.h" 19 | 20 | #define TRUE 1; 21 | #define FALSE 0; 22 | 23 | int h3lib_write_object_from_file(H3_Handle handle, char *bucketName, char *objectName, char *filename) { 24 | off_t offset = 0; 25 | uint32_t userId = 0; 26 | 27 | H3_Auth auth; 28 | auth.userId = userId; 29 | 30 | struct stat st; 31 | if (stat(filename, &st) == -1) { 32 | return FALSE; 33 | } 34 | size_t size = st.st_size; 35 | int fd = open(filename, O_RDONLY); 36 | if (fd == -1) { 37 | return FALSE; 38 | } 39 | 40 | H3_Status return_value = H3_WriteObjectFromFile(handle, &auth, bucketName, objectName, fd, size, offset); 41 | if (return_value != H3_SUCCESS && return_value != H3_CONTINUE) { 42 | close(fd); 43 | return FALSE; 44 | } 45 | close(fd); 46 | 47 | return TRUE; 48 | } 49 | 50 | int h3lib_read_object_to_file(H3_Handle handle, char *bucketName, char *objectName, char *filename) { 51 | off_t offset = 0; 52 | size_t size = 0; 53 | uint32_t userId = 0; 54 | 55 | H3_Auth auth; 56 | auth.userId = userId; 57 | 58 | int fd = open(filename, O_CREAT|O_WRONLY|O_TRUNC, 0644); 59 | if (fd == -1) { 60 | return FALSE; 61 | } 62 | 63 | H3_Status return_value = H3_ReadObjectToFile(handle, &auth, bucketName, objectName, offset, fd, &size); 64 | if (return_value != H3_SUCCESS && return_value != H3_CONTINUE) { 65 | close(fd); 66 | return FALSE; 67 | } 68 | close(fd); 69 | 70 | return TRUE; 71 | } 72 | -------------------------------------------------------------------------------- /h3lib/cmake/Modules/Findkreonrdma.cmake: -------------------------------------------------------------------------------- 1 | #https://gitlab.kitware.com/cmake/community/-/wikis/doc/tutorials/How-To-Find-Libraries 2 | 3 | # - Try to find Kreon RDMA 4 | # Once done this will define 5 | # KREON_RDMA_FOUND - System has Kreon 6 | # KREON_RDMA_INCLUDE_DIRS - The Kreon include directories 7 | # KREON_RDMA_LIBRARIES - The libraries needed to use Kreon 8 | # KREON_RDMA_DEFINITIONS - Compiler switches required for using Kreon 9 | 10 | 11 | # Use pkg-config to detect include/library paths kreon_client kreon_rdma kreonr 12 | find_package(PkgConfig) 13 | pkg_check_modules(PC_KREON_RDMA QUIET kreonr) 14 | set(KREON_RDMA_DEFINITIONS ${PC_KREON_RDMA_CFLAGS_OTHER}) 15 | 16 | 17 | # Dependencies use plural forms, the package itself uses the singular forms defined by find_path and find_library 18 | find_path(KREON_RDMA_INCLUDE_DIR kreon/kreon_rdma_client.h 19 | HINTS ${PC_KREON_RDMA_INCLUDEDIR} ${PC_KREON_RDMA_INCLUDE_DIRS} 20 | PATH_SUFFIXES include ) 21 | 22 | find_library(KREON_RDMA_LIBRARY_CLIENT kreon_client 23 | HINTS ${PC_KREON_RDMA_LIBDIR} ${PC_KREON_RDMA_LIBRARY_DIRS} ) 24 | 25 | find_library(KREON_RDMA_LIBRARY_RDMA kreon_rdma 26 | HINTS ${PC_KREON_RDMA_LIBDIR} ${PC_KREON_RDMA_LIBRARY_DIRS} ) 27 | 28 | find_library(KREON_RDMA_LIBRARY kreonr 29 | HINTS ${PC_KREON_RDMA_LIBDIR} ${PC_KREON_RDMA_LIBRARY_DIRS} ) 30 | 31 | # Call the find_package_handle_standard_args() macro to set the _FOUND variable and print a success or failure message 32 | include(FindPackageHandleStandardArgs) 33 | 34 | 35 | find_package_handle_standard_args(kreonrdma DEFAULT_MSG 36 | KREON_RDMA_LIBRARY KREON_RDMA_LIBRARY_CLIENT KREON_RDMA_LIBRARY_RDMA KREON_RDMA_INCLUDE_DIR) 37 | 38 | 39 | # https://cmake.org/cmake/help/latest/command/mark_as_advanced.html?highlight=s 40 | # Don't show vars in CMakeGUI unless the "show advanced" option is set 41 | #mark_as_advanced(KREON_RDMA_INCLUDE_DIR KREON_RDMA_LIBRARY ) 42 | 43 | 44 | if(KREON_RDMA_FOUND) 45 | set(KREON_RDMA_LIBRARIES ${KREON_RDMA_LIBRARY} ${KREON_RDMA_LIBRARY_CLIENT} ${KREON_RDMA_LIBRARY_RDMA}) 46 | set(KREON_RDMA_DEFINITIONS ${KREON_RDMA_INCLUDE_DIR}) 47 | message(STATUS "Kreon RDMA found") 48 | else() 49 | message(STATUS "Kreon RDMA not found") 50 | endif() 51 | 52 | -------------------------------------------------------------------------------- /JH3lib/JH3HadoopFS/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 4.0.0 6 | 7 | gr.forth.ics.JH3lib 8 | JH3lib 9 | 0.1-beta 10 | 11 | 12 | JH3HadoopFS 13 | 0.1-beta 14 | JH3HadoopFS 15 | http://maven.apache.org 16 | 17 | 18 | 19 | 20 | maven-assembly-plugin 21 | 3.1.1 22 | 23 | 24 | jar-with-dependencies 25 | 26 | 27 | 28 | 29 | make-assembly 30 | package 31 | 32 | single 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | junit 43 | junit 44 | 3.8.1 45 | test 46 | 47 | 48 | 49 | 50 | org.apache.hadoop 51 | hadoop-common 52 | 3.1.2 53 | 54 | 55 | 56 | 57 | log4j 58 | log4j 59 | 1.2.17 60 | 61 | 62 | 63 | gr.forth.ics.JH3lib 64 | JH3 65 | 0.1-beta 66 | compile 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /h3lib/util.c: -------------------------------------------------------------------------------- 1 | // Copyright [2019] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "util.h" 20 | 21 | // http://web.theurbanpenguin.com/adding-color-to-your-output-from-c/ 22 | static const char* color[] = {"\033[0;33m", // H3_INFO --> Yellow 23 | "\033[0;32m", // H3_DEBUG --> Green 24 | "\033[1;31m", // H3_ERROR --> Red 25 | "" }; // H3_NumberOfLevels 26 | 27 | void _LogActivity(H3_MsgLevel level, const char* function, int lineNumber, const char *format, ...) { 28 | 29 | char buffer[H3_HEADER_SIZE + H3_MSG_SIZE]; 30 | va_list arg; 31 | 32 | // Create header 33 | snprintf(buffer, H3_HEADER_SIZE, "%s%s @ %d - ", color[level], function, lineNumber); 34 | 35 | // Create message 36 | va_start(arg, format); 37 | vsnprintf(&buffer[strlen(buffer)], H3_MSG_SIZE - 10, format, arg); 38 | va_end(arg); 39 | 40 | // Print the message 41 | printf("%s\033[0m", buffer); 42 | } 43 | 44 | // res < 0 A < B 45 | // res = 0 A == B 46 | // res > 0 A > B 47 | int64_t Compare(struct timespec* a, struct timespec* b){ 48 | if(a->tv_sec == b->tv_sec) 49 | return a->tv_nsec - b->tv_nsec; 50 | 51 | return a->tv_sec - b->tv_sec; 52 | } 53 | 54 | struct timespec Posterior(struct timespec* a, struct timespec* b){ 55 | return Compare(a,b)>0?*a:*b; 56 | } 57 | 58 | struct timespec Anterior(struct timespec* a, struct timespec* b){ 59 | return Compare(a,b)<0?*a:*b; 60 | } 61 | 62 | void* ReAllocFreeOnFail(void* buffer, size_t size){ 63 | void* tmp = realloc(buffer, size); 64 | if(!tmp){ 65 | free(buffer); 66 | } 67 | 68 | return tmp; 69 | } 70 | -------------------------------------------------------------------------------- /Jclouds-H3/src/main/java/org/jclouds/h3/util/internal/H3BlobUtilsImpl.java: -------------------------------------------------------------------------------- 1 | package org.jclouds.h3.util.internal; 2 | 3 | import com.google.inject.Inject; 4 | import org.jclouds.blobstore.LocalStorageStrategy; 5 | import org.jclouds.blobstore.domain.BlobBuilder; 6 | import org.jclouds.blobstore.options.ListContainerOptions; 7 | import org.jclouds.blobstore.util.BlobUtils; 8 | import org.jclouds.h3.strategy.internal.H3StorageStrategyImpl; 9 | 10 | import javax.inject.Provider; 11 | 12 | import static com.google.common.base.Preconditions.checkNotNull; 13 | 14 | public class H3BlobUtilsImpl implements BlobUtils { 15 | 16 | protected final H3StorageStrategyImpl storageStrategy; 17 | protected final Provider blobBuilders; 18 | 19 | @Inject 20 | public H3BlobUtilsImpl(LocalStorageStrategy storageStrategy, Provider blobBuilders) { 21 | this.storageStrategy = (H3StorageStrategyImpl) checkNotNull(storageStrategy, "H3 Storage Strategy"); 22 | this.blobBuilders = checkNotNull(blobBuilders, "H3 blobBuilders"); 23 | } 24 | 25 | @Override 26 | public BlobBuilder blobBuilder() { 27 | return blobBuilders.get(); 28 | } 29 | 30 | @Override 31 | public boolean directoryExists(String containerName, String directory) { 32 | // return storageStrategy.directoryExists(containerName, directory); 33 | System.out.println("[Jclouds-H3][directoryExists]Not yet implemented"); 34 | 35 | return false; 36 | } 37 | 38 | @Override 39 | public void createDirectory(String containerName, String directory) { 40 | // storageStrategy.createDirectory(containerName, directory); 41 | System.out.println("[Jclouds-H3][createDirectory]Not yet implemented"); 42 | 43 | } 44 | 45 | @Override 46 | public long countBlobs(String container, ListContainerOptions options) { 47 | // return storageStrategy.countBlobs(container, options); 48 | System.out.println("[Jclouds-H3][createDirectory]Not yet implemented"); 49 | 50 | return 0; 51 | } 52 | 53 | @Override 54 | public void clearContainer(String container, ListContainerOptions options) { 55 | storageStrategy.clearContainer(container, options); 56 | } 57 | 58 | @Override 59 | public void deleteDirectory(String container, String directory) { 60 | System.out.println("[Jclouds-H3][deleteDirectory]Not yet implemented"); 61 | 62 | // storageStrategy.deleteDirectory(container, directory); 63 | } 64 | }; 65 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeBucketStats.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.NativeLong; 4 | import com.sun.jna.Pointer; 5 | import com.sun.jna.Structure; 6 | 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | /** 11 | * JNA representation of H3_BucketStats 12 | * @author Giorgos Kalaentzis 13 | * @version 0.1-beta 14 | */ 15 | 16 | public class NativeBucketStats extends Structure { 17 | 18 | public NativeLong size; 19 | public long nObjects; 20 | public NativeTimespec lastAccess; 21 | public NativeTimespec lastModification; 22 | 23 | public NativeBucketStats() { super(); } 24 | 25 | /** 26 | * Create native bucket stats. 27 | * @param size The size of all objects contained in the bucket 28 | * @param nObjects Number of objects contained in the bucket 29 | * @param lastAccess Last time an object was accessed 30 | * @param lastModification Last time an object was modified 31 | */ 32 | public NativeBucketStats(NativeLong size, long nObjects, 33 | NativeTimespec lastAccess, NativeTimespec lastModification) { 34 | super(); 35 | this.size = size; 36 | this.nObjects = nObjects; 37 | this.lastAccess = lastAccess; 38 | this.lastModification = lastModification; 39 | } 40 | 41 | @Override 42 | protected List getFieldOrder(){ 43 | return Arrays.asList("size", "nObjects", "lastAccess", "lastModification"); 44 | } 45 | 46 | /** 47 | * Create native bucket stats from the real native pointer. 48 | * @param peer Pointer value of the real native pointer 49 | */ 50 | public NativeBucketStats(Pointer peer) { super(peer); } 51 | 52 | @Override 53 | public String toString() { 54 | return "NativeBucketStats{" + 55 | "size=" + size + 56 | ", nObjects=" + nObjects + 57 | ", lastAccess=" + lastAccess + 58 | ", lastModification=" + lastModification + 59 | '}'; 60 | } 61 | 62 | public static abstract class ByReference extends NativeBucketStats implements Structure.ByReference{ 63 | 64 | }; 65 | 66 | public static abstract class ByValue extends NativeBucketStats implements Structure.ByValue{ 67 | 68 | }; 69 | 70 | } 71 | -------------------------------------------------------------------------------- /JH3lib/README.md: -------------------------------------------------------------------------------- 1 | # H3 Java wrapper and tests 2 | 3 | A Java suite containing modules related to h3lib. Project was created using Maven 3.6.0. Java version 9 or higher is required. 4 | 5 | #### Compile all modules 6 | - mvn compile 7 | 8 | #### Test all modules 9 | - mvn test 10 | 11 | #### Package all modules (create .jar) 12 | - mvn package 13 | - packages are created in each module's `target` folder 14 | 15 | #### Install all modules 16 | - mvn install 17 | 18 | *To compile/package/install specific modules use -pl flag and the name of the module(s). 19 | e.g mvn -pl JH3,JH3Interface compile* 20 | 21 | *Testing phase can be skipped by using -DskipTests flag* 22 | 23 | ### JH3Interface: 24 | A Java Interface for h3lib. This project uses [JNA](https://github.com/java-native-access/jna) to provide native access 25 | to h3lib. Navigate into the module's folder to compile/package/install. 26 | 27 | #### Compile module 28 | - mvn compile 29 | 30 | #### Test module 31 | - mvn test 32 | 33 | #### Package module 34 | - mvn package 35 | 36 | #### Build only package with dependencies 37 | - mvn compile assembly:single 38 | 39 | #### Install module 40 | - mvn install 41 | 42 | ### JH3: 43 | Uses JH3Interface to provide a class to make h3lib access more Java-like. 44 | Instructions are same as JH3Interface module. 45 | 46 | ### JH3HadoopFS: 47 | A H3lib plugin for Hadoop FileSystem API. 48 | To specify the back-end storage, set the environment variable H3LIB\_STORAGE\_URI. 49 | 50 | #### How to add JH3HadoopFS to Hadoop: 51 | ```$xslt 52 | cp path/to/JH3lib/JH3HadoopFS/target/ path/to/hadoop/share/common/lib 53 | cd path/to/built-hadoop/etc/hadoop 54 | open core-site.xml and add the following configuration elements: 55 | 56 | fs.h3.impl 57 | gr.forth.ics.JH3lib.JH3HadoopFS 58 | The filesystem for H3lib URIs 59 | 60 | 61 | 62 | fs.AbstractFileSystem.h3.impl 63 | gr.forth.ics.JH3lib.JH3HadoopAbstractFS 64 | The AbstractFileSystem for H3lib 65 | 66 | ``` 67 | 68 | #### How to use JH3HadoopFS in Spark: 69 | ```$xslt 70 | cd path/to/spark/conf 71 | cp spark-env.sh.template spart-env.sh 72 | add to spark-env: 73 | export SPARK_DIST_CLASSPATH=$(/path/to/built-hadoop/bin/hadoop classpath) 74 | ``` 75 | When ``h3://`` is used as the schema part of the URI in Spark, 76 | Hadoop will use the plugin instead of hdfs, local etc. 77 | See spark-examples folder under JH3HadoopFS module. 78 | -------------------------------------------------------------------------------- /Jclouds-H3/README.md: -------------------------------------------------------------------------------- 1 | # JClouds-H3 Storage API Provider 2 | 3 | Jclouds-H3 is a [Jclouds](https://github.com/apache/jclouds) API provider for the [H3](https://github.com/CARV-ICS-FORTH/H3/) object store. 4 | Provides a minimal but functional storage API for jclouds-compatible technologies (i.e. [s3proxy](https://github.com/gaul/s3proxy)). 5 | 6 | Implemented the Jclouds BlobStore interface, utilizing H3's Java wrapper(JH3), 7 | similar to Amazon's S3, providing an efficient execution throughout the stack. 8 | 9 | ## Installation 10 | 11 | To install locally, you first need to have H3 and its wrappers installed: 12 | - Instructions of how to install the shared libraries of H3lib, are [here](https://github.com/CARV-ICS-FORTH/H3/tree/master/h3lib) . 13 | - Instructions of how to install the Java wrapper of H3, are [here](https://github.com/CARV-ICS-FORTH/H3/tree/master/JH3lib) . 14 | 15 | **Note**: use JDK 8. 16 | 17 | Install Module with: 18 | ```bash 19 | mvn install -Drat.skip=true -Dcheckstyle.skip=true 20 | ``` 21 | 22 | - need to skip RAT::check 23 | - need to skip checkstyle::check 24 | 25 | ## Configuration 26 | ```properties 27 | jclouds.provider=h3 28 | jclouds.h3.basedir=file:///tmp/demo_h3via_jclouds 29 | ``` 30 | 31 | ## Usage test 32 | 33 | ```java 34 | // setup where the provider must store the files 35 | Properties properties = new Properties(); 36 | properties.setProperty(H3Constants.PROPERTY_BASEDIR, "file:///tmp/demo_h3via_jclouds"); 37 | // setup the container name used by the provider (like bucket in S3) 38 | String containerName = "testbucket"; 39 | 40 | // get a context with h3 that offers the portable BlobStore api 41 | BlobStoreContext context = ContextBuilder.newBuilder("h3") 42 | .overrides(properties) 43 | .buildView(BlobStoreContext.class); 44 | 45 | // create a container in the default location 46 | BlobStore blobStore = context.getBlobStore(); 47 | blobStore.createContainerInLocation(null, containerName); 48 | 49 | // add blob 50 | BlobBuilder builder = blobStore.blobBuilder("test"); 51 | builder.payload("test data"); 52 | Blob blob = builder.build(); 53 | blobStore.putBlob(containerName, blob); 54 | 55 | // retrieve blob 56 | Blob blobRetrieved = blobStore.getBlob(containerName, "test"); 57 | 58 | // delete blob 59 | blobStore.removeBlob(containerName, "test"); 60 | 61 | //close context 62 | context.close(); 63 | ``` 64 | 65 | ## Known issues 66 | 67 | - Unstable download when downloading parts with multiple connections per file (>100MB). When used with [Cyberduck](https://cyberduck.io), uncheck the option "Segmented downloads with multiple connections per file" in "General" preferences. 68 | -------------------------------------------------------------------------------- /h3controllers/expiresAtController.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import time 16 | import argparse 17 | import struct 18 | 19 | import pyh3lib 20 | 21 | def ExpiresAt(h3): 22 | """ 23 | Delete all the objects that have the ExpiresAt attribute 24 | and the time that is specified in the ExpiresAt has come. 25 | 26 | :param now: the time, now 27 | :type now: float 28 | :returns: nothing 29 | """ 30 | 31 | now = time.clock_gettime(time.CLOCK_REALTIME) 32 | 33 | # list all the buckets 34 | for h3_bucket in h3.list_buckets(): 35 | done = False 36 | offset = 0 37 | 38 | # list all the objects that have the ExpiresAt attribute 39 | while not done: 40 | h3_objects = h3.list_objects_with_metadata(h3_bucket, "ExpiresAt", offset) 41 | 42 | for h3_object in h3_objects: 43 | # the h3_object contains the object's name 44 | expire_at = h3.read_object_metadata(h3_bucket, h3_object, "ExpiresAt") 45 | 46 | if expire_at != b'': 47 | # try to parse 48 | try: 49 | h3_object_remove_timestamp = struct.unpack('d', expire_at) 50 | except struct.error: 51 | continue 52 | 53 | # Check if we must remove the object 54 | if (now >= h3_object_remove_timestamp[0]) : 55 | h3.delete_object(h3_bucket, h3_object) 56 | 57 | done = h3_objects.done 58 | offset = h3_objects.nextOffset 59 | 60 | def main(cmd=None): 61 | parser = argparse.ArgumentParser(description='ExpiresAt Controller') 62 | parser.add_argument('--storage', required=True, help=f'H3 storage URI') 63 | 64 | args = parser.parse_args(cmd) 65 | config_path = args.storage 66 | 67 | if config_path: 68 | h3 = pyh3lib.H3(config_path) 69 | 70 | ExpiresAt(h3) 71 | else: 72 | parser.print_help(sys.stderr) 73 | sys.exit(1) 74 | 75 | if __name__ == '__main__': 76 | main() 77 | -------------------------------------------------------------------------------- /pyh3lib/tests/test_multipart.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import pyh3lib 17 | import random 18 | 19 | MEGABYTE = 1048576 20 | 21 | def test_simple(h3): 22 | """Create, delete an object.""" 23 | 24 | # All empty. 25 | assert h3.list_buckets() == [] 26 | 27 | with pytest.raises(pyh3lib.H3FailureError): 28 | h3.create_multipart('b1', 'm1') 29 | 30 | # Create a bucket. 31 | assert h3.create_bucket('b1') == True 32 | 33 | assert h3.list_multiparts('b1') == [] 34 | 35 | # Create a multipart object. 36 | multipart = h3.create_multipart('b1', 'm1') 37 | 38 | assert h3.list_multiparts('b1') == ['m1'] 39 | 40 | with open('/dev/urandom', 'rb') as f: 41 | data = f.read(3 * MEGABYTE) 42 | 43 | h3.create_object('b1', 'o1', data) 44 | 45 | h3.create_part(multipart, 1, data) 46 | 47 | h3.create_part(multipart, 0, data[:MEGABYTE]) 48 | 49 | h3.create_part(multipart, 2, data) 50 | 51 | h3.create_part_copy('o1', 0, MEGABYTE, multipart, 0) 52 | 53 | parts = h3.list_parts(multipart) 54 | assert len(parts) == 3 55 | for part_number, size in parts: 56 | if part_number == 0: 57 | assert size == MEGABYTE 58 | else: 59 | assert size == (3 * MEGABYTE) 60 | 61 | h3.complete_multipart(multipart) 62 | 63 | with pytest.raises(pyh3lib.H3NotExistsError): 64 | h3.complete_multipart(multipart) 65 | 66 | with pytest.raises(pyh3lib.H3NotExistsError): 67 | h3.abort_multipart(multipart) 68 | 69 | assert 'm1' in h3.list_objects('b1') 70 | 71 | object_info = h3.info_object('b1', 'm1') 72 | assert not object_info.is_bad 73 | assert object_info.size == (7 * MEGABYTE) 74 | assert type(object_info.creation) == float 75 | assert type(object_info.last_access) == float 76 | assert type(object_info.last_modification) == float 77 | assert type(object_info.last_change) == float 78 | 79 | # Delete objects. 80 | h3.delete_object('b1', 'm1') 81 | 82 | h3.delete_object('b1', 'o1') 83 | 84 | assert h3.list_objects('b1') == [] 85 | 86 | assert h3.delete_bucket('b1') == True 87 | 88 | assert h3.list_buckets() == [] 89 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeObjectInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import com.sun.jna.NativeLong; 4 | import com.sun.jna.Pointer; 5 | import com.sun.jna.Structure; 6 | 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | /** 11 | * JNA representation of H3_ObjectInfo. 12 | * @author Giorgos Kalaentzis 13 | * @version 0.1-beta 14 | */ 15 | public class NativeObjectInfo extends Structure { 16 | public byte isBad; 17 | public NativeLong size; 18 | public NativeTimespec creation; 19 | public NativeTimespec lastAccess; 20 | public NativeTimespec lastModification; 21 | public NativeTimespec lastChange; 22 | public int mode; /* mode_t in native h3lib */ 23 | public int uid; /* uid_t in native h3lib */ 24 | public int gid; /* gid_t in native h3lib */ 25 | 26 | 27 | public NativeObjectInfo() { super(); } 28 | 29 | @Override 30 | protected List getFieldOrder() { 31 | return Arrays.asList("isBad", "size", "creation", "lastAccess", 32 | "lastModification", "lastChange", "mode", "uid", "gid"); 33 | } 34 | 35 | /** 36 | * Create native object info. 37 | * @param isBad Data are corrupt 38 | * @param size Object size 39 | * @param creation Creation timestamp 40 | * @param lastAccess Last access timestamp 41 | * @param lastModification Last modification timestamp 42 | * @param lastChange The last time the object's attributes were changed (e.g. permissions) 43 | * @param mode File type and mode (used by h3fuse) 44 | * @param uid User ID (used by h3fuse) 45 | * @param gid Group ID (used by h3fuse) 46 | */ 47 | public NativeObjectInfo(byte isBad, NativeLong size, NativeTimespec creation, NativeTimespec lastAccess, 48 | NativeTimespec lastModification, NativeTimespec lastChange, int mode, int uid, int gid) { 49 | super(); 50 | this.isBad = isBad; 51 | this.size = size; 52 | this.creation = creation; 53 | this.lastAccess = lastAccess; 54 | this.lastModification = lastModification; 55 | this.lastChange = lastChange; 56 | this.mode = mode; 57 | this.uid = uid; 58 | this.gid = gid; 59 | } 60 | 61 | /** 62 | * Create native object info from the real native pointer. 63 | * @param peer Pointer value of the real native pointer 64 | */ 65 | public NativeObjectInfo(Pointer peer) { 66 | super(peer); 67 | } 68 | 69 | public static class ByReference extends NativeObjectInfo implements Structure.ByReference { 70 | 71 | }; 72 | public static class ByValue extends NativeObjectInfo implements Structure.ByValue { 73 | 74 | }; 75 | } -------------------------------------------------------------------------------- /h3controllers/readOnlyAfterController.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import time 16 | import argparse 17 | import struct 18 | 19 | import pyh3lib 20 | 21 | def ReadOnlyAfter(h3): 22 | """ 23 | Set's the permissions to read only in all objects that have the ReadOnlyAfter attribute and the "now" time exceeds 24 | the time that is specified in the ReadOnlyAfter plus the time from the last time that the object has been modified. 25 | 26 | :param now: the time, now 27 | :type now: float 28 | :returns: nothing 29 | """ 30 | 31 | now = time.clock_gettime(time.CLOCK_REALTIME) 32 | 33 | # list all the buckets 34 | for h3_bucket in h3.list_buckets(): 35 | done = False 36 | offset = 0 37 | 38 | # list all the objects that have the ExpiresAt attribute 39 | while not done: 40 | h3_objects = h3.list_objects_with_metadata(h3_bucket, "ReadOnlyAfter", offset) 41 | 42 | for h3_object in h3_objects: 43 | # the h3_object contains the object's name 44 | read_only_after = h3.read_object_metadata(h3_bucket, h3_object, "ReadOnlyAfter") 45 | 46 | if read_only_after != b'': 47 | # try to parse 48 | try: 49 | h3_object_remove_timestamp = struct.unpack('d', read_only_after) 50 | except struct.error: 51 | continue 52 | 53 | h3_object_info = h3.info_object(h3_bucket, h3_object) 54 | 55 | # Check if we must change the permissions of the object to read only 56 | if ((not h3_object_info.read_only) and (now >= (h3_object_info.last_modification + h3_object_remove_timestamp[0]))): 57 | h3.make_object_read_only(h3_bucket, h3_object) 58 | 59 | done = h3_objects.done 60 | offset = h3_objects.nextOffset 61 | 62 | def main(cmd=None): 63 | parser = argparse.ArgumentParser(description='ReadOnlyAfter Controller') 64 | parser.add_argument('--storage', required=True, help=f'H3 storage URI') 65 | 66 | args = parser.parse_args(cmd) 67 | config_path = args.storage 68 | 69 | if config_path: 70 | h3 = pyh3lib.H3(config_path) 71 | 72 | ReadOnlyAfter(h3) 73 | else: 74 | parser.print_help(sys.stderr) 75 | sys.exit(1) 76 | 77 | if __name__ == '__main__': 78 | main() 79 | -------------------------------------------------------------------------------- /JH3lib/JH3Interface/src/main/java/gr/forth/ics/JH3lib/NativeAttribute.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import com.sun.jna.Pointer; 7 | import com.sun.jna.Structure; 8 | import com.sun.jna.Union; 9 | 10 | /** 11 | * JNA Representation of H3_Attribute 12 | * @author Giorgos Kalaentzis 13 | * @version 0.1-beta 14 | */ 15 | public class NativeAttribute extends Structure { 16 | 17 | public int type; 18 | public AttributeUnion unionval; 19 | 20 | @Override 21 | protected List getFieldOrder(){ 22 | return Arrays.asList("type", "unionval"); 23 | } 24 | 25 | public static class AttributeUnion extends Union { 26 | 27 | /** Permissions in octal mode similar to chmod() */ 28 | public int mode; 29 | /** Structure that holds user, group ID */ 30 | public AttributeStruct structval; 31 | 32 | public static class AttributeStruct extends Structure { 33 | /** User ID, adhering to chown() semanics */ 34 | public int uid; 35 | /** Group ID, adhering to chown() semantics */ 36 | public int gid; 37 | 38 | @Override 39 | protected List getFieldOrder(){ 40 | return Arrays.asList("uid", "gid"); 41 | } 42 | public AttributeStruct(){ 43 | super(); 44 | } 45 | 46 | public AttributeStruct(int uid, int gid){ 47 | super(); 48 | this.uid = uid; 49 | this.gid = gid; 50 | } 51 | 52 | public AttributeStruct(Pointer peer) { 53 | super(peer); 54 | } 55 | 56 | public static class ByReference extends AttributeStruct implements Structure.ByReference{}; 57 | 58 | public static class ByValue extends AttributeStruct implements Structure.ByValue {}; 59 | 60 | }; 61 | 62 | public AttributeUnion(){ 63 | super(); 64 | } 65 | 66 | public AttributeUnion(int mode){ 67 | super(); 68 | this.mode = mode; 69 | setType(int.class); 70 | } 71 | 72 | public AttributeUnion(AttributeStruct structval){ 73 | super(); 74 | this.structval = structval; 75 | setType(AttributeStruct.class); 76 | } 77 | 78 | public AttributeUnion(Pointer peer){ 79 | super(peer); 80 | } 81 | 82 | public static class ByReference extends AttributeUnion implements Structure.ByReference{}; 83 | 84 | public static class ByValue extends AttributeUnion implements Structure.ByValue{}; 85 | }; 86 | 87 | public NativeAttribute(){ 88 | super(); 89 | } 90 | 91 | public NativeAttribute(int type, AttributeUnion unionval){ 92 | super(); 93 | this.type = type; 94 | this.unionval = unionval; 95 | } 96 | 97 | public NativeAttribute(Pointer peer){ 98 | super(peer); 99 | } 100 | 101 | public static class ByReference extends NativeAttribute implements Structure.ByReference{}; 102 | 103 | public static class ByValue extends NativeAttribute implements Structure.ByValue{}; 104 | } -------------------------------------------------------------------------------- /h3cp/main.go: -------------------------------------------------------------------------------- 1 | // Copyright [2020] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | // #cgo CFLAGS: -I/usr/local/include 18 | // #cgo LDFLAGS: -L/usr/local/lib -lh3lib 19 | // #include 20 | // #include "h3wrapper.h" 21 | import "C" 22 | import ( 23 | "fmt" 24 | "flag" 25 | "os" 26 | "strings" 27 | "unsafe" 28 | ) 29 | 30 | func parseH3Path(H3Path string) (string, string) { 31 | if strings.HasPrefix(H3Path, "h3://") { 32 | parts := strings.Split(H3Path[5:], "/") 33 | return parts[0], strings.Join(parts[1:], "/") 34 | } 35 | return "", H3Path 36 | } 37 | 38 | func usage() { 39 | fmt.Printf("Usage: %s -s \n", os.Args[0]) 40 | flag.PrintDefaults() 41 | } 42 | 43 | func main() { 44 | storageUri := "" 45 | 46 | flag.Usage = usage 47 | flag.StringVar(&storageUri, "s", "", "H3 storage URI") 48 | flag.Parse() 49 | 50 | args := flag.Args() 51 | if storageUri == "" || len(args) != 2 { 52 | usage() 53 | return 54 | } 55 | 56 | srcBucket, srcPath := parseH3Path(args[0]) 57 | dstBucket, dstPath := parseH3Path(args[1]) 58 | if (srcBucket == "" && dstBucket == "") || (srcBucket != "" && dstBucket != "") { 59 | fmt.Printf("Error: Only one of the arguments must a path in H3 (starting with \"h3://\").\n") 60 | return 61 | } 62 | if strings.HasSuffix(srcPath, "/") || strings.HasSuffix(dstPath, "/") { 63 | fmt.Printf("Error: Please do not end an object with \"/\".\n") 64 | return 65 | } 66 | 67 | fmt.Println("Info: Using storage URI:", storageUri) 68 | storageUriStr := C.CString(storageUri) 69 | defer C.free(unsafe.Pointer(storageUriStr)) 70 | handle := C.H3_Init(storageUriStr) 71 | defer C.H3_Free(handle) 72 | 73 | if srcBucket != "" { 74 | fmt.Println("Info: Read", srcPath, "from", srcBucket, "into", dstPath) 75 | 76 | bucketName := C.CString(srcBucket) 77 | defer C.free(unsafe.Pointer(bucketName)) 78 | objectName := C.CString(srcPath) 79 | defer C.free(unsafe.Pointer(objectName)) 80 | filename := C.CString(dstPath) 81 | defer C.free(unsafe.Pointer(filename)) 82 | 83 | C.h3lib_read_object_to_file(handle, bucketName, objectName, filename) // XXX Check... 84 | } else if dstBucket != "" { 85 | fmt.Println("Info: Write", srcPath, "to", dstPath, "into", dstBucket) 86 | 87 | bucketName := C.CString(dstBucket) 88 | defer C.free(unsafe.Pointer(bucketName)) 89 | objectName := C.CString(dstPath) 90 | defer C.free(unsafe.Pointer(objectName)) 91 | filename := C.CString(srcPath) 92 | defer C.free(unsafe.Pointer(filename)) 93 | 94 | C.h3lib_write_object_from_file(handle, bucketName, objectName, filename) // XXX Check... 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pyh3lib/tests/test_bucket.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import pyh3lib 17 | import random 18 | 19 | def test_simple(h3): 20 | """List, create, delete a bucket.""" 21 | 22 | assert h3.list_buckets() == [] 23 | 24 | assert h3.create_bucket('b1') == True 25 | 26 | with pytest.raises(pyh3lib.H3ExistsError): 27 | h3.create_bucket('b1') 28 | 29 | bucket_info = h3.info_bucket('b1') 30 | assert bucket_info.stats == None 31 | assert type(bucket_info.creation) == float 32 | 33 | bucket_info = h3.info_bucket('b1', get_stats=True) 34 | assert bucket_info.stats != None 35 | assert bucket_info.stats.size == 0 36 | assert bucket_info.stats.count == 0 37 | 38 | assert h3.list_buckets() == ['b1'] 39 | 40 | assert h3.delete_bucket('b1') == True 41 | 42 | with pytest.raises(pyh3lib.H3NotExistsError): 43 | h3.delete_bucket('b1') 44 | 45 | with pytest.raises(pyh3lib.H3NotExistsError): 46 | h3.info_bucket('b1') 47 | 48 | assert h3.list_buckets() == [] 49 | 50 | def test_arguments(h3): 51 | """Pass invalid arguments.""" 52 | 53 | # Empty name 54 | with pytest.raises(pyh3lib.H3InvalidArgsError): 55 | h3.create_bucket('') 56 | 57 | with pytest.raises(TypeError): 58 | h3.create_bucket(None) 59 | 60 | # Large name 61 | with pytest.raises(pyh3lib.H3NameTooLongError): 62 | h3.create_bucket('a' * (h3.BUCKET_NAME_SIZE + 1)) 63 | 64 | # Invalid name 65 | with pytest.raises(pyh3lib.H3InvalidArgsError): 66 | h3.create_bucket('/bucketId') 67 | 68 | h3.create_bucket('\bucketId') 69 | h3.delete_bucket('\bucketId') 70 | 71 | def test_many(h3): 72 | """Manage many buckets.""" 73 | 74 | count = 100 # More than 10 75 | 76 | assert h3.list_buckets() == [] 77 | 78 | for i in range(count): 79 | assert h3.create_bucket('bucket%d' % i) == True 80 | 81 | for i in random.sample(range(count), 10): 82 | with pytest.raises(pyh3lib.H3ExistsError): 83 | h3.create_bucket('bucket%d' % i) 84 | 85 | for i in range(count): 86 | bucket_info = h3.info_bucket('bucket%d' % i) 87 | assert bucket_info.stats == None 88 | assert type(bucket_info.creation) == float 89 | 90 | assert h3.list_buckets() == [('bucket%d' % i) for i in range(count)] 91 | 92 | for i in range(count): 93 | assert h3.delete_bucket('bucket%d' % i) == True 94 | 95 | for i in random.sample(range(count), 10): 96 | with pytest.raises(pyh3lib.H3NotExistsError): 97 | h3.delete_bucket('bucket%d' % i) 98 | 99 | for i in random.sample(range(count), 10): 100 | with pytest.raises(pyh3lib.H3NotExistsError): 101 | h3.info_bucket('bucket%d' % i) 102 | 103 | assert h3.list_buckets() == [] 104 | -------------------------------------------------------------------------------- /h3fuse/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | project(h3fuse VERSION 1.2 LANGUAGES C) 4 | 5 | # Code checking 6 | find_program(CMAKE_C_CPPCHECK NAMES cppcheck) 7 | 8 | # The configuration file will be auto-generated into the binary tree i.e. build directory 9 | # Note that the build directory has to be added as an include-dir 10 | configure_file(h3fuse_config.h.in h3fuse_config.h) 11 | 12 | # https://cmake.org/cmake/help/v3.10/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type 13 | # https://cmake.org/cmake/help/v3.10/variable/CMAKE_CONFIGURATION_TYPES.html#variable:CMAKE_CONFIGURATION_TYPES 14 | set(DEFAULT_BUILD_TYPE "Release") 15 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 16 | message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") 17 | set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE STRING "Choose the type of build." FORCE) 18 | # Set the possible values of build type for cmake-gui 19 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") 20 | endif() 21 | 22 | # Define GNU standard installation directories as variables CMAKE_INSTALL_xxxx 23 | # https://cmake.org/cmake/help/v3.10/module/GNUInstallDirs.html 24 | include(GNUInstallDirs) 25 | 26 | #https://cmake.org/cmake/help/v3.10/variable/CMAKE_C_STANDARD.html?highlight=cmake_c_standard 27 | set(CMAKE_C_STANDARD 11) 28 | 29 | #https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/Useful-Variables 30 | #https://cmake.org/cmake/help/v3.10/command/add_compile_options.html?highlight=add_compile_options 31 | add_compile_options(-Wall -Werror) 32 | SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -O0 -DDEBUG") 33 | SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3") 34 | 35 | #https://cmake.org/cmake/help/v3.10/command/add_library.html 36 | set(SOURCE_FILES h3fuse.c ) 37 | add_executable(${PROJECT_NAME} ${SOURCE_FILES}) 38 | 39 | #https://cmake.org/cmake/help/v3.10/module/FindPkgConfig.html#command:pkg_search_module 40 | find_package(PkgConfig) 41 | pkg_search_module(FUSE REQUIRED fuse3) 42 | pkg_search_module(GLIB REQUIRED glib-2.0) 43 | target_include_directories(${PROJECT_NAME} PRIVATE ${FUSE_INCLUDE_DIRS} ${GLIB_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}) 44 | target_link_libraries(${PROJECT_NAME} PRIVATE ${FUSE_LIBRARIES} ${GLIB_LDFLAGS} h3lib) 45 | SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}") 46 | 47 | message(STATUS "Fuse version: ${FUSE_VERSION}" ) 48 | 49 | install(TARGETS ${PROJECT_NAME}) 50 | 51 | set(CPACK_PACKAGE_NAME ${PROJECT_NAME}) 52 | set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION}) 53 | set(CPACK_PACKAGE_RELEASE 1) 54 | set(CPACK_PACKAGE_VENDOR "FORTH-ICS") 55 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An embedded object store filesystem") 56 | set(CPACK_GENERATOR "RPM") 57 | set(CPACK_RPM_PACKAGE_LICENSE "ASL v2.0") 58 | set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") 59 | set(CPACK_RPM_PACKAGE_REQUIRES "glib2 >= 2.56, fuse3-libs >= 3.6.1, h3lib >= 1.2") 60 | set(CPACK_RPM_PACKAGE_DESCRIPTION "H3 is an embedded High speed, High volume, and High availability object store, backed by a high-performance key-value store (Kreon, RocksDB, etc.) or a filesystem. H3 FUSE provides a filesystem using the h3lib library.") 61 | #set(CPACK_GENERATOR "DEB") 62 | #set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) 63 | #set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Antony Chazapis chazapis@ics.forth.gr") 64 | set(CPACK_PACKAGING_INSTALL_PREFIX /usr) 65 | set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}.${CMAKE_SYSTEM_PROCESSOR}") 66 | include(CPack) 67 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/main/java/gr/forth/ics/JH3lib/JH3ObjectInfo.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | /** 4 | * JH3 Object information. 5 | * @author Giorgos Kalaentzis 6 | * @version 0.1-beta 7 | */ 8 | public class JH3ObjectInfo { 9 | private boolean corrupt; 10 | private long size; 11 | private JH3Timespec creation; 12 | private JH3Timespec lastAccess; 13 | private JH3Timespec lastModification; 14 | private JH3Timespec lastChange; 15 | private int mode; 16 | private int uid; 17 | private int gid; 18 | 19 | 20 | /** 21 | * Create Object information without attribute information 22 | * @param corrupt Data are corrupt 23 | * @param size Object size 24 | * @param creation Creation timestamp 25 | * @param lastAccess Last access timestamp 26 | * @param lastModification Last modification timestamp 27 | */ 28 | // public JH3ObjectInfo(boolean corrupt, long size, JH3Timespec creation, JH3Timespec lastAccess, JH3Timespec lastModification) { 29 | // this.corrupt = corrupt; 30 | // this.size = size; 31 | // this.creation = creation; 32 | // this.lastAccess = lastAccess; 33 | // this.lastModification = lastModification; 34 | // } 35 | 36 | 37 | /** 38 | * Create Object information including attribute information 39 | * @param corrupt Data are corrupt 40 | * @param size Object size 41 | * @param creation Creation timestamp 42 | * @param lastAccess Last access timestamp 43 | * @param lastModification Last modification timestamp 44 | * @param lastChange Last time the object's attributes were changed 45 | * @param mode File type and mode 46 | * @param uid User ID 47 | * @param gid Group ID 48 | */ 49 | public JH3ObjectInfo(boolean corrupt, long size, JH3Timespec creation, JH3Timespec lastAccess, JH3Timespec lastModification, 50 | JH3Timespec lastChange, int mode, int uid, int gid) { 51 | this.corrupt = corrupt; 52 | this.size = size; 53 | this.creation = creation; 54 | this.lastAccess = lastAccess; 55 | this.lastModification = lastModification; 56 | this.lastChange = lastChange; 57 | this.mode = mode; 58 | this.uid = uid; 59 | this.gid = gid; 60 | } 61 | 62 | 63 | /** 64 | * Check if the stored object is corrupt. 65 | * @return the corruption flag. 66 | */ 67 | public boolean isCorrupt() { 68 | return corrupt; 69 | } 70 | 71 | /** 72 | * Get the size of the JH3Object. 73 | * @return the size of the object. 74 | */ 75 | public long getSize() { 76 | return size; 77 | } 78 | 79 | /** 80 | * Get the creation timestamp. 81 | * @return the creation timestamp. 82 | */ 83 | public JH3Timespec getCreation() { 84 | return creation; 85 | } 86 | 87 | /** 88 | * Get the timestamp of when the JH3Object was last accessed. 89 | * @return the last accessed timestamp. 90 | */ 91 | public JH3Timespec getLastAccess() { 92 | return lastAccess; 93 | } 94 | 95 | /** 96 | * Get the timestamp of when the JH3Object was last modified. 97 | * @return the last modified timestamp. 98 | */ 99 | public JH3Timespec getLastModification() { 100 | return lastModification; 101 | } 102 | 103 | /** 104 | * Get the last time the JH3Object's attributes were changed. 105 | */ 106 | public JH3Timespec getLastChange(){ 107 | return lastChange; 108 | } 109 | 110 | /** 111 | * Get the attribute mode of JH3Object. 112 | */ 113 | public int getMode(){ 114 | return mode; 115 | } 116 | 117 | /** 118 | * Get the user ID of the JH3Object. 119 | */ 120 | public int getUid(){ 121 | return uid; 122 | } 123 | 124 | public int getGid(){ 125 | return gid; 126 | } 127 | 128 | 129 | @Override 130 | public String toString() { 131 | return "JH3ObjectInfo{" + 132 | " corrupt='" + corrupt + "'" + 133 | ", size='" + size + "'" + 134 | ", creation='" + creation + "'" + 135 | ", lastAccess='" + lastAccess + "'" + 136 | ", lastModification='" + lastModification + "'" + 137 | ", lastChange='" + lastChange + "'" + 138 | ", mode='" + mode + "'" + 139 | ", uid='" + uid + "'" + 140 | ", gid='" + gid + "'" + 141 | "}"; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /h3lib/cmake/Modules/Findrocksdb.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2014 Stefan.Eilemann@epfl.ch 2 | 3 | # - Try to find the Rocksdb library 4 | # Once done this will define 5 | # 6 | # ROCKSDB_ROOT - Set this variable to the root installation 7 | # 8 | # Read-Only variables: 9 | # ROCKSDB_FOUND - system has the Rocksdb library 10 | # ROCKSDB_INCLUDE_DIR - the Rocksdb include directory 11 | # ROCKSDB_LIBRARIES - The libraries needed to use Rocksdb 12 | # ROCKSDB_VERSION - This is set to $major.$minor.$patch (eg. 0.9.8) 13 | 14 | include(FindPackageHandleStandardArgs) 15 | 16 | if(rocksdb_FIND_REQUIRED) 17 | set(_ROCKSDB_output_type FATAL_ERROR) 18 | set(_ROCKSDB_output 1) 19 | else() 20 | set(_ROCKSDB_output_type STATUS) 21 | if(rocksdb_FIND_QUIETLY) 22 | set(_ROCKSDB_output) 23 | else() 24 | set(_ROCKSDB_output 1) 25 | endif() 26 | endif() 27 | 28 | find_path(_ROCKSDB_INCLUDE_DIR rocksdb/db.h 29 | HINTS ${CMAKE_SOURCE_DIR}/../../.. $ENV{ROCKSDB_ROOT} ${ROCKSDB_ROOT} 30 | PATH_SUFFIXES include 31 | PATHS /usr /usr/local /opt /opt/local) 32 | 33 | if(EXISTS "${_ROCKSDB_INCLUDE_DIR}/rocksdb/version.h") 34 | set(_ROCKSDB_Version_file "${_ROCKSDB_INCLUDE_DIR}/rocksdb/version.h") 35 | elseif(EXISTS "${_ROCKSDB_INCLUDE_DIR}/rocksdb/db.h") 36 | set(_ROCKSDB_Version_file "${_ROCKSDB_INCLUDE_DIR}/rocksdb/db.h") 37 | endif() 38 | if(_ROCKSDB_INCLUDE_DIR AND _ROCKSDB_Version_file) 39 | file(READ ${_ROCKSDB_Version_file} _ROCKSDB_header_contents) 40 | string(REGEX REPLACE ".*kMajorVersion = ([0-9]+).*kMinorVersion = ([0-9]+).*" 41 | "\\1.\\2" _ROCKSDB_VERSION "${_ROCKSDB_header_contents}") 42 | string(REGEX REPLACE ".*ROCKSDB_MAJOR ([0-9]+).*ROCKSDB_MINOR ([0-9]+).*" 43 | "\\1.\\2" _ROCKSDB_VERSION "${_ROCKSDB_header_contents}") 44 | set(ROCKSDB_VERSION ${_ROCKSDB_VERSION} CACHE INTERNAL 45 | "The version of rocksdb which was detected") 46 | else() 47 | set(_ROCKSDB_EPIC_FAIL TRUE) 48 | if(_ROCKSDB_output) 49 | message(${_ROCKSDB_output_type} 50 | "Can't find rocksdb header file rocksdb/db.h.") 51 | endif() 52 | endif() 53 | 54 | # Version checking 55 | if(ROCKSDB_FIND_VERSION AND ROCKSDB_VERSION) 56 | if(ROCKSDB_FIND_VERSION_EXACT) 57 | if(NOT ROCKSDB_VERSION VERSION_EQUAL ${ROCKSDB_FIND_VERSION}) 58 | set(_ROCKSDB_version_not_exact TRUE) 59 | endif() 60 | else() 61 | # version is too low 62 | if(NOT ROCKSDB_VERSION VERSION_EQUAL ${ROCKSDB_FIND_VERSION} AND 63 | NOT ROCKSDB_VERSION VERSION_GREATER ${ROCKSDB_FIND_VERSION}) 64 | set(_ROCKSDB_version_not_high_enough TRUE) 65 | endif() 66 | endif() 67 | endif() 68 | 69 | find_library(ROCKSDB_LIBRARY rocksdb 70 | HINTS ${CMAKE_SOURCE_DIR}/../../.. $ENV{ROCKSDB_ROOT} ${ROCKSDB_ROOT} 71 | PATH_SUFFIXES lib lib64 72 | PATHS /usr /usr/local /opt /opt/local) 73 | 74 | # Inform the users with an error message based on what version they 75 | # have vs. what version was required. 76 | if(NOT ROCKSDB_VERSION) 77 | set(_ROCKSDB_EPIC_FAIL TRUE) 78 | if(_ROCKSDB_output) 79 | message(${_ROCKSDB_output_type} 80 | "Version not found in ${_ROCKSDB_Version_file}.") 81 | endif() 82 | elseif(_ROCKSDB_version_not_high_enough) 83 | set(_ROCKSDB_EPIC_FAIL TRUE) 84 | if(_ROCKSDB_output) 85 | message(${_ROCKSDB_output_type} 86 | "Version ${ROCKSDB_FIND_VERSION} or higher of rocksdb is required. " 87 | "Version ${ROCKSDB_VERSION} was found in ${_ROCKSDB_Version_file}.") 88 | endif() 89 | elseif(_ROCKSDB_version_not_exact) 90 | set(_ROCKSDB_EPIC_FAIL TRUE) 91 | if(_ROCKSDB_output) 92 | message(${_ROCKSDB_output_type} 93 | "Version ${ROCKSDB_FIND_VERSION} of rocksdb is required exactly. " 94 | "Version ${ROCKSDB_VERSION} was found.") 95 | endif() 96 | else() 97 | if(ROCKSDB_FIND_REQUIRED) 98 | if(ROCKSDB_LIBRARY MATCHES "ROCKSDB_LIBRARY-NOTFOUND") 99 | message(FATAL_ERROR "Missing the rocksdb library.\n" 100 | "Consider using CMAKE_PREFIX_PATH or the ROCKSDB_ROOT environment variable. " 101 | "See the ${CMAKE_CURRENT_LIST_FILE} for more details.") 102 | endif() 103 | endif() 104 | find_package_handle_standard_args(rocksdb DEFAULT_MSG 105 | ROCKSDB_LIBRARY _ROCKSDB_INCLUDE_DIR) 106 | endif() 107 | 108 | if(_ROCKSDB_EPIC_FAIL) 109 | # Zero out everything, we didn't meet version requirements 110 | set(ROCKSDB_FOUND FALSE) 111 | set(ROCKSDB_LIBRARY) 112 | set(_ROCKSDB_INCLUDE_DIR) 113 | set(ROCKSDB_INCLUDE_DIRS) 114 | set(ROCKSDB_LIBRARIES) 115 | else() 116 | set(ROCKSDB_INCLUDE_DIRS ${_ROCKSDB_INCLUDE_DIR}) 117 | set(ROCKSDB_LIBRARIES ${ROCKSDB_LIBRARY}) 118 | if(_ROCKSDB_output) 119 | message(STATUS 120 | "Found rocksdb ${ROCKSDB_VERSION} in ${ROCKSDB_INCLUDE_DIRS};${ROCKSDB_LIBRARIES}") 121 | endif() 122 | endif() 123 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # H3 builder image 3 | ################################################################################ 4 | FROM centos:7.7.1908 as h3-builder 5 | 6 | # Install dependencies 7 | RUN yum groupinstall -y "Development Tools" && \ 8 | yum install -y epel-release && \ 9 | yum install -y cmake3 glib2-devel libuuid-devel hiredis-devel cppcheck fuse3 fuse3-devel python3-devel python3-wheel java-8-openjdk-headless maven && \ 10 | yum clean all \ 11 | && rm -rf /var/cache/yum \ 12 | /tmp/* \ 13 | /var/tmp/* \ 14 | /usr/share/man \ 15 | /usr/share/doc \ 16 | /usr/share/doc-base 17 | 18 | # Copy in H3, build, and install 19 | COPY h3lib /root/h3lib/ 20 | COPY h3fuse /root/h3fuse/ 21 | COPY pyh3lib /root/pyh3lib/ 22 | COPY JH3lib /root/JH3lib/ 23 | COPY Jclouds-H3 /root/Jclouds-H3/ 24 | 25 | ARG BUILD_TYPE=Release 26 | 27 | WORKDIR /root/h3lib 28 | RUN mkdir build && \ 29 | (cd build && cmake3 -DCMAKE_INSTALL_PREFIX="/usr" -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" .. && make package && make install) 30 | 31 | WORKDIR /root/h3fuse 32 | RUN mkdir build && \ 33 | (cd build && cmake3 -DCMAKE_INSTALL_PREFIX="/usr" -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" .. && make package && make install) 34 | 35 | WORKDIR /root/pyh3lib 36 | RUN ./setup.py bdist_wheel && \ 37 | pip3 install dist/pyh3lib-1.2-cp36-cp36m-linux_x86_64.whl 38 | 39 | WORKDIR /root/JH3lib 40 | RUN mvn clean install 41 | 42 | WORKDIR /root/Jclouds-H3 43 | RUN mvn clean install -DskipTests=true -Drat.skip=true -Dcheckstyle.skip=true 44 | 45 | WORKDIR /root 46 | 47 | #################################################################################################### 48 | # H3 distribution 49 | #################################################################################################### 50 | FROM centos:7.7.1908 as h3 51 | 52 | COPY --from=h3-builder /root/h3lib/build/h3lib*.rpm /root/ 53 | COPY --from=h3-builder /root/h3fuse/build/h3fuse*.rpm /root/ 54 | COPY --from=h3-builder /root/pyh3lib/dist/pyh3lib*.whl /root/ 55 | WORKDIR /root 56 | RUN yum install -y epel-release && \ 57 | yum install -y python3 fuse3 && \ 58 | yum install -y h3lib*.rpm h3fuse*.rpm && \ 59 | pip3 install pyh3lib*.whl && \ 60 | rm -rf *.rpm *.whl && \ 61 | yum clean all \ 62 | && rm -rf /var/cache/yum \ 63 | /tmp/* \ 64 | /var/tmp/* \ 65 | /usr/share/man \ 66 | /usr/share/doc \ 67 | /usr/share/doc-base 68 | 69 | ################################################################################ 70 | # S3proxy builder image 71 | ################################################################################ 72 | FROM maven:3.5.0-jdk-8-alpine as s3proxy-builder 73 | 74 | COPY --from=h3-builder /root/.m2/ /root/.m2/ 75 | 76 | RUN mkdir /opt && \ 77 | cd /opt && \ 78 | curl -sLO https://github.com/gaul/s3proxy/archive/s3proxy-1.7.1.tar.gz && \ 79 | tar -zxvf s3proxy-1.7.1.tar.gz && \ 80 | mv s3proxy-s3proxy-1.7.1 s3proxy 81 | 82 | COPY Jclouds-H3/config/s3proxy.conf /opt/s3proxy/ 83 | #COPY Jclouds-H3/config/keystore.jks /opt/s3proxy/ 84 | #COPY Jclouds-H3/config/run-docker-container.sh /opt/s3proxy/src/main/resources/ 85 | 86 | WORKDIR /opt/s3proxy 87 | RUN sed -i 's/filesystem/h3/g' pom.xml && \ 88 | mvn package -DskipTests 89 | 90 | #################################################################################################### 91 | # S3proxy with Jclouds-H3 distribution 92 | #################################################################################################### 93 | FROM h3-builder as h3-s3proxy 94 | 95 | WORKDIR /opt/s3proxy 96 | 97 | COPY --from=s3proxy-builder /opt/s3proxy/target/s3proxy /opt/s3proxy/ 98 | COPY Jclouds-H3/config/keystore.jks /opt/s3proxy/ 99 | COPY Jclouds-H3/config/run-docker-container.sh /opt/s3proxy/ 100 | 101 | ENV \ 102 | LOG_LEVEL="info" \ 103 | S3PROXY_AUTHORIZATION="aws-v2-or-v4" \ 104 | S3PROXY_IDENTITY="test:tester" \ 105 | S3PROXY_CREDENTIAL="testing" \ 106 | S3PROXY_CORS_ALLOW_ALL="false" \ 107 | S3PROXY_CORS_ALLOW_ORIGINS="" \ 108 | S3PROXY_CORS_ALLOW_METHODS="" \ 109 | S3PROXY_CORS_ALLOW_HEADERS="" \ 110 | S3PROXY_KEYSTORE_PATH="keystore.jks" \ 111 | S3PROXY_KEYSTORE_PASSWORD="CARVICS" \ 112 | S3PROXY_IGNORE_UNKNOWN_HEADERS="false" \ 113 | JCLOUDS_PROVIDER="h3" \ 114 | S3PROXY_ENDPOINT="https://0.0.0.0:8080" \ 115 | JCLOUDS_REGION="" \ 116 | JCLOUDS_KEYSTONE_VERSION="" \ 117 | JCLOUDS_BASEDIR="file:///data/h3container" \ 118 | JCLOUDS_KEYSTONE_SCOPE="" \ 119 | JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME="" 120 | 121 | EXPOSE 8080 122 | VOLUME /data 123 | 124 | RUN mkdir /data/h3container 125 | 126 | ENTRYPOINT ["/opt/s3proxy/run-docker-container.sh"] 127 | -------------------------------------------------------------------------------- /h3lib/kv_interface.h: -------------------------------------------------------------------------------- 1 | // Copyright [2019] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KV_INTERFACE_H_ 16 | #define KV_INTERFACE_H_ 17 | 18 | #include 19 | 20 | #define KV_LIST_BUFFER_SIZE (256*1024) 21 | 22 | 23 | typedef void* KV_Handle; 24 | typedef char* KV_Key; 25 | typedef unsigned char* KV_Value; 26 | 27 | // Error codes 28 | typedef enum { 29 | KV_FAILURE, KV_KEY_EXIST, KV_KEY_NOT_EXIST, KV_SUCCESS, KV_CONTINUE, KV_KEY_TOO_LONG, KV_INVALID_KEY 30 | }KV_Status; 31 | 32 | typedef struct { 33 | unsigned long totalSpace; 34 | unsigned long freeSpace; 35 | unsigned long usedSpace; 36 | } KV_StorageInfo; 37 | 38 | // Key-value operations 39 | typedef struct KV_Operations { 40 | KV_Handle (*init)(const char* storageUri); 41 | void (*free)(KV_Handle handle); 42 | KV_Status (*storage_info)(KV_Handle handle, KV_StorageInfo* storageInfo); 43 | KV_Status (*validate_key)(KV_Key key); 44 | 45 | /* 46 | * --- Read Operations -- 47 | * For functions metadata_read() and read() argument "size" is in/out, i.e. 48 | * the caller sets it with the chunk size to retrieve and the storage-backend 49 | * sets it to the size it managed to retrieve. 50 | * 51 | * If argument "*value" is NULL then the storage-backend will allocate a data buffer 52 | * big enough to hold the whole value and the caller is expected to release it. 53 | * Otherwise the backend will fill it with up to "size" data. 54 | * 55 | * 56 | * --- Write/Update Operations --- 57 | * Write operations create a key if doesn't exist or update its value otherwise. For 58 | * functions metadata_write() and write(), argument "size" indicates the size of the 59 | * caller supplied value, whereas argument "offset" (for updates) indicates the starting-position 60 | * within the current buffer associated with the key that will be replaced by value. 61 | * If the size of the buffer is smaller than the offset the buffer will be padded 62 | * with 0x00 to make the offset fit. Padding is applied even if the key is just created. 63 | * 64 | * 65 | * --- Create Operations --- 66 | * Creates are identical to Writes but fail if key already exists. 67 | * 68 | * 69 | * --- List Operations --- 70 | * This operation accept a pointer to store the entries and another to store their number. 71 | * The size argument is in/out, i.e. it is set by the caller to indicate the desired number 72 | * of matching entries wishing to retrieve and by the function to indicate the number 73 | * actually retrieved. Setting the number to 0x00 means to retrieve all the objects. 74 | * If the buffer pointer is NULL then we only count the number of matching keys. 75 | * The caller may also indicate the number of entries to be skipped. 76 | * 77 | * 78 | * --- Move/Copy Operations --- 79 | * The destination will be overwritten if exists 80 | * 81 | * 82 | * --- Sync Operation --- 83 | * This may be useful for an external storage device. 84 | */ 85 | 86 | KV_Status (*metadata_read)(KV_Handle handle, KV_Key key, off_t offset, KV_Value* value, size_t* size); 87 | KV_Status (*metadata_write)(KV_Handle handle, KV_Key key, KV_Value value, size_t size); 88 | KV_Status (*metadata_create)(KV_Handle handle, KV_Key key, KV_Value value, size_t size); 89 | KV_Status (*metadata_delete)(KV_Handle handle, KV_Key key); 90 | KV_Status (*metadata_move)(KV_Handle handle, KV_Key srcKey, KV_Key dstKey); 91 | KV_Status (*metadata_exists)(KV_Handle handle, KV_Key key); 92 | 93 | KV_Status (*list)(KV_Handle handle, KV_Key prefix, uint8_t nTrim, KV_Key key, uint32_t offset, uint32_t* nKeys); 94 | KV_Status (*exists)(KV_Handle handle, KV_Key key); 95 | KV_Status (*read)(KV_Handle handle, KV_Key key, off_t offset, KV_Value* value, size_t* size); 96 | KV_Status (*create)(KV_Handle handle, KV_Key key, KV_Value value, size_t size); 97 | KV_Status (*update)(KV_Handle handle, KV_Key key, KV_Value value, off_t offset, size_t size); 98 | KV_Status (*write)(KV_Handle handle, KV_Key key, KV_Value value, size_t size); 99 | KV_Status (*copy)(KV_Handle handle, KV_Key srcKey, KV_Key dstKey); 100 | KV_Status (*move)(KV_Handle handle, KV_Key srcKey, KV_Key dstKey); 101 | KV_Status (*delete)(KV_Handle handle, KV_Key key); 102 | KV_Status (*sync)(KV_Handle handle); 103 | } KV_Operations; 104 | 105 | #endif /* KV_INTERFACE_H_ */ 106 | 107 | 108 | -------------------------------------------------------------------------------- /pyh3lib/tests/test_file.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import pyh3lib 17 | import os 18 | import hashlib 19 | 20 | MEGABYTE = 1048576 21 | 22 | def test_copy_folder(h3): 23 | """Copy in a folder.""" 24 | 25 | cwd = os.getcwd() 26 | print('Working in directory: %s' % cwd) 27 | files = [] 28 | for (dirpath, dirnames, filenames) in os.walk(cwd): 29 | for filename in filenames: 30 | name = os.path.join(dirpath, filename)[len(cwd):].lstrip('/') 31 | with open(name, 'rb') as fp: 32 | md5 = hashlib.md5(fp.read()).hexdigest() 33 | files.append({'name': name, 34 | 'size': os.path.getsize(name), 35 | 'md5': md5}) 36 | 37 | # Create bucket 38 | bucket_name = 'test-copy' 39 | assert bucket_name not in h3.list_buckets() 40 | assert h3.create_bucket(bucket_name) == True 41 | 42 | # Copy in all files in current directory and verify sizes 43 | for f in files: 44 | print('Copying: %s -> h3://%s/%s (%s bytes)' % (f['name'], bucket_name, f['name'], f['size'])) 45 | assert h3.write_object_from_file(bucket_name, f['name'], f['name']) == True 46 | info = h3.info_object(bucket_name, f['name']) 47 | assert info.is_bad == False 48 | assert info.size == f['size'] 49 | 50 | # Get the list of objects 51 | objects = [] 52 | while True: 53 | result = h3.list_objects(bucket_name, offset=len(objects)) 54 | objects += result 55 | if result.done: 56 | break 57 | 58 | assert len(objects) == len(files) 59 | 60 | # Verify sizes again 61 | for f in files: 62 | print('Verifying size: h3://%s/%s' % (bucket_name, f['name'])) 63 | info = h3.info_object(bucket_name, f['name']) 64 | assert info.is_bad == False 65 | assert info.size == f['size'] 66 | 67 | # Verify data and rewrite 68 | copy_folder = 'copy' 69 | for f in files: 70 | print('Verifying: h3://%s/%s' % (bucket_name, f['name'])) 71 | 72 | data = b'' 73 | while True: 74 | result = h3.read_object(bucket_name, f['name'], offset=len(data)) 75 | print('Got %d bytes' % len(result)) 76 | data += result 77 | if result.done: 78 | break 79 | 80 | md5 = hashlib.md5(data).hexdigest() 81 | assert md5 == f['md5'] 82 | 83 | print('Writing: h3://%s/%s/%s' % (bucket_name, copy_folder, f['name'])) 84 | h3.write_object(bucket_name, copy_folder + '/' + f['name'], data) 85 | 86 | # Verify copy 87 | for f in files: 88 | print('Verifying: h3://%s/%s/%s' % (bucket_name, copy_folder, f['name'])) 89 | info = h3.info_object(bucket_name, copy_folder + '/' + f['name']) 90 | assert info.is_bad == False 91 | assert info.size == f['size'] 92 | 93 | data = b'' 94 | while True: 95 | result = h3.read_object(bucket_name, copy_folder + '/' + f['name'], offset=len(data)) 96 | print('Got %d bytes' % len(result)) 97 | data += result 98 | if result.done: 99 | break 100 | 101 | md5 = hashlib.md5(data).hexdigest() 102 | assert md5 == f['md5'] 103 | 104 | # Empty and delete bucket 105 | assert h3.purge_bucket(bucket_name) == True 106 | assert h3.delete_bucket(bucket_name) == True 107 | 108 | def test_large_file(h3): 109 | """Copy in a folder.""" 110 | 111 | # Create bucket 112 | bucket_name = 'test-large' 113 | assert bucket_name not in h3.list_buckets() 114 | assert h3.create_bucket(bucket_name) == True 115 | 116 | # Write and read object 117 | with open('/dev/urandom', 'rb') as f: 118 | object_data = f.read(30 * MEGABYTE) 119 | 120 | h3.write_object(bucket_name, 'largefile', object_data) 121 | 122 | object_info = h3.info_object(bucket_name, 'largefile') 123 | assert not object_info.is_bad 124 | assert object_info.size == (30 * MEGABYTE) 125 | assert type(object_info.creation) == float 126 | assert type(object_info.last_access) == float 127 | assert type(object_info.last_modification) == float 128 | assert type(object_info.last_change) == float 129 | 130 | data = b'' 131 | while True: 132 | result = h3.read_object(bucket_name, 'largefile', offset=len(data)) 133 | data += result 134 | if result.done: 135 | break 136 | 137 | assert data == object_data 138 | 139 | object_info = h3.info_object(bucket_name, 'largefile') 140 | assert not object_info.is_bad 141 | assert object_info.size == (30 * MEGABYTE) 142 | assert type(object_info.creation) == float 143 | assert type(object_info.last_access) == float 144 | assert type(object_info.last_modification) == float 145 | assert type(object_info.last_change) == float 146 | 147 | assert h3.list_objects(bucket_name) == ['largefile'] 148 | 149 | # Empty and delete bucket 150 | assert h3.purge_bucket(bucket_name) == True 151 | assert h3.delete_bucket(bucket_name) == True 152 | -------------------------------------------------------------------------------- /pyh3lib/tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import pyh3lib 17 | import struct 18 | 19 | def test_metadata_arguments(h3): 20 | """Test metadata exceptions""" 21 | 22 | assert h3.list_buckets() == [] 23 | 24 | assert h3.create_bucket('b1') 25 | 26 | h3.create_object('b1', 'o1', b'') 27 | 28 | assert h3.create_object_metadata('b1', 'o1', 'ExpireAt', b'') 29 | 30 | # the metadata name is not string 31 | with pytest.raises(TypeError): 32 | h3.create_object_metadata('b1', 'o1', None , b'') 33 | 34 | # big metadata name 35 | with pytest.raises(pyh3lib.H3NameTooLongError): 36 | h3.create_object_metadata('b1', 'o1', 'm' * (h3.METADATA_NAME_SIZE + 3) , b'') 37 | 38 | # the metadata name is empty 39 | with pytest.raises(pyh3lib.H3InvalidArgsError): 40 | h3.create_object_metadata('b1', 'o1', '' , b'') 41 | 42 | # the metadata name starts with '#' 43 | with pytest.raises(pyh3lib.H3InvalidArgsError): 44 | h3.create_object_metadata('b1', 'o1', '#ExpireAt', b'') 45 | 46 | assert h3.purge_bucket('b1') 47 | 48 | assert h3.delete_bucket('b1') 49 | 50 | def test_metadata_error(h3): 51 | """Test metadata exceptions""" 52 | 53 | assert h3.list_buckets() == [] 54 | 55 | assert h3.create_bucket('b1') 56 | 57 | h3.create_object('b1', 'o1', b'') 58 | 59 | assert h3.create_object_metadata('b1', 'o1', 'ExpireAt', b'') 60 | 61 | with pytest.raises(pyh3lib.H3NotExistsError): 62 | h3.create_object_metadata('b1', 'o2', 'ExpireAt', b'') 63 | 64 | with pytest.raises(pyh3lib.H3NotExistsError): 65 | h3.delete_object_metadata('b1', 'o1', 'ReadOnlyAfter') 66 | 67 | with pytest.raises(pyh3lib.H3NotExistsError): 68 | h3.read_object_metadata('b1', 'o1', 'ReadOnlyAfter') 69 | 70 | with pytest.raises(pyh3lib.H3NotExistsError): 71 | h3.move_object_metadata('b1', 'o1', 'o2') 72 | 73 | with pytest.raises(pyh3lib.H3NotExistsError): 74 | h3.copy_object_metadata('b1', 'o1', 'o2') 75 | 76 | assert h3.purge_bucket('b1') 77 | 78 | assert h3.delete_bucket('b1') 79 | 80 | def test_metadata(h3): 81 | """Create, search and delete metadata to an empty object.""" 82 | 83 | assert h3.list_buckets() == [] 84 | 85 | assert h3.create_bucket('b1') 86 | 87 | h3.create_object('b1', 'o1', b'') 88 | 89 | h3.create_object('b1', 'o2', b'') 90 | 91 | h3.create_object('b1', 'o3', b'') 92 | 93 | assert h3.create_object_metadata('b1', 'o1', 'ExpireAt', b'') 94 | assert h3.create_object_metadata('b1', 'o2', 'ExpireAt', b'') 95 | assert h3.create_object_metadata('b1', 'o3', 'ExpireAt', b'') 96 | 97 | assert h3.create_object_metadata('b1', 'o3', 'Content-Type', 'XML'.encode('utf-8')) 98 | 99 | assert h3.read_object_metadata('b1', 'o1', 'ExpireAt') == b'' 100 | 101 | assert h3.read_object_metadata('b1', 'o3', 'Content-Type').decode('utf-8') == 'XML' 102 | 103 | assert h3.create_object_metadata('b1', 'o1', 'ReadOnlyAfter', struct.pack('i', 258)) 104 | 105 | assert h3.create_object_metadata('b1', 'o1', 'ExpireAt', struct.pack('i', 259)) 106 | 107 | assert struct.unpack('i', h3.read_object_metadata('b1', 'o1', 'ReadOnlyAfter'))[0] == 258 108 | 109 | assert struct.unpack('i', h3.read_object_metadata('b1', 'o1', 'ExpireAt'))[0] == 259 110 | 111 | assert set(h3.list_objects_with_metadata('b1', 'ExpireAt')) == set(['o1','o2','o3']) 112 | 113 | assert h3.copy_object('b1', 'o1', 'o2') 114 | 115 | assert struct.unpack('i', h3.read_object_metadata('b1', 'o2', 'ExpireAt'))[0] == 259 116 | 117 | assert set(h3.list_objects_with_metadata('b1', 'ExpireAt')) == set(['o1','o2','o3']) 118 | 119 | assert h3.delete_object_metadata('b1', 'o1', 'ExpireAt') 120 | 121 | assert set(h3.list_objects_with_metadata('b1', 'ExpireAt')) == set(['o2','o3']) 122 | 123 | assert h3.move_object('b1', 'o1', 'o2') 124 | 125 | assert h3.move_object('b1', 'o2', 'o4') 126 | 127 | assert h3.truncate_object('b1', 'o4') 128 | 129 | object_info = h3.info_object('b1', 'o4') 130 | assert object_info.size == 0 131 | 132 | assert h3.list_objects_with_metadata('b1', 'ReadOnlyAfter') == ['o4'] 133 | assert h3.list_objects_with_metadata('b1', 'ExpireAt') == ['o3'] 134 | 135 | assert h3.delete_object('b1', 'o4') 136 | 137 | assert h3.list_objects_with_metadata('b1', 'ReadOnlyAfter') == [] 138 | 139 | assert h3.purge_bucket('b1') 140 | 141 | assert h3.list_objects_with_metadata('b1', 'ExpireAt') == [] 142 | 143 | assert h3.delete_bucket('b1') 144 | 145 | def test_list_metadata(h3): 146 | """List object metadata.""" 147 | 148 | assert h3.list_buckets() == [] 149 | 150 | assert h3.create_bucket('b1') 151 | 152 | for i in range(1, 1500): 153 | h3.create_object('b1', f'object_with_very_very_large_name_to_test_metadata_{i}', b'') 154 | h3.create_object_metadata('b1', f'object_with_very_very_large_name_to_test_metadata_{i}', 'metadata', b'') 155 | 156 | done = False 157 | offset = 0 158 | totalObjects = 0 159 | 160 | while not done: 161 | objects = h3.list_objects_with_metadata('b1', 'metadata', offset) 162 | offset = objects.nextOffset 163 | done = objects.done 164 | 165 | totalObjects += len(objects) 166 | 167 | assert totalObjects == 1499 168 | 169 | h3.purge_bucket('b1') 170 | 171 | assert h3.delete_bucket('b1') -------------------------------------------------------------------------------- /h3lib/common.h: -------------------------------------------------------------------------------- 1 | // Copyright [2019] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #define _GNU_SOURCE 16 | #include 17 | #include 18 | #include 19 | //#include 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | 26 | #include "h3lib.h" 27 | #include "h3lib_config.h" 28 | #include "kv_interface.h" 29 | 30 | 31 | #ifndef TRUE 32 | #define TRUE 1 33 | #endif 34 | 35 | #ifndef FALSE 36 | #define FALSE !TRUE 37 | #endif 38 | 39 | #ifndef UUID_STR_LEN 40 | #define UUID_STR_LEN 36 41 | #endif 42 | 43 | #ifndef REG_NOERROR 44 | #define REG_NOERROR 0 45 | #endif 46 | 47 | #define H3_PART_SIZE (1048576 * 1) // = 2Mb - Key - 4Kb kreon metadata 48 | #define H3_CHUNK (H3_PART_SIZE * 16) 49 | #define H3_SYSTEM_ID 0x00 50 | 51 | #define H3_BUCKET_BATCH_SIZE 10 52 | #define H3_PART_BATCH_SIZE 10 53 | 54 | #define H3_USERID_SIZE 128 55 | #define H3_MULIPARTID_SIZE (UUID_STR_LEN + 1) 56 | 57 | 58 | typedef char H3_UserId[H3_USERID_SIZE+1]; 59 | typedef char H3_BucketId[H3_BUCKET_NAME_SIZE+2]; 60 | typedef char H3_ObjectId[H3_BUCKET_NAME_SIZE + H3_OBJECT_NAME_SIZE + 1]; 61 | typedef char H3_UUID[UUID_STR_LEN]; 62 | typedef char H3_PartId[50]; // '_' + UUID[36+1byte] + '#' + + ['.' + ] 63 | typedef char H3_ObjectMetadataId[H3_BUCKET_NAME_SIZE + H3_OBJECT_NAME_SIZE + H3_METADATA_NAME_SIZE + 2]; // bucket_name + '#' + object_name + '#' + metadata_name 64 | 65 | typedef enum { 66 | H3_STORE_FILESYSTEM = 0, // Mounted filesystem 67 | H3_STORE_KREON, // Kreon local 68 | H3_STORE_KREON_RDMA, // Kreon distributed with RDMA 69 | H3_STORE_ROCKSDB, // RocksDB server 70 | H3_STORE_REDIS, // Redis 71 | H3_NumOfStores // Not an option, used for iteration purposes 72 | } H3_StoreType; 73 | 74 | typedef enum { 75 | MoveReplace, // Overwrite destination if exists 76 | MoveNoReplace, // Do not overwrite destination if exists 77 | MoveExchange // Swap data with destination (must exist) 78 | }H3_MovePolicy; 79 | 80 | typedef struct { 81 | H3_StoreType type; 82 | 83 | // Store specific 84 | KV_Handle handle; 85 | KV_Operations* operation; 86 | }H3_Context; 87 | 88 | typedef struct{ 89 | uint nBuckets; 90 | H3_BucketId bucket[]; 91 | }H3_UserMetadata; 92 | 93 | typedef struct{ 94 | H3_UserId userId; 95 | struct timespec creation; 96 | }H3_BucketMetadata; 97 | 98 | typedef struct{ 99 | uint number; 100 | int subNumber; 101 | size_t size; 102 | off_t offset; // For multipart uploads, the offset is set when the upload completes 103 | }H3_PartMetadata; 104 | 105 | typedef struct{ 106 | char isBad; 107 | H3_UserId userId; 108 | uuid_t uuid; 109 | struct timespec creation; 110 | struct timespec lastAccess; // Access - the last time the file was read 111 | struct timespec lastModification; // Modify - the last time the file was modified (content has been modified) 112 | struct timespec lastChange; // Change - the last time meta data of the file was changed (e.g. permissions) 113 | char readOnly; // The object is read only (used by the h3controllers) 114 | mode_t mode; 115 | uid_t uid; 116 | gid_t gid; 117 | uint nParts; 118 | H3_PartMetadata part[]; 119 | }H3_ObjectMetadata; 120 | 121 | typedef struct{ 122 | H3_UserId userId; 123 | H3_ObjectId objectId; 124 | }H3_MultipartMetadata; 125 | 126 | 127 | H3_Status ValidBucketName(KV_Operations* op,char* name); 128 | H3_Status ValidObjectName(KV_Operations* op,char* name); 129 | H3_Status ValidMetadataName(KV_Operations* op,char* name); 130 | H3_Status ValidPrefix(KV_Operations* op,char* name); 131 | int GetUserId(H3_Token token, H3_UserId id); 132 | int GetBucketId(H3_Name bucketName, H3_BucketId id); 133 | int GetBucketIndex(H3_UserMetadata* userMetadata, H3_Name bucketName); 134 | void GetObjectId(H3_Name bucketName, H3_Name objectName, H3_ObjectId id); 135 | void GetMultipartObjectId(H3_Name bucketName, H3_Name objectName, H3_ObjectId id); 136 | void GetObjectMetadataId(H3_ObjectMetadataId metadataId, H3_Name bucketName, H3_Name objectName, H3_Name metadataName); 137 | char* GetBucketFromId(H3_ObjectId objId, H3_BucketId bucketId); 138 | void GetBucketAndObjectFromId(H3_Name* bucketName, H3_Name* objectName, H3_ObjectId id); 139 | void InitMode(H3_ObjectMetadata* objMeta); 140 | H3_MultipartId GenerateMultipartId(uuid_t uuid); 141 | H3_Name GenerateDummyObjectName(); 142 | void CreatePartId(H3_PartId partId, uuid_t uuid, int partNumber, int subPartNumber); 143 | char* PartToId(H3_PartId partId, uuid_t uuid, H3_PartMetadata* part); 144 | int GrantBucketAccess(H3_UserId id, H3_BucketMetadata* meta); 145 | int GrantObjectAccess(H3_UserId id, H3_ObjectMetadata* meta); 146 | int GrantMultipartAccess(H3_UserId id, H3_MultipartMetadata* meta); 147 | char* ConvertToOdrinary(H3_ObjectId id); 148 | H3_Status DeleteObject(H3_Context* ctx, H3_UserId userId, H3_ObjectId objId, char truncate); 149 | KV_Status WriteData(H3_Context* ctx, H3_ObjectMetadata* meta, KV_Value value, size_t size, off_t offset); 150 | KV_Status ReadData(H3_Context* ctx, H3_ObjectMetadata* meta, KV_Value value, size_t* size, off_t offset); 151 | KV_Status CopyData(H3_Context* ctx, H3_UserId userId, H3_ObjectId srcObjId, H3_ObjectId dstObjId, off_t srcOffset, size_t* size, uint8_t noOverwrite, off_t dstOffset); 152 | H3_Status PurgeObjectMetadata(H3_Context* ctx, H3_UserId userId, H3_Name bucketName, H3_Name objectName); 153 | H3_Status CopyOrMoveObjectMetadata(H3_Context* ctx, H3_UserId userId, H3_Name bucketName, H3_Name srcObjectName, H3_Name dstObjectName, char move); -------------------------------------------------------------------------------- /h3lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") 3 | 4 | project(h3lib VERSION 1.2 LANGUAGES C) 5 | 6 | # Code checking 7 | find_program(CMAKE_C_CPPCHECK NAMES cppcheck) 8 | # if (CMAKE_C_CPPCHECK) 9 | # list( 10 | # APPEND CMAKE_C_CPPCHECK 11 | # "--enable=all" 12 | # ) 13 | # endif() 14 | 15 | # The configuration file will be auto-generated into the binary tree i.e. build directory 16 | configure_file(h3lib_config.h.in h3lib_config.h) 17 | 18 | 19 | # https://cmake.org/cmake/help/v3.10/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type 20 | # https://cmake.org/cmake/help/v3.10/variable/CMAKE_CONFIGURATION_TYPES.html#variable:CMAKE_CONFIGURATION_TYPES 21 | set(DEFAULT_BUILD_TYPE "Release") 22 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 23 | message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") 24 | set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE STRING "Choose the type of build." FORCE) 25 | # Set the possible values of build type for cmake-gui 26 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") 27 | endif() 28 | 29 | # Define GNU standard installation directories as variables CMAKE_INSTALL_xxxx 30 | # https://cmake.org/cmake/help/v3.10/module/GNUInstallDirs.html 31 | include(GNUInstallDirs) 32 | 33 | #https://cmake.org/cmake/help/v3.10/variable/CMAKE_C_STANDARD.html?highlight=cmake_c_standard 34 | set(CMAKE_C_STANDARD 11) 35 | 36 | #https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/Useful-Variables 37 | #https://cmake.org/cmake/help/v3.10/command/add_compile_options.html?highlight=add_compile_options 38 | #add_compile_options(-Wall -Wno-uninitialized -Wno-unused-result -Werror) 39 | #add_compile_options(-Wall -Wno-uninitialized -Wno-unused-result -Wl,--no-as-needed) <-- libs are considered irrespective of the order they are presented to the compiler/linker 40 | add_compile_options(-Wall -Wno-uninitialized -Wno-unused-result) 41 | 42 | SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -O0 -DDEBUG") 43 | SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3") 44 | 45 | 46 | find_package(PkgConfig REQUIRED) 47 | pkg_search_module(GLIB REQUIRED glib-2.0) 48 | 49 | find_package(rocksdb) 50 | find_package(kreon) 51 | find_package(kreonrdma) 52 | find_package(hiredis) 53 | 54 | #https://cmake.org/cmake/help/v3.10/command/add_library.html 55 | set(SOURCE_FILES h3lib.c bucket.c object.c multipart.c kv_fs.c util.c url_parser.c) 56 | if(ROCKSDB_FOUND) 57 | set(SOURCE_FILES ${SOURCE_FILES} kv_rocksdb.c) 58 | add_definitions(-DH3LIB_USE_ROCKSDB) 59 | endif() 60 | 61 | if(KREON_FOUND) 62 | set(SOURCE_FILES ${SOURCE_FILES} kv_kreon.c) 63 | add_definitions(-DH3LIB_USE_KREON) 64 | endif() 65 | 66 | if(KREON_RDMA_FOUND) 67 | set(SOURCE_FILES ${SOURCE_FILES} kv_kreon_rdma.c) 68 | add_definitions(-DH3LIB_USE_KREON_RDMA) 69 | endif() 70 | 71 | if(HIREDIS_FOUND) 72 | set(SOURCE_FILES ${SOURCE_FILES} kv_redis.c) 73 | add_definitions(-DH3LIB_USE_REDIS) 74 | endif() 75 | 76 | add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES}) 77 | 78 | #https://cmake.org/cmake/help/v3.10/command/target_include_directories.html 79 | target_include_directories( ${PROJECT_NAME} PUBLIC "${PROJECT_BINARY_DIR}" "${GLIB_HEADERS}") 80 | target_include_directories(${PROJECT_NAME} PRIVATE ${GLIB_INCLUDE_DIRS}) 81 | target_link_libraries(${PROJECT_NAME} PRIVATE ${GLIB_LDFLAGS} uuid) 82 | 83 | if(ROCKSDB_FOUND) 84 | target_include_directories(${PROJECT_NAME} PRIVATE ${ROCKSDB_INCLUDE_DIR}) 85 | target_link_libraries(${PROJECT_NAME} PRIVATE ${ROCKSDB_LIBRARIES} dl) 86 | endif() 87 | 88 | if(KREON_FOUND) 89 | target_include_directories(${PROJECT_NAME} PRIVATE ${KREON_INCLUDE_DIR}) 90 | target_link_libraries(${PROJECT_NAME} PRIVATE ${KREON_LIBRARIES} log) 91 | endif() 92 | 93 | if(KREON_RDMA_FOUND) 94 | target_include_directories(${PROJECT_NAME} PRIVATE ${KREON_RDMA_INCLUDE_DIR}) 95 | target_link_libraries(${PROJECT_NAME} PRIVATE ${KREON_RDMA_LIBRARIES} zookeeper_mt rdmacm ibverbs log) 96 | endif() 97 | 98 | if(HIREDIS_FOUND) 99 | target_include_directories(${PROJECT_NAME} PRIVATE ${HIREDIS_INCLUDE_DIR}) 100 | target_link_libraries(${PROJECT_NAME} PRIVATE ${HIREDIS_LIBRARIES}) 101 | endif() 102 | 103 | if(H3LIB_USE_COMPRESSION) 104 | find_library(ZSTD_LIBRARY zstd REQUIRED) 105 | message(STATUS "Zstandard found") 106 | add_definitions(-DH3LIB_USE_COMPRESSION) 107 | target_link_libraries(${PROJECT_NAME} PRIVATE ${ZSTD_LIBRARY}) 108 | endif() 109 | 110 | #target_include_directories( ${PROJECT_NAME} PUBLIC 111 | # $ 112 | # $ 113 | # PRIVATE src) 114 | # 115 | 116 | # https://cmake.org/cmake/help/v3.10/command/set_target_properties.html 117 | # https://cmake.org/cmake/help/v3.10/prop_tgt/SOVERSION.html 118 | set_target_properties( ${PROJECT_NAME} PROPERTIES 119 | VERSION ${PROJECT_VERSION} 120 | SOVERSION 1) 121 | 122 | # Use "sudo make install" to apply. 123 | # https://cmake.org/cmake/help/v3.10/command/install.html 124 | install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) 125 | install(FILES h3lib.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME} ) 126 | 127 | 128 | set(CPACK_PACKAGE_NAME ${PROJECT_NAME}) 129 | set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION}) 130 | set(CPACK_PACKAGE_RELEASE 1) 131 | set(CPACK_PACKAGE_VENDOR "FORTH-ICS") 132 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An embedded object store") 133 | set(CPACK_GENERATOR "RPM") 134 | set(CPACK_RPM_PACKAGE_LICENSE "ASL v2.0") 135 | set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") 136 | set(CPACK_RPM_PACKAGE_REQUIRES "glib2 >= 2.56, libuuid >= 2.23") 137 | set(CPACK_RPM_PACKAGE_DESCRIPTION "H3 is an embedded High speed, High volume, and High availability object store, backed by a high-performance key-value store (Kreon, RocksDB, etc.) or a filesystem. H3 is implemented in the h3lib library, which provides a cloud-friendly API, similar to Amazon's S3. Python and Java wrappers are also available.") 138 | #set(CPACK_GENERATOR "DEB") 139 | #set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) 140 | #set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Antony Chazapis chazapis@ics.forth.gr") 141 | set(CPACK_PACKAGING_INSTALL_PREFIX /usr) 142 | set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}.${CMAKE_SYSTEM_PROCESSOR}") 143 | include(CPack) 144 | -------------------------------------------------------------------------------- /Jclouds-H3/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 22 | 4.0.0 23 | 24 | org.apache.jclouds 25 | jclouds-project 26 | 2.2.1 27 | 28 | org.apache.jclouds.api 29 | h3 30 | jclouds h3 core 31 | jclouds components to access h3 32 | bundle 33 | 2.2.1 34 | 35 | org.jclouds.h3*;version="${project.version}" 36 | org.jclouds*;version="${project.version}",* 37 | 1.7 38 | 1.7 39 | 7 40 | 1.7 41 | false 42 | true 43 | true 44 | 45 | 46 | 47 | 48 | 49 | gr.forth.ics.JH3lib 50 | JH3 51 | 0.1-beta 52 | jar 53 | 54 | 55 | org.apache.jclouds 56 | jclouds-blobstore 57 | ${project.version} 58 | jar 59 | 60 | 61 | org.apache.jclouds.driver 62 | jclouds-slf4j 63 | ${project.version} 64 | 65 | 66 | com.google.auto.service 67 | auto-service 68 | true 69 | 70 | 71 | org.apache.maven 72 | maven-toolchain 73 | 2.2.1 74 | 75 | 76 | 77 | 78 | 79 | 80 | maven-assembly-plugin 81 | 3.1.1 82 | 83 | 84 | jar-with-dependencies 85 | 86 | 87 | 88 | 89 | package 90 | 91 | single 92 | 93 | 94 | 95 | 96 | 97 | org.codehaus.mojo 98 | animal-sniffer-maven-plugin 99 | 1.16 100 | 101 | 102 | package 103 | 104 | check 105 | 106 | 107 | 108 | 109 | true 110 | 111 | org.codehaus.mojo.signature 112 | java17 113 | 1.0 114 | 115 | ${skip.checksignature} 116 | 117 | 118 | 119 | org.apache.maven.plugins 120 | maven-compiler-plugin 121 | 3.8.1 122 | 123 | true 124 | true 125 | /usr/lib/jvm/java-7-openjdk/bin/javac 126 | true 127 | ${maven.compile.source} 128 | ${maven.compile.target} 129 | 130 | 131 | 132 | org.basepom.maven 133 | duplicate-finder-maven-plugin 134 | 1.1.2 135 | 136 | 137 | default 138 | verify 139 | 140 | check 141 | 142 | 143 | 144 | 145 | ${skip.checkduplicate} 146 | 147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/test/java/gr/forth/ics/JH3lib/JH3MultipartTest.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | 6 | import java.io.File; 7 | import java.nio.file.Files; 8 | import java.util.ArrayList; 9 | import java.util.Arrays; 10 | import java.util.Random; 11 | 12 | import static org.junit.Assert.*; 13 | 14 | public class JH3MultipartTest { 15 | 16 | // Initialize storage URI; if H3LIB_STORAGE_URI is not set, use default uri (local filesystem: /tmp/h3) 17 | private static String storageURI = null; 18 | static { 19 | storageURI = System.getenv("H3LIB_STORAGE_URI"); 20 | if(storageURI == null){ 21 | storageURI = "file:///tmp/h3"; 22 | } 23 | } 24 | 25 | private int userId = 0; // Dummy userId 26 | private int MEGABYTE = 1048576; // Size of a megabyte in bytes 27 | 28 | 29 | /** 30 | * Cleanup backend before each test. 31 | */ 32 | @Before 33 | public void cleanup(){ 34 | try{ 35 | JH3 client = new JH3(storageURI, userId); 36 | ArrayList buckets = client.listBuckets(); 37 | for (String bucket : buckets){ 38 | ArrayList objects = client.listObjects(bucket, 0); 39 | for (String object : objects){ 40 | client.deleteObject(bucket, object); 41 | } 42 | client.deleteBucket(bucket); 43 | } 44 | }catch(JH3Exception e){} 45 | } 46 | 47 | /** 48 | * Create / Delete an object. 49 | */ 50 | @Test 51 | public void simpleTest(){ 52 | try { 53 | //Initialize client 54 | JH3 client = new JH3(storageURI, userId); 55 | ArrayList buckets; 56 | 57 | // Create some random data and store them into an H3Object 58 | byte[] data = new byte[3 * MEGABYTE]; 59 | new Random().nextBytes(data); 60 | JH3Object dataObj = new JH3Object(data, 3* MEGABYTE); 61 | 62 | // Check if there are any buckets 63 | buckets = client.listBuckets(); 64 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 65 | assertTrue(buckets.isEmpty()); 66 | 67 | // Create a bucket 68 | assertTrue(client.createBucket("b1")); 69 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 70 | 71 | // Create an object in H3 72 | assertTrue(client.createObject("b1", "o1", dataObj)); 73 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 74 | 75 | // List multiparts in bucket 76 | assertTrue(client.listMultiparts("b1").isEmpty()); 77 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 78 | 79 | // Create a multipart object 80 | JH3MultipartId multipart = client.createMultipart("b1", "m1"); 81 | assertNotNull(multipart); 82 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 83 | 84 | // Create a part 85 | assertTrue(client.createPart(dataObj, multipart, 1)); 86 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 87 | 88 | // Create a second H3Object from a segment of the first 89 | JH3Object dataObj2 = new JH3Object(Arrays.copyOfRange(dataObj.getData(), 0, MEGABYTE), MEGABYTE); 90 | 91 | // Create a second part 92 | assertTrue(client.createPart(dataObj2, multipart, 0)); 93 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 94 | 95 | // Create a third part 96 | assertTrue(client.createPart(dataObj, multipart, 2)); 97 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 98 | 99 | // Create a part by copying 100 | assertTrue(client.createPartCopy("o1", 0, MEGABYTE, multipart, 0)); 101 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 102 | 103 | // List parts in multipart 104 | ArrayList parts = client.listParts(multipart); 105 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 106 | assertNotNull(parts); 107 | assertEquals(3, parts.size()); 108 | 109 | for (JH3PartInfo part : parts){ 110 | if (part.getPartNumber() == 0) 111 | assertEquals(MEGABYTE, part.getSize()); 112 | else 113 | assertEquals(3 * MEGABYTE, part.getSize()); 114 | } 115 | 116 | 117 | // Complete a multipart object 118 | assertTrue(client.completeMultipart(multipart)); 119 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 120 | 121 | // Try to complete again 122 | assertFalse(client.completeMultipart(multipart)); 123 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 124 | 125 | // Try to abort completed multipart 126 | assertFalse(client.abortMultipart(multipart)); 127 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 128 | 129 | // Check if completed multipart appears as object in bucket 130 | assertTrue(client.listObjects("b1", 0).contains("m1")); 131 | 132 | // Check info of the object 133 | JH3ObjectInfo objectInfo = client.infoObject("b1", "m1"); 134 | assertNotNull(objectInfo); 135 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 136 | assertFalse(objectInfo.isCorrupt()); 137 | assertEquals(7 * MEGABYTE, objectInfo.getSize()); 138 | //assertEquals(0, objectInfo.getCreation()); 139 | //assertEquals(0, objectInfo.getLastAccess()); 140 | //assertEquals(0, objectInfo.getLastModification()); 141 | 142 | // Delete objects 143 | assertTrue(client.deleteObject("b1", "m1")); 144 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 145 | 146 | assertTrue(client.deleteObject("b1", "o1")); 147 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 148 | 149 | // List objects 150 | assertTrue(client.listObjects("b1", 0).isEmpty()); 151 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 152 | 153 | // Delete bucket 154 | assertTrue(client.deleteBucket("b1")); 155 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 156 | 157 | // List buckets 158 | assertTrue(client.listBuckets().isEmpty()); 159 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 160 | 161 | } catch (JH3Exception e) { 162 | e.printStackTrace(); 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /JH3lib/JH3/src/test/java/gr/forth/ics/JH3lib/JH3BucketTest.java: -------------------------------------------------------------------------------- 1 | package gr.forth.ics.JH3lib; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | 6 | import java.io.File; 7 | import java.nio.file.Files; 8 | import java.util.ArrayList; 9 | import java.util.Random; 10 | 11 | import static org.junit.Assert.*; 12 | 13 | public class JH3BucketTest { 14 | 15 | // Initialize storage URI; if H3LIB_STORAGE_URI is not set, use default uri (local filesystem: /tmp/h3) 16 | private static String storageURI = null; 17 | static { 18 | storageURI = System.getenv("H3LIB_STORAGE_URI"); 19 | if(storageURI == null){ 20 | storageURI = "file:///tmp/h3"; 21 | } 22 | } 23 | 24 | private int userId = 0; // Dummy userId 25 | 26 | 27 | /** 28 | * Cleanup backend before each test. 29 | */ 30 | @Before 31 | public void cleanup(){ 32 | try{ 33 | JH3 client = new JH3(storageURI, userId); 34 | ArrayList buckets = client.listBuckets(); 35 | for (String bucket : buckets){ 36 | ArrayList objects = client.listObjects(bucket, 0); 37 | for (String object : objects){ 38 | client.deleteObject(bucket, object); 39 | } 40 | client.deleteBucket(bucket); 41 | } 42 | } catch(JH3Exception e){} 43 | } 44 | 45 | /** 46 | * List / create / delete a bucket 47 | */ 48 | @Test 49 | public void simpleTest(){ 50 | try { 51 | ArrayList buckets; 52 | 53 | // Initialize client 54 | JH3 client = new JH3(storageURI, userId); 55 | 56 | // Check if there are any buckets 57 | buckets = client.listBuckets(); 58 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 59 | assertTrue(buckets.isEmpty()); 60 | 61 | // Create a bucket 62 | assertTrue(client.createBucket("b1")); 63 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 64 | 65 | // Create the same bucket 66 | assertFalse(client.createBucket("b1")); 67 | assertEquals(JH3Status.JH3_EXISTS, client.getStatus()); 68 | 69 | // Get info of bucket without stats 70 | JH3BucketInfo bucketInfo = client.infoBucket("b1"); 71 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 72 | assertNull(bucketInfo.getStats()); 73 | assertNotEquals(0, bucketInfo.getCreation()); 74 | 75 | // Get info of bucket with stats 76 | bucketInfo = client.infoBucket("b1", true); 77 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 78 | assertNotNull(bucketInfo.getStats()); 79 | assertEquals(0, bucketInfo.getStats().getSize()); 80 | assertEquals(0, bucketInfo.getStats().getNumObjects()); 81 | 82 | ArrayList expected = new ArrayList<>(); 83 | expected.add("b1"); 84 | 85 | // List buckets should contain the bucket now 86 | assertEquals(expected, client.listBuckets()); 87 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 88 | 89 | // Delete a bucket 90 | assertTrue(client.deleteBucket("b1")); 91 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 92 | 93 | // Delete the same bucket 94 | assertFalse(client.deleteBucket("b1")); 95 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 96 | 97 | // Get info of non-existent bucket 98 | assertNull(client.infoBucket("b1")); 99 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 100 | 101 | // Check if there are any buckets 102 | buckets = client.listBuckets(); 103 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 104 | assertTrue(buckets.isEmpty()); 105 | 106 | } catch (JH3Exception e) { 107 | e.printStackTrace(); 108 | } 109 | 110 | } 111 | 112 | 113 | /** 114 | * Pass invalid arguments 115 | */ 116 | @Test 117 | public void argumentsTest(){ 118 | try { 119 | // Initialize client 120 | JH3 client = new JH3(storageURI, userId); 121 | 122 | // Bucket with empty name 123 | assertFalse(client.createBucket("")); 124 | assertEquals(JH3Status.JH3_INVALID_ARGS, client.getStatus()); 125 | 126 | // Bucket with null 127 | try { 128 | client.createBucket(null); 129 | } catch (NullPointerException e) { 130 | // Null pointer is the expected exception; don't handle other exceptions 131 | } 132 | 133 | // Large bucket name 134 | String largeName = new String(new char[JH3.H3_BUCKET_NAME_SIZE + 1]).replace("\0", "a"); 135 | assertFalse(client.createBucket(largeName)); 136 | assertEquals(JH3Status.JH3_NAME_TOO_LONG, client.getStatus()); 137 | 138 | // Invalid bucket names 139 | assertFalse(client.createBucket("/bucketId")); 140 | assertEquals(JH3Status.JH3_INVALID_ARGS, client.getStatus()); 141 | 142 | assertTrue(client.createBucket("\bucketId")); 143 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 144 | 145 | assertTrue(client.deleteBucket("\bucketId")); 146 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 147 | 148 | } catch (JH3Exception e) { 149 | e.printStackTrace(); 150 | } 151 | 152 | } 153 | 154 | 155 | /** 156 | * Manage many buckets 157 | */ 158 | @Test 159 | public void manyBucketsTest(){ 160 | 161 | try { 162 | int count = 100; 163 | ArrayList buckets; 164 | 165 | // Initialize client 166 | JH3 client = new JH3(storageURI, userId); 167 | 168 | // Check if there are any buckets 169 | buckets = client.listBuckets(); 170 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 171 | assertTrue(buckets.isEmpty()); 172 | 173 | // Create buckets 174 | for(int i = 0; i < count; i++){ 175 | assertTrue(client.createBucket("bucket" + i)); 176 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 177 | } 178 | 179 | // Randomly create 10 already existing buckets 180 | Random rn = new Random(); 181 | for(int i = 0; i < 10; i++){ 182 | assertFalse(client.createBucket("bucket" + rn.nextInt(count))); 183 | assertEquals(JH3Status.JH3_EXISTS, client.getStatus()); 184 | } 185 | 186 | // Get info of all buckets (without stats) 187 | for(int i = 0; i < count; i++){ 188 | JH3BucketInfo bucketInfo = client.infoBucket("bucket" + i); 189 | assertNull(bucketInfo.getStats()); 190 | assertNotEquals(0, bucketInfo.getCreation()); 191 | } 192 | 193 | ArrayList expected = new ArrayList<>(); 194 | for (int i = 0; i < count; i++) { 195 | expected.add("bucket" + i); 196 | } 197 | 198 | // List all buckets 199 | assertEquals(expected, client.listBuckets()); 200 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 201 | 202 | // Delete all buckets 203 | for (int i = 0; i < count; i++) { 204 | assertTrue(client.deleteBucket("bucket" + i)); 205 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 206 | } 207 | 208 | // Randomly delete already deleted buckets 209 | for (int i = 0; i < 10; i++) { 210 | assertFalse(client.deleteBucket("bucket" + i)); 211 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 212 | } 213 | 214 | // Randomly get info of non-existent buckets 215 | for (int i = 0; i < 10; i++) { 216 | assertNull(client.infoBucket("bucket" + rn.nextInt(count))); 217 | assertEquals(JH3Status.JH3_NOT_EXISTS, client.getStatus()); 218 | } 219 | 220 | // List buckets should be empty 221 | buckets = client.listBuckets(); 222 | assertEquals(JH3Status.JH3_SUCCESS, client.getStatus()); 223 | assertTrue(buckets.isEmpty()); 224 | 225 | } catch (JH3Exception e) { 226 | e.printStackTrace(); 227 | } 228 | 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /h3lib/url_parser.c: -------------------------------------------------------------------------------- 1 | /*_ 2 | * Copyright 2010-2011 Scyphus Solutions Co. Ltd. All rights reserved. 3 | * 4 | * Authors: 5 | * Hirochika Asai 6 | */ 7 | 8 | /* 9 | * Original code found at: http://draft.scyphus.co.jp/lang/c/url_parser.html. 10 | * Changes marked with "FIXED". 11 | * 12 | * Copyright 2020 FORTH-ICS 13 | * 14 | * Authors: 15 | * Antony Chazapis 16 | */ 17 | 18 | #include "url_parser.h" 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | /* 26 | * Prototype declarations 27 | */ 28 | static __inline__ int _is_scheme_char(int); 29 | 30 | /* 31 | * Check whether the character is permitted in scheme string 32 | */ 33 | static __inline__ int 34 | _is_scheme_char(int c) 35 | { 36 | return (!isalpha(c) && '+' != c && '-' != c && '.' != c) ? 0 : 1; 37 | } 38 | 39 | /* 40 | * See RFC 1738, 3986 41 | */ 42 | struct parsed_url * 43 | parse_url(const char *url) 44 | { 45 | struct parsed_url *purl; 46 | const char *tmpstr; 47 | const char *curstr; 48 | size_t len; /* FIXED: Change type */ 49 | int i; 50 | int userpass_flag; 51 | int bracket_flag; 52 | 53 | /* Allocate the parsed url storage */ 54 | purl = malloc(sizeof(struct parsed_url)); 55 | if ( NULL == purl ) { 56 | return NULL; 57 | } 58 | purl->scheme = NULL; 59 | purl->host = NULL; 60 | purl->port = NULL; 61 | purl->path = NULL; 62 | purl->query = NULL; 63 | purl->fragment = NULL; 64 | purl->username = NULL; 65 | purl->password = NULL; 66 | 67 | curstr = url; 68 | 69 | /* 70 | * : 71 | * := [a-z\+\-\.]+ 72 | * upper case = lower case for resiliency 73 | */ 74 | /* Read scheme */ 75 | tmpstr = strchr(curstr, ':'); 76 | if ( NULL == tmpstr ) { 77 | /* Not found the character */ 78 | parsed_url_free(purl); 79 | return NULL; 80 | } 81 | /* Get the scheme length */ 82 | len = tmpstr - curstr; 83 | /* Check restrictions */ 84 | for ( i = 0; i < len; i++ ) { 85 | if ( !_is_scheme_char(curstr[i]) ) { 86 | /* Invalid format */ 87 | parsed_url_free(purl); 88 | return NULL; 89 | } 90 | } 91 | /* Copy the scheme to the storage */ 92 | purl->scheme = malloc(sizeof(char) * (len + 1)); 93 | if ( NULL == purl->scheme ) { 94 | parsed_url_free(purl); 95 | return NULL; 96 | } 97 | (void)strncpy(purl->scheme, curstr, len); 98 | purl->scheme[len] = '\0'; 99 | /* Make the character to lower if it is upper case. */ 100 | for ( i = 0; i < len; i++ ) { 101 | purl->scheme[i] = tolower(purl->scheme[i]); 102 | } 103 | /* Skip ':' */ 104 | tmpstr++; 105 | curstr = tmpstr; 106 | 107 | /* 108 | * //:@:/ 109 | * Any ":", "@" and "/" must be encoded. 110 | */ 111 | /* Eat "//" */ 112 | for ( i = 0; i < 2; i++ ) { 113 | if ( '/' != *curstr ) { 114 | parsed_url_free(purl); 115 | return NULL; 116 | } 117 | curstr++; 118 | } 119 | 120 | /* Check if the user (and password) are specified. */ 121 | userpass_flag = 0; 122 | tmpstr = curstr; 123 | while ( '\0' != *tmpstr ) { 124 | if ( '@' == *tmpstr ) { 125 | /* Username and password are specified */ 126 | userpass_flag = 1; 127 | break; 128 | } else if ( '/' == *tmpstr ) { 129 | /* End of : specification */ 130 | userpass_flag = 0; 131 | break; 132 | } 133 | tmpstr++; 134 | } 135 | 136 | /* User and password specification */ 137 | tmpstr = curstr; 138 | if ( userpass_flag ) { 139 | /* Read username */ 140 | while ( '\0' != *tmpstr && ':' != *tmpstr && '@' != *tmpstr ) { 141 | tmpstr++; 142 | } 143 | len = tmpstr - curstr; 144 | purl->username = malloc(sizeof(char) * (len + 1)); 145 | if ( NULL == purl->username ) { 146 | parsed_url_free(purl); 147 | return NULL; 148 | } 149 | (void)strncpy(purl->username, curstr, len); 150 | purl->username[len] = '\0'; 151 | /* Proceed current pointer */ 152 | curstr = tmpstr; 153 | if ( ':' == *curstr ) { 154 | /* Skip ':' */ 155 | curstr++; 156 | /* Read password */ 157 | tmpstr = curstr; 158 | while ( '\0' != *tmpstr && '@' != *tmpstr ) { 159 | tmpstr++; 160 | } 161 | len = tmpstr - curstr; 162 | purl->password = malloc(sizeof(char) * (len + 1)); 163 | if ( NULL == purl->password ) { 164 | parsed_url_free(purl); 165 | return NULL; 166 | } 167 | (void)strncpy(purl->password, curstr, len); 168 | purl->password[len] = '\0'; 169 | curstr = tmpstr; 170 | } 171 | /* Skip '@' */ 172 | if ( '@' != *curstr ) { 173 | parsed_url_free(purl); 174 | return NULL; 175 | } 176 | curstr++; 177 | } 178 | 179 | if ( '[' == *curstr ) { 180 | bracket_flag = 1; 181 | } else { 182 | bracket_flag = 0; 183 | } 184 | /* Proceed on by delimiters with reading host */ 185 | tmpstr = curstr; 186 | while ( '\0' != *tmpstr ) { 187 | if ( bracket_flag && ']' == *tmpstr ) { 188 | /* End of IPv6 address. */ 189 | tmpstr++; 190 | break; 191 | } else if ( !bracket_flag && (':' == *tmpstr || '/' == *tmpstr) ) { 192 | /* Port number is specified. */ 193 | break; 194 | } 195 | tmpstr++; 196 | } 197 | len = tmpstr - curstr; 198 | /* FIXED: Make host optional */ 199 | if ( len != 0 ) { 200 | purl->host = malloc(sizeof(char) * (len + 1)); 201 | if ( NULL == purl->host || len <= 0 ) { 202 | parsed_url_free(purl); 203 | return NULL; 204 | } 205 | (void)strncpy(purl->host, curstr, len); 206 | purl->host[len] = '\0'; 207 | } 208 | curstr = tmpstr; 209 | 210 | /* Is port number specified? */ 211 | if ( ':' == *curstr ) { 212 | curstr++; 213 | /* Read port number */ 214 | tmpstr = curstr; 215 | while ( '\0' != *tmpstr && '/' != *tmpstr ) { 216 | tmpstr++; 217 | } 218 | len = tmpstr - curstr; 219 | purl->port = malloc(sizeof(char) * (len + 1)); 220 | if ( NULL == purl->port ) { 221 | parsed_url_free(purl); 222 | return NULL; 223 | } 224 | (void)strncpy(purl->port, curstr, len); 225 | purl->port[len] = '\0'; 226 | curstr = tmpstr; 227 | } 228 | 229 | /* End of the string */ 230 | if ( '\0' == *curstr ) { 231 | return purl; 232 | } 233 | 234 | /* Skip '/' */ 235 | if ( '/' != *curstr ) { 236 | parsed_url_free(purl); 237 | return NULL; 238 | } 239 | curstr++; 240 | 241 | /* Parse path */ 242 | tmpstr = curstr; 243 | while ( '\0' != *tmpstr && '#' != *tmpstr && '?' != *tmpstr ) { 244 | tmpstr++; 245 | } 246 | len = tmpstr - curstr; 247 | purl->path = malloc(sizeof(char) * (len + 1)); 248 | if ( NULL == purl->path ) { 249 | parsed_url_free(purl); 250 | return NULL; 251 | } 252 | (void)strncpy(purl->path, curstr, len); 253 | purl->path[len] = '\0'; 254 | curstr = tmpstr; 255 | 256 | /* Is query specified? */ 257 | if ( '?' == *curstr ) { 258 | /* Skip '?' */ 259 | curstr++; 260 | /* Read query */ 261 | tmpstr = curstr; 262 | while ( '\0' != *tmpstr && '#' != *tmpstr ) { 263 | tmpstr++; 264 | } 265 | len = tmpstr - curstr; 266 | purl->query = malloc(sizeof(char) * (len + 1)); 267 | if ( NULL == purl->query ) { 268 | parsed_url_free(purl); 269 | return NULL; 270 | } 271 | (void)strncpy(purl->query, curstr, len); 272 | purl->query[len] = '\0'; 273 | curstr = tmpstr; 274 | } 275 | 276 | /* Is fragment specified? */ 277 | if ( '#' == *curstr ) { 278 | /* Skip '#' */ 279 | curstr++; 280 | /* Read fragment */ 281 | tmpstr = curstr; 282 | while ( '\0' != *tmpstr ) { 283 | tmpstr++; 284 | } 285 | len = tmpstr - curstr; 286 | purl->fragment = malloc(sizeof(char) * (len + 1)); 287 | if ( NULL == purl->fragment ) { 288 | parsed_url_free(purl); 289 | return NULL; 290 | } 291 | (void)strncpy(purl->fragment, curstr, len); 292 | purl->fragment[len] = '\0'; 293 | curstr = tmpstr; 294 | } 295 | 296 | return purl; 297 | } 298 | 299 | /* 300 | * Free memory of parsed url 301 | */ 302 | void 303 | parsed_url_free(struct parsed_url *purl) 304 | { 305 | if ( NULL != purl ) { 306 | if ( NULL != purl->scheme ) { 307 | free(purl->scheme); 308 | } 309 | if ( NULL != purl->host ) { 310 | free(purl->host); 311 | } 312 | if ( NULL != purl->port ) { 313 | free(purl->port); 314 | } 315 | if ( NULL != purl->path ) { 316 | free(purl->path); 317 | } 318 | if ( NULL != purl->query ) { 319 | free(purl->query); 320 | } 321 | if ( NULL != purl->fragment ) { 322 | free(purl->fragment); 323 | } 324 | if ( NULL != purl->username ) { 325 | free(purl->username); 326 | } 327 | if ( NULL != purl->password ) { 328 | free(purl->password); 329 | } 330 | free(purl); 331 | } 332 | } 333 | 334 | /* 335 | * Local variables: 336 | * tab-width: 4 337 | * c-basic-offset: 4 338 | * End: 339 | * vim600: sw=4 ts=4 fdm=marker 340 | * vim<600: sw=4 ts=4 341 | */ 342 | -------------------------------------------------------------------------------- /h3lib/kv_kreon.c: -------------------------------------------------------------------------------- 1 | // Copyright [2019] [FORTH-ICS] 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #define _GNU_SOURCE 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "kv_interface.h" 26 | #include "common.h" 27 | #include "util.h" 28 | #include "url_parser.h" 29 | 30 | #include 31 | 32 | void KV_Kreon_Free(KV_Handle handle) { 33 | klc_sync(handle); 34 | // free(handle->volume_name); 35 | return; 36 | } 37 | 38 | KV_Handle KV_Kreon_Init(const char* storageUri) { 39 | struct parsed_url *url = parse_url(storageUri); 40 | if (url == NULL) { 41 | LogActivity(H3_ERROR_MSG, "ERROR: Unrecognized storage URI\n"); 42 | return NULL; 43 | } 44 | 45 | char *path; 46 | if (url->path != NULL) { 47 | path = malloc(strlen(url->path) + 2); 48 | path[0] = '/'; 49 | strcpy(&(path[1]), url->path); 50 | LogActivity(H3_INFO_MSG, "INFO: Path in URI: %s\n", path); 51 | } else { 52 | path = strdup("/tmp/h3/kreon.dat"); 53 | LogActivity(H3_INFO_MSG, "WARNING: No path in URI. Using default: /tmp/h3/kreon.dat\n"); 54 | } 55 | parsed_url_free(url); 56 | 57 | int64_t size; 58 | int fd = open(path, O_RDONLY); 59 | if (fd == -1) { 60 | LogActivity(H3_ERROR_MSG, "ERROR: Can not open %s\n", path); 61 | free(path); 62 | return NULL; 63 | } 64 | if (ioctl(fd, BLKGETSIZE64, &size) == -1) { 65 | // Maybe this is a file 66 | size = lseek(fd, 0, SEEK_END); 67 | if (size == -1) { 68 | LogActivity(H3_ERROR_MSG, "ERROR: Can not determine size of %s\n", path); 69 | free(path); 70 | return NULL; 71 | } 72 | } 73 | close(fd); 74 | 75 | klc_db_options db_options; 76 | db_options.volume_name = path; 77 | db_options.db_name = path; 78 | db_options.volume_start = 0; 79 | db_options.volume_size = size; 80 | db_options.create_flag = KLC_CREATE_DB; 81 | 82 | klc_handle handle = klc_open(&db_options); 83 | if (!handle) { 84 | LogActivity(H3_ERROR_MSG, "ERROR: Failed to initialize Kreon\n"); 85 | free(path); 86 | return NULL; 87 | } 88 | 89 | return handle; 90 | } 91 | 92 | KV_Status KV_Kreon_List(KV_Handle handle, KV_Key prefix, uint8_t nTrim, KV_Key buffer, uint32_t offset, uint32_t* nKeys){ 93 | size_t remaining = KV_LIST_BUFFER_SIZE; 94 | uint32_t nRequiredKeys = *nKeys>0?*nKeys:UINT32_MAX; 95 | uint32_t nMatchingKeys = 0; 96 | 97 | klc_scanner scanner; 98 | struct klc_key prefix_key; 99 | struct klc_key kreon_key; 100 | 101 | prefix_key.size = strlen(prefix); 102 | prefix_key.data = prefix; 103 | scanner = klc_init_scanner(handle, &prefix_key, KLC_GREATER_OR_EQUAL); 104 | if (!scanner) 105 | return KV_FAILURE; 106 | while (klc_is_valid(scanner)) { 107 | kreon_key = klc_get_key(scanner); 108 | if (memcmp(prefix, kreon_key.data, strlen(prefix)) != 0) 109 | break; 110 | 111 | LogActivity(H3_DEBUG_MSG, "key size: %u data: '%*s'\n", kreon_key.size, kreon_key.size, kreon_key.data); 112 | 113 | if(offset) 114 | offset--; 115 | 116 | else { 117 | 118 | // Copy the keys if a buffer is provided... 119 | if(buffer){ 120 | size_t entrySize = kreon_key.size - nTrim; 121 | if(remaining >= entrySize){ 122 | memcpy(&buffer[KV_LIST_BUFFER_SIZE - remaining], &kreon_key.data[nTrim], entrySize); 123 | remaining -= entrySize; 124 | nMatchingKeys++; 125 | } 126 | else 127 | break; 128 | } 129 | 130 | // ... otherwise just count them. 131 | else 132 | nMatchingKeys++; 133 | } 134 | if (nMatchingKeys >= nRequiredKeys) 135 | break; 136 | 137 | if (klc_get_next(scanner) && !klc_is_valid(scanner)) 138 | break; 139 | } 140 | klc_close_scanner(scanner); 141 | 142 | *nKeys = nMatchingKeys; 143 | return KV_SUCCESS; 144 | } 145 | 146 | KV_Status KV_Kreon_Exists(KV_Handle handle, KV_Key key) { 147 | struct klc_key kreon_key; 148 | kreon_key.size = strlen(key) + 1; 149 | kreon_key.data = key; 150 | if (klc_exists(handle, &kreon_key) == KLC_SUCCESS) 151 | return KV_KEY_EXIST; 152 | 153 | return KV_KEY_NOT_EXIST; 154 | } 155 | 156 | KV_Status KV_Kreon_Read(KV_Handle handle, KV_Key key, off_t offset, KV_Value* value, size_t* size) { 157 | KV_Status status = KV_FAILURE; 158 | 159 | size_t segmentSize; 160 | char *segment = NULL; 161 | 162 | struct klc_key kreon_key; 163 | kreon_key.size = strlen(key)+1; 164 | kreon_key.data = key; 165 | struct klc_value kreon_value; 166 | struct klc_value *kreon_value_p; 167 | kreon_value_p = &kreon_value; 168 | switch (klc_get(handle, &kreon_key, &kreon_value_p)) { 169 | case KLC_SUCCESS: 170 | if (offset > kreon_value.size){ 171 | *size = 0; 172 | return KV_SUCCESS; 173 | } 174 | 175 | if (*value == NULL) { 176 | segmentSize = kreon_value.size - offset; 177 | if ((segment = malloc(segmentSize))) { 178 | memcpy(segment, kreon_value.data + offset, segmentSize); 179 | *value = (KV_Value)segment; 180 | *size = segmentSize; 181 | status = KV_SUCCESS; 182 | } 183 | } else { 184 | segmentSize = min(kreon_value.size - offset, *size); 185 | memcpy(*value, kreon_value.data + offset, segmentSize); 186 | *size = segmentSize; 187 | status = KV_SUCCESS; 188 | } 189 | 190 | break; 191 | case KLC_KEY_NOT_FOUND: 192 | return KV_KEY_NOT_EXIST; 193 | default: 194 | return KV_FAILURE; 195 | } 196 | 197 | return status; 198 | } 199 | 200 | KV_Status KV_Kreon_Update(KV_Handle handle, KV_Key key, KV_Value value, off_t offset, size_t size) { 201 | KV_Status status; 202 | 203 | KV_Value newValue; 204 | size_t newSize; 205 | 206 | struct klc_key kreon_key; 207 | kreon_key.size = strlen(key)+1; 208 | kreon_key.data = key; 209 | struct klc_value kreon_value; 210 | struct klc_value *kreon_value_p; 211 | kreon_value_p = &kreon_value; 212 | switch (klc_get(handle, &kreon_key, &kreon_value_p)) { 213 | case KLC_SUCCESS: 214 | if (offset + size <= kreon_value.size) { 215 | newSize = kreon_value.size; 216 | } else { 217 | newSize = offset + size; 218 | } 219 | newValue = (KV_Value)malloc(newSize); 220 | memcpy(newValue, kreon_value.data, kreon_value.size); 221 | memcpy(newValue + offset, value, size); 222 | break; 223 | case KLC_KEY_NOT_FOUND: 224 | newSize = offset + size; 225 | newValue = (KV_Value)calloc(1, newSize); 226 | memcpy(newValue + offset, value, size); 227 | break; 228 | default: 229 | return KV_FAILURE; 230 | } 231 | 232 | // Convert key blob to string 233 | struct klc_key_value kreon_key_value; 234 | kreon_key_value.k.size = strlen(key)+1; 235 | kreon_key_value.k.data = key; 236 | kreon_key_value.v.size = newSize; 237 | kreon_key_value.v.data = (char *)newValue; 238 | if(klc_put(handle, &kreon_key_value) == KLC_SUCCESS) { 239 | status = KV_SUCCESS; 240 | } else { 241 | status = KV_FAILURE; 242 | } 243 | free(newValue); 244 | 245 | return status; 246 | } 247 | 248 | KV_Status KV_Kreon_Write(KV_Handle handle, KV_Key key, KV_Value value, size_t size) { 249 | 250 | // Convert key blob to string 251 | struct klc_key_value kreon_key_value; 252 | kreon_key_value.k.size = strlen(key)+1; 253 | kreon_key_value.k.data = key; 254 | kreon_key_value.v.size = size; 255 | kreon_key_value.v.data = (char *)value; 256 | if(klc_put(handle, &kreon_key_value) == KLC_SUCCESS) 257 | return KV_SUCCESS; 258 | 259 | return KV_FAILURE; 260 | } 261 | 262 | KV_Status KV_Kreon_Delete(KV_Handle handle, KV_Key key) { 263 | 264 | KV_Status status; 265 | 266 | struct klc_key kreon_key; 267 | kreon_key.size = strlen(key) + 1; 268 | kreon_key.data = key; 269 | switch(klc_delete(handle, &kreon_key)){ 270 | case KLC_SUCCESS: status = KV_SUCCESS; break; 271 | case KLC_KEY_NOT_FOUND: status = KV_KEY_NOT_EXIST; break; 272 | default: status = KV_FAILURE; break; 273 | 274 | } 275 | 276 | return status; 277 | } 278 | 279 | KV_Status KV_Kreon_Create(KV_Handle handle, KV_Key key, KV_Value value, size_t size){ 280 | KV_Status status; 281 | 282 | if( (status = KV_Kreon_Exists(handle, key)) == KV_KEY_NOT_EXIST){ 283 | status = KV_Kreon_Write(handle, key, value, size); 284 | } 285 | 286 | return status; 287 | } 288 | 289 | KV_Status KV_Kreon_Copy(KV_Handle handle, KV_Key src_key, KV_Key dest_key) { 290 | size_t size = 0x00; 291 | KV_Value value = NULL; 292 | KV_Status status; 293 | 294 | if((status = KV_Kreon_Read(handle, src_key, 0, &value, &size)) == KV_SUCCESS){ 295 | status = KV_Kreon_Write(handle, dest_key, value, size); 296 | } 297 | 298 | return status; 299 | } 300 | 301 | KV_Status KV_Kreon_Move(KV_Handle handle, KV_Key src_key, KV_Key dest_key) { 302 | size_t size = 0x00; 303 | KV_Value value = NULL; 304 | KV_Status status; 305 | 306 | if( (status = KV_Kreon_Read(handle, src_key, 0, &value, &size)) == KV_SUCCESS && 307 | (status = KV_Kreon_Write(handle, dest_key, value, size)) == KV_SUCCESS ){ 308 | status = KV_Kreon_Delete(handle, src_key); 309 | } 310 | 311 | return status; 312 | } 313 | 314 | KV_Status KV_Kreon_Sync(KV_Handle handle) { 315 | return KV_SUCCESS; 316 | } 317 | 318 | KV_Operations operationsKreon = { 319 | .init = KV_Kreon_Init, 320 | .free = KV_Kreon_Free, 321 | .storage_info = NULL, 322 | .validate_key = NULL, 323 | 324 | .metadata_read = KV_Kreon_Read, 325 | .metadata_write = KV_Kreon_Write, 326 | .metadata_create = KV_Kreon_Create, 327 | .metadata_delete = KV_Kreon_Delete, 328 | .metadata_move = KV_Kreon_Move, 329 | .metadata_exists = KV_Kreon_Exists, 330 | 331 | .list = KV_Kreon_List, 332 | .exists = KV_Kreon_Exists, 333 | .read = KV_Kreon_Read, 334 | .create = KV_Kreon_Create, 335 | .update = KV_Kreon_Update, 336 | .write = KV_Kreon_Write, 337 | .copy = KV_Kreon_Copy, 338 | .move = KV_Kreon_Move, 339 | .delete = KV_Kreon_Delete, 340 | .sync = KV_Kreon_Sync 341 | }; 342 | -------------------------------------------------------------------------------- /h3controllers/cacheController.py: -------------------------------------------------------------------------------- 1 | # Copyright [2019] [FORTH-ICS] 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import pyh3lib 17 | import threading 18 | import time 19 | import struct 20 | from datetime import timedelta 21 | 22 | DEFAULT_EXPIRES_AT_TIME = 1800 23 | """Maximum default time that the objects stay in the hot storage (30 minutes)""" 24 | 25 | EXPIRE_INTERVAL = 1200 26 | """Periodical time that an expiration event will executed (20 minutes default)""" 27 | 28 | EVICT_INTERVAL = 600 29 | """Periodical time that an eviction event will executed (10 minutes default)""" 30 | 31 | storageLock = threading.Lock() 32 | """A lock to make sure that the eviction event and the expire event is not running at the same time""" 33 | 34 | class CacheControllerJob(threading.Thread): 35 | def __init__(self, interval, execute, *args): 36 | threading.Thread.__init__(self) 37 | self.interval = interval 38 | self.execute = execute 39 | 40 | self.daemon = False 41 | self.stopped = threading.Event() 42 | self.args = args 43 | 44 | def stop(self): 45 | self.stopped.set() 46 | self.join() 47 | 48 | def run(self): 49 | while not self.stopped.wait(self.interval.total_seconds()): 50 | self.execute(*self.args) 51 | 52 | def WatermarkRange(arg): 53 | """A range check function for the watermark arguments. 54 | 55 | :param arg: The watermark argument. 56 | :type arg: unknown 57 | """ 58 | 59 | try: 60 | percentage = int(arg) 61 | except ValueError: 62 | raise argparse.ArgumentTypeError("The percentage must be an integer number") 63 | if percentage < 0 or percentage > 100: 64 | raise argparse.ArgumentTypeError("The percentage argument must be > 0 and < 100") 65 | return percentage 66 | 67 | def WriteBackToCold(h3_hot, h3_cold, h3_bucket, h3_object): 68 | """Moves an object from the hot storage back to the cold. 69 | 70 | :param h3_hot: The H3 hot storage. 71 | :param h3_cold: The H3 cold storage. 72 | :param h3_bucket: The bucket that the object belongs to. 73 | :param h3_object: The object that we need to move. 74 | :type h3_hot: object 75 | :type h3_cold: object 76 | :type h3_bucket: string 77 | :type h3_object: string 78 | """ 79 | 80 | offset = 0 81 | while True: 82 | data = h3_hot.read_object(h3_bucket, h3_object, offset) 83 | if data is None: 84 | data = b'' 85 | 86 | h3_cold.write_object(h3_bucket, h3_object, data, offset) 87 | 88 | if data.done: 89 | break 90 | 91 | offset += len(data) 92 | 93 | # remove the object from the hot storage 94 | h3_hot.delete_object(h3_bucket, h3_object) 95 | 96 | def EvictObjects(h3_hot, h3_cold, watermark_low, watermark_high): 97 | """Checks if the hot storage is nearly full (based on the watermark_high w) 98 | in order to start evicting objects, until we reach the watermark_low 99 | percentage. 100 | 101 | :param h3_hot: The H3 hot storage. 102 | :param h3_cold: The H3 cold storage. 103 | :param watermark_low: The percentage of the disk we need to reach if we start evicting. 104 | :param watermark_high: The percentage of the disk we need to reach to start evicting. 105 | :type h3_hot: object 106 | :type h3_cold: object 107 | :type watermark_low: int 108 | :type watermark_high: int 109 | 110 | .. note:: 111 | The objects eviction follows the LRU policy. 112 | """ 113 | 114 | # acquire the storage lock 115 | storageLock.acquire() 116 | 117 | # take the cache usage 118 | storage_info = h3_hot.info_storage() 119 | 120 | # if the hot storage has reached the high watermark start to evict objects 121 | if storage_info and ( storage_info.used_space >= int(watermark_high * storage_info.total_space / 100) ): 122 | 123 | lru = [] 124 | 125 | # iterate all the objects in the cache 126 | for h3_bucket in h3_hot.list_buckets(): 127 | for h3_object in h3_hot.list_objects(h3_bucket): 128 | 129 | # take the info for each object 130 | info = h3_cold.info_object(h3_bucket, h3_object) 131 | 132 | # Todo(dimos): we can add a size threshold 133 | # append it to the lru list 134 | lru.append({ 135 | 'bucket' : h3_bucket, 136 | 'object' : h3_object, 137 | 'modified': info.last_modification, 138 | 'size' : info.size 139 | }) 140 | 141 | lru = sorted(lru, key = lambda obj: (obj['modified'], obj['size'])) 142 | 143 | watermark_low_size = int(watermark_low * storage_info.total_space / 100) 144 | watermark_high_size = storage_info.used_space 145 | 146 | # start the eviction 147 | for lru_object in lru: 148 | 149 | h3_bucket = lru_object['bucket'] 150 | h3_object = lru_object['object'] 151 | 152 | # evict the object from the cache 153 | WriteBackToCold(h3_hot, h3_cold, h3_bucket, h3_object) 154 | 155 | try: 156 | h3_cold.delete_object_metadata(h3_bucket, h3_object, 'ExpireFromCache') 157 | except pyh3lib.H3NotExistsError: 158 | pass 159 | 160 | try: 161 | h3_cold.delete_object_metadata(h3_bucket, h3_object, 'CachedAt') 162 | except pyh3lib.H3NotExistsError: 163 | pass 164 | 165 | # check if we reached the low watermark in order to stop the eviction 166 | watermark_high_size -= lru_object['size'] 167 | if watermark_high_size <= watermark_low_size: 168 | break 169 | 170 | # release the storage lock 171 | storageLock.release() 172 | 173 | def ExpireObjects(h3_hot, h3_cold, expires_time): 174 | """Checks all the objects in the hot storage if some of them 175 | have expired in order to evict them. 176 | 177 | :param h3_hot: The H3 hot storage. 178 | :param h3_cold: The H3 cold storage. 179 | :param expire_time: The time after an object assumed expired. 180 | :type h3_hot: object 181 | :type h3_cold: object 182 | :type expire_time: int 183 | 184 | .. note:: 185 | The objects expiration is based on the ``CachedAt`` and 186 | the `ExpireFromCache` attributes of an object. 187 | """ 188 | 189 | # acquire the storage lock 190 | storageLock.acquire() 191 | 192 | now = time.clock_gettime(time.CLOCK_REALTIME) 193 | 194 | # List all the buckets in the hot storage 195 | for h3_bucket in h3_hot.list_buckets(): 196 | 197 | # List all the objects in the hot bucket 198 | for h3_object in h3_hot.list_objects(h3_bucket): 199 | cached_at = None 200 | expires_from_cache = None 201 | 202 | # get when it expires from the hot 203 | try: 204 | cached_at = h3_cold.read_object_metadata(h3_bucket, h3_object, 'CachedAt') 205 | cached_at = struct.unpack('d', cached_at)[0] 206 | except struct.error: 207 | cached_at = None 208 | except pyh3lib.H3NotExistsError: 209 | pass 210 | 211 | # the user has defined a specific time from the object to be delete it from the hot 212 | try: 213 | expires_from_cache = h3_cold.read_object_metadata(h3_bucket, h3_object, 'ExpireFromCache') 214 | expires_from_cache = struct.unpack('d', expires_from_cache)[0] 215 | except struct.error: 216 | expires_from_cache = None 217 | except pyh3lib.H3NotExistsError: 218 | pass 219 | 220 | # the object must be moved back to the cold storage 221 | if expires_from_cache: 222 | if expires_from_cache <= now: 223 | # delete the CachedAt attribute 224 | h3_cold.delete_object_metadata(h3_bucket, h3_object, 'CachedAt') 225 | 226 | # delete the CachedAt attribute 227 | h3_cold.delete_object_metadata(h3_bucket, h3_object, 'ExpireFromCache') 228 | 229 | WriteBackToCold(h3_hot, h3_cold, h3_bucket, h3_object) 230 | 231 | elif cached_at: 232 | if (cached_at + expires_time) <= now: 233 | # delete the CachedAt attribute 234 | h3_cold.delete_object_metadata(h3_bucket, h3_object, 'CachedAt') 235 | 236 | WriteBackToCold(h3_hot, h3_cold, h3_bucket, h3_object) 237 | 238 | # release the storage lock 239 | storageLock.release() 240 | 241 | def main(cmd=None): 242 | parser = argparse.ArgumentParser(description='The controller responsible for managing the H3Cache.') 243 | parser.add_argument('--hot_storage', required=True, help=f'Hot H3 storage URI.') 244 | parser.add_argument('--cold_storage', required=True, help=f'Cold H3 storage URI.') 245 | parser.add_argument('--watermark_low', required=False, help=f'The percentage of the disk to stop evicting.', type=WatermarkRange, default=50) 246 | parser.add_argument('--watermark_high', required=False, help=f'The percentage of the disk to start evicting.', type=WatermarkRange, default=90) 247 | parser.add_argument('--expires_time', required=False, help=f'The time that an object is allowed to stay in the hot storage (in seconds). The default value is 30 minutes.', type=int, default=DEFAULT_EXPIRES_AT_TIME) 248 | parser.add_argument('--expire_interval', required=False, help=f'The interval between two expiration events (in seconds). The default value is 20 minutes.', type=int, default=EXPIRE_INTERVAL) 249 | parser.add_argument('--evict_interval', required=False, help=f'The interval between two eviction events (in seconds). The default value is 10 minutes.', type=int, default=EVICT_INTERVAL) 250 | 251 | commands = parser.parse_args(cmd) 252 | hot_url = commands.hot_storage 253 | cold_url = commands.cold_storage 254 | watermark_low = commands.watermark_low 255 | watermark_high = commands.watermark_high 256 | expires_time = commands.expires_time 257 | expire_interval = commands.expire_interval 258 | evict_interval = commands.evict_interval 259 | 260 | if hot_url and cold_url: 261 | h3_hot = pyh3lib.H3(hot_url) 262 | h3_cold = pyh3lib.H3(cold_url) 263 | 264 | expire_job = CacheControllerJob(timedelta(seconds=expire_interval), ExpireObjects, h3_hot, h3_cold, expires_time) 265 | expire_job.start() 266 | 267 | evict_job = CacheControllerJob(timedelta(seconds=evict_interval), EvictObjects, h3_hot, h3_cold, watermark_low, watermark_high) 268 | evict_job.start() 269 | 270 | while True: 271 | try: 272 | continue 273 | except InterruptedError: 274 | print("Cache controller has been stopped. Running clean up code") 275 | expire_job.stop() 276 | evict_job.stop() 277 | break 278 | else: 279 | parser.print_help(sys.stderr) 280 | sys.exit(1) 281 | 282 | if __name__ == '__main__': 283 | main() --------------------------------------------------------------------------------