├── example ├── hello-world.cc └── Makefile ├── meson_options.txt ├── .gitignore ├── containers ├── ubuntu-22.04-dev └── fedora-38-dev ├── fuse ├── destroy.c ├── init.c ├── chmod.c ├── chown.c ├── rmdir.c ├── unlink.c ├── access.c ├── utimens.c ├── read.c ├── julea-fuse.c ├── truncate.c ├── mkdir.c ├── create.c ├── readdir.c ├── julea-fuse.h ├── write.c └── getattr.c ├── include ├── julea-hdf5.h ├── julea-kv.h ├── julea-object.h ├── julea-item.h ├── julea-db.h ├── kv │ ├── jkv-internal.h │ └── jkv-iterator.h ├── object │ ├── jobject-internal.h │ └── jobject-iterator.h ├── hdf5 │ └── jhdf5.h ├── core │ ├── jdistribution-internal.h │ ├── jconnection-pool-internal.h │ ├── jconnection-pool.h │ ├── joperation-cache-internal.h │ ├── jconfiguration-internal.h │ ├── jbackground-operation-internal.h │ ├── jlist-internal.h │ ├── jbatch-internal.h │ ├── joperation.h │ ├── jlist-iterator.h │ ├── jdir-iterator.h │ ├── jcache.h │ ├── jmemory-chunk.h │ ├── jstatistics.h │ ├── jcredentials.h │ └── jlist.h ├── db │ ├── jdb-error.h │ └── jdb-type.h ├── julea.h ├── item │ ├── jcollection-iterator.h │ ├── jitem-iterator.h │ ├── jcollection-internal.h │ └── jcollection.h └── db-util │ └── jbson.h ├── doc ├── README.md ├── containers.md ├── hdf5.md ├── debugging.md ├── implementing-backend.md ├── configuration.md └── installation-usage.md ├── CITATION.cff ├── cli ├── cli.h ├── list.c ├── delete.c └── create.c ├── scripts ├── install-dependencies.sh ├── benchmark.sh ├── ci │ └── build.sh ├── check.sh ├── test.sh ├── format.sh ├── environment.sh └── setup.sh ├── server └── server.h ├── benchmark ├── cache.c ├── memory-chunk.c ├── benchmark.h ├── background-operation.c └── message.c ├── lib ├── core │ ├── joperation.c │ ├── distribution │ │ └── distribution.h │ ├── jmemory-chunk.c │ ├── jlist-iterator.c │ └── jcache.c ├── item │ ├── jcollection-iterator.c │ └── jitem-iterator.c ├── db │ └── jdb.c └── hdf5-db │ └── jhdf5-db-object.c ├── test ├── hdf5 │ ├── hdf-helper.h │ └── hdf-helper.c ├── core │ ├── background-operation.c │ ├── cache.c │ ├── credentials.c │ ├── memory-chunk.c │ ├── semantics.c │ └── list-iterator.c ├── item │ ├── collection.c │ ├── item-iterator.c │ └── collection-iterator.c ├── test.h └── test.c └── .github └── workflows ├── containers.yml └── dependencies.yml /example/hello-world.cc: -------------------------------------------------------------------------------- 1 | hello-world.c -------------------------------------------------------------------------------- /meson_options.txt: -------------------------------------------------------------------------------- 1 | # FIXME Rename to meson.options once we require Meson >= 1.1 2 | option('gdbm_prefix', 3 | type: 'string', 4 | value: '', 5 | description: 'Prefix for GDBM', 6 | ) 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | dependencies/ 3 | 4 | # Example 5 | example/hello-world 6 | 7 | # Doxygen 8 | html/ 9 | 10 | # Meson 11 | bld*/ 12 | 13 | # Patch 14 | *.orig 15 | *.rej 16 | 17 | # Sublime Text 18 | *.sublime-* 19 | 20 | # Vim 21 | *.swp 22 | 23 | # Visual Studio Code 24 | .vscode/ 25 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS := -std=c11 -Wall -Wextra -Wpedantic -Og $(shell pkg-config --cflags glib-2.0 julea julea-object julea-kv julea-db) 2 | LDLIBS := $(shell pkg-config --libs glib-2.0 julea julea-object julea-kv julea-db) 3 | 4 | BIN = hello-world 5 | 6 | all: $(BIN) 7 | 8 | clean: 9 | rm -f $(BIN) 10 | 11 | run: $(BIN) 12 | ../scripts/setup.sh start 13 | ./$(BIN) 14 | ../scripts/setup.sh stop 15 | -------------------------------------------------------------------------------- /containers/ubuntu-22.04-dev: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # FIXME psmisc is required for killall (setup.sh) 4 | RUN apt update && apt --yes --no-install-recommends install build-essential psmisc meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev 5 | 6 | WORKDIR /julea 7 | -------------------------------------------------------------------------------- /containers/fedora-38-dev: -------------------------------------------------------------------------------- 1 | FROM fedora:38 2 | 3 | # FIXME hostname is required for hostname (julea-config) 4 | # FIXME psmisc is required for killall (setup.sh) 5 | RUN dnf --refresh --assumeyes install gcc libasan libubsan hostname psmisc meson ninja-build pkgconf glib2-devel libbson-devel libfabric-devel lmdb-devel sqlite-devel leveldb-devel mongo-c-driver-devel mariadb-connector-c-devel rocksdb-devel fuse3-devel librados-devel 6 | 7 | WORKDIR /julea 8 | -------------------------------------------------------------------------------- /fuse/destroy.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | void 24 | jfs_destroy(void* data) 25 | { 26 | (void)data; 27 | } 28 | -------------------------------------------------------------------------------- /include/julea-hdf5.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_HDF5_H 24 | #define JULEA_HDF5_H 25 | 26 | #include 27 | 28 | #undef JULEA_HDF5_H 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /fuse/init.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | void* 24 | jfs_init(struct fuse_conn_info* conn, struct fuse_config* cfg) 25 | { 26 | (void)conn; 27 | (void)cfg; 28 | 29 | return NULL; 30 | } 31 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | JULEA is a flexible storage framework that allows offering arbitrary client interfaces to applications. 4 | To be able to rapidly prototype new approaches, it offers object and key-value backends that can either be client-side or server-side; 5 | backends for popular storage technologies such as POSIX, LevelDB and MongoDB have already been implemented. 6 | 7 | Additionally, JULEA allows dynamically adapting the I/O operations' semantics and can thus be adjusted to different use-cases. 8 | It runs completely in user space, which eases development and debugging. 9 | Its goal is to provide a solid foundation for storage research and teaching. 10 | 11 | ## Links 12 | 13 | * [Quick Start](../README.md) 14 | * [Dependencies](dependencies.md) 15 | * [Containers](containers.md) 16 | * [Installation and Usage](installation-usage.md) 17 | * [Configuration](configuration.md) 18 | * [Debugging](debugging.md) 19 | * [Implementing a Backend](implementing-backend.md) 20 | * [JULEA-DB Details](db-code.md) 21 | * [HDF5 Support](hdf5.md) 22 | -------------------------------------------------------------------------------- /include/julea-kv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_KV_H 24 | #define JULEA_KV_H 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #undef JULEA_KV_H 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use JULEA, please cite the paper below." 3 | authors: 4 | - family-names: "Kuhn" 5 | given-names: "Michael" 6 | title: "JULEA -- A Flexible Storage Framework for HPC" 7 | preferred-citation: 8 | type: conference-paper 9 | authors: 10 | - family-names: "Kuhn" 11 | given-names: "Michael" 12 | title: "JULEA: A Flexible Storage Framework for HPC" 13 | doi: "10.1007/978-3-319-67630-2_51" 14 | collection-title: "High Performance Computing - ISC High Performance 2017 International Workshops, DRBSD, ExaComm, HCPM, HPC-IODC, IWOPH, IXPUG, P^3MA, VHPC, Visualization at Scale, WOPSSS, Revised Selected Papers" 15 | editors: 16 | - family-names: "Kunkel" 17 | given-names: "Julian M." 18 | - family-names: "Yokota" 19 | given-names: "Rio" 20 | - family-names: "Taufer" 21 | given-names: "Michela" 22 | - family-names: "Shalf" 23 | given-names: "John" 24 | conference: 25 | name: "ISC High Performance 2017" 26 | city: "Frankfurt" 27 | country: DE 28 | date-start: "2017-06-18" 29 | date-end: "2017-06-22" 30 | year: 2017 31 | -------------------------------------------------------------------------------- /fuse/chmod.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_chmod(char const* path, mode_t mode, struct fuse_file_info* fi) 27 | { 28 | gint ret = -ENOENT; 29 | 30 | (void)path; 31 | (void)mode; 32 | (void)fi; 33 | 34 | return ret; 35 | } 36 | -------------------------------------------------------------------------------- /include/julea-object.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_OBJECT_H 24 | #define JULEA_OBJECT_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #undef JULEA_OBJECT_H 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /include/julea-item.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_ITEM_H 24 | #define JULEA_ITEM_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #undef JULEA_ITEM_H 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /fuse/chown.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_chown(char const* path, uid_t uid, gid_t gid, struct fuse_file_info* fi) 27 | { 28 | gint ret = -ENOENT; 29 | 30 | (void)path; 31 | (void)uid; 32 | (void)gid; 33 | (void)fi; 34 | 35 | return ret; 36 | } 37 | -------------------------------------------------------------------------------- /include/julea-db.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2019 Benjamin Warnke 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DB_H 24 | #define JULEA_DB_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #undef JULEA_DB_H 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /include/kv/jkv-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2017-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_KV_KV_INTERNAL_H 24 | #define JULEA_KV_KV_INTERNAL_H 25 | 26 | #if !defined(JULEA_KV_H) && !defined(JULEA_KV_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | G_GNUC_INTERNAL JBackend* j_kv_get_backend(void); 37 | 38 | G_END_DECLS 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /include/object/jobject-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2017-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_OBJECT_OBJECT_INTERNAL_H 24 | #define JULEA_OBJECT_OBJECT_INTERNAL_H 25 | 26 | #if !defined(JULEA_OBJECT_H) && !defined(JULEA_OBJECT_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | G_GNUC_INTERNAL JBackend* j_object_get_backend(void); 37 | 38 | G_END_DECLS 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /cli/cli.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | void j_cmd_usage(void); 27 | 28 | guint j_cmd_arguments_length(gchar const**); 29 | 30 | gboolean j_cmd_error_last(JURI*); 31 | 32 | gboolean j_cmd_create(gchar const**, gboolean); 33 | gboolean j_cmd_copy(gchar const**); 34 | gboolean j_cmd_delete(gchar const**); 35 | gboolean j_cmd_list(gchar const**); 36 | gboolean j_cmd_status(gchar const**); 37 | -------------------------------------------------------------------------------- /fuse/rmdir.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_rmdir(char const* path) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | 33 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 34 | kv = j_kv_new("posix", path); 35 | 36 | j_kv_delete(kv, batch); 37 | 38 | if (j_batch_execute(batch)) 39 | { 40 | ret = 0; 41 | } 42 | 43 | return ret; 44 | } 45 | -------------------------------------------------------------------------------- /include/hdf5/jhdf5.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2017-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_HDF5_HDF5_H 24 | #define JULEA_HDF5_HDF5_H 25 | 26 | #if !defined(JULEA_HDF5_H) && !defined(JULEA_HDF5_KV_COMPILATION) && !defined(JULEA_HDF5_DB_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | 36 | G_BEGIN_DECLS 37 | 38 | void j_hdf5_set_semantics(JSemantics*); 39 | 40 | G_END_DECLS 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /include/core/jdistribution-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DISTRIBUTION_INTERNAL_H 24 | #define JULEA_DISTRIBUTION_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | #include 36 | 37 | G_BEGIN_DECLS 38 | 39 | G_GNUC_INTERNAL void j_distribution_init(void); 40 | 41 | G_END_DECLS 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /scripts/install-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2017-2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | SELF_BASE="${SELF_PATH##*/}" 24 | 25 | # shellcheck source=scripts/common 26 | . "${SELF_DIR}/common" 27 | # shellcheck source=scripts/spack 28 | . "${SELF_DIR}/spack" 29 | 30 | usage () 31 | { 32 | echo "Usage: ${SELF_BASE}" 33 | exit 1 34 | } 35 | 36 | if test -z "${JULEA_SPACK_DIR}" 37 | then 38 | JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" 39 | fi 40 | 41 | spack_install_dependencies 42 | -------------------------------------------------------------------------------- /doc/containers.md: -------------------------------------------------------------------------------- 1 | # Containers 2 | 3 | For easier setup or testing JULEA, you can use the development container, which contains JULEA's required dependencies. 4 | 5 | ## Development Container 6 | 7 | Setting up and using the container looks like the following: 8 | 9 | ```console 10 | $ git clone https://github.com/parcio/julea.git 11 | 12 | $ docker pull ghcr.io/parcio/ubuntu-dev:22.04 13 | $ docker run -v $PWD/julea:/julea -it ghcr.io/parcio/ubuntu-dev:22.04 14 | ``` 15 | 16 | Continue with the following commands inside the container: 17 | 18 | ```console 19 | $ . scripts/environment.sh 20 | 21 | $ meson setup --prefix="/julea/install" -Db_sanitize=address,undefined bld 22 | $ ninja -C bld 23 | 24 | $ julea-config --user \ 25 | --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" \ 26 | --object-backend=posix --object-path="/tmp/julea-$(id -u)/posix" \ 27 | --kv-backend=lmdb --kv-path="/tmp/julea-$(id -u)/lmdb" \ 28 | --db-backend=sqlite --db-path="/tmp/julea-$(id -u)/sqlite" 29 | 30 | $ ./scripts/setup.sh start 31 | $ ./scripts/test.sh 32 | $ ./scripts/setup.sh stop 33 | ``` 34 | 35 | ### Building Container 36 | 37 | The Dockerfile can be found at `containers/ubuntu-22.04-dev`. 38 | To build the container, use the following commands: 39 | 40 | ```console 41 | $ cd julea/containers 42 | $ docker build -f ubuntu-22.04-dev -t parcio/ubuntu-dev:22.04 . 43 | ``` 44 | -------------------------------------------------------------------------------- /fuse/unlink.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_unlink(char const* path) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | g_autoptr(JObject) obj = NULL; 33 | 34 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 35 | kv = j_kv_new("posix", path); 36 | obj = j_object_new("posix", path); 37 | 38 | j_kv_delete(kv, batch); 39 | // we do not support hard links so deleting here is safe 40 | j_object_delete(obj, batch); 41 | 42 | if (j_batch_execute(batch)) 43 | { 44 | ret = 0; 45 | } 46 | 47 | return ret; 48 | } 49 | -------------------------------------------------------------------------------- /fuse/access.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_access(char const* path, int mask) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | gpointer value; 33 | guint32 len; 34 | 35 | (void)mask; 36 | 37 | if (g_strcmp0(path, "/") == 0) 38 | { 39 | return 0; 40 | } 41 | 42 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 43 | kv = j_kv_new("posix", path); 44 | 45 | j_kv_get(kv, &value, &len, batch); 46 | 47 | if (j_batch_execute(batch)) 48 | { 49 | ret = 0; 50 | 51 | g_free(value); 52 | } 53 | 54 | return ret; 55 | } 56 | -------------------------------------------------------------------------------- /fuse/utimens.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_utimens(char const* path, const struct timespec ts[2], struct fuse_file_info* fi) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | gpointer value; 33 | guint32 len; 34 | 35 | (void)ts; 36 | (void)fi; 37 | 38 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 39 | kv = j_kv_new("posix", path); 40 | 41 | j_kv_get(kv, &value, &len, batch); 42 | 43 | if (j_batch_execute(batch)) 44 | { 45 | /// \todo 46 | ret = 0; 47 | 48 | g_free(value); 49 | } 50 | 51 | return ret; 52 | } 53 | -------------------------------------------------------------------------------- /include/core/jconnection-pool-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_CONNECTION_POOL_INTERNAL_H 24 | #define JULEA_CONNECTION_POOL_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | #include 32 | 33 | #include 34 | 35 | G_BEGIN_DECLS 36 | 37 | /** 38 | * \addtogroup JConnectionPool 39 | * 40 | * @{ 41 | **/ 42 | 43 | G_GNUC_INTERNAL void j_connection_pool_init(JConfiguration*); 44 | G_GNUC_INTERNAL void j_connection_pool_fini(void); 45 | 46 | /** 47 | * @} 48 | **/ 49 | 50 | G_END_DECLS 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /doc/hdf5.md: -------------------------------------------------------------------------------- 1 | # HDF5 Support 2 | 3 | JULEA supports HDF5 applications via two Virtual Object Layer (VOL) plugins: 4 | 1. The `julea-kv` VOL plugin stores data as distributed objects using the object client and metadata as key-value pairs using the kv client. 5 | 2. The `julea-db` VOL plugin stores data as distributed objects using the object client and metadata as database entries using the db client. 6 | 7 | To make use of JULEA's HDF5 support, make sure that you have set up JULEA using either the [Quick Start](../README.md#quick-start) or the [Installation and Usage](installation-usage.md) documentation. 8 | 9 | JULEA's environment script will set `HDF5_PLUGIN_PATH`, which allows HDF5 to find JULEA's VOL plugins. 10 | The VOL plugin to use can be selected using the `HDF5_VOL_CONNECTOR` environment variable: 11 | 12 | ```console 13 | $ . scripts/environment.sh 14 | $ export HDF5_VOL_CONNECTOR=julea-kv 15 | $ my-application 16 | ``` 17 | 18 | ## Example: Enzo 19 | 20 | The be able to test JULEA's HDF5 plugins using real applications, [Enzo](https://enzo-project.org/) can be used. 21 | The following steps install Enzo via Spack and run one of its examples using JULEA's `julea-db` VOL plugin. 22 | 23 | ```console 24 | $ . scripts/environment.sh 25 | $ spack install enzo@main 26 | $ spack load enzo 27 | $ export HDF5_VOL_CONNECTOR=julea-db 28 | $ ./scripts/setup.sh start 29 | $ enzo -d "$(spack location -i enzo)/run/Hydro/Hydro-3D/CollapseTestNonCosmological/CollapseTestNonCosmological.enzo" 30 | $ ./scripts/setup.sh stop 31 | ``` 32 | -------------------------------------------------------------------------------- /fuse/read.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_read(char const* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | g_autoptr(JObject) object = NULL; 33 | guint64 bytes_read; 34 | 35 | (void)fi; 36 | 37 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 38 | kv = j_kv_new("posix", path); 39 | object = j_object_new("posix", path); 40 | 41 | j_object_read(object, buf, size, offset, &bytes_read, batch); 42 | 43 | if (j_batch_execute(batch)) 44 | { 45 | ret = bytes_read; 46 | } 47 | 48 | return ret; 49 | } 50 | -------------------------------------------------------------------------------- /include/core/jconnection-pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_CONNECTION_POOL_H 24 | #define JULEA_CONNECTION_POOL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | #include 32 | 33 | #include 34 | 35 | G_BEGIN_DECLS 36 | 37 | /** 38 | * \defgroup JConnectionPool Connection Pool 39 | * 40 | * Data structures and functions for managing connection pools. 41 | * 42 | * @{ 43 | **/ 44 | 45 | gpointer j_connection_pool_pop(JBackendType, guint32); 46 | void j_connection_pool_push(JBackendType, guint32, gpointer); 47 | 48 | /** 49 | * @} 50 | **/ 51 | 52 | G_END_DECLS 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /include/core/joperation-cache-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_OPERATION_CACHE_INTERNAL_H 24 | #define JULEA_OPERATION_CACHE_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | G_GNUC_INTERNAL void j_operation_cache_init(void); 37 | G_GNUC_INTERNAL void j_operation_cache_fini(void); 38 | 39 | /** 40 | * Flush the current cache of in-flight eventual consistency batches. 41 | */ 42 | G_GNUC_INTERNAL gboolean j_operation_cache_flush(void); 43 | 44 | G_GNUC_INTERNAL gboolean j_operation_cache_add(JBatch*); 45 | 46 | G_END_DECLS 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /server/server.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #ifndef JULEA_SERVER_H 20 | #define JULEA_SERVER_H 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | G_GNUC_INTERNAL extern JStatistics* jd_statistics; 32 | G_GNUC_INTERNAL extern GMutex jd_statistics_mutex[1]; 33 | 34 | G_GNUC_INTERNAL extern JBackend* jd_object_backend; 35 | G_GNUC_INTERNAL extern JBackend* jd_kv_backend; 36 | G_GNUC_INTERNAL extern JBackend* jd_db_backend; 37 | 38 | G_GNUC_INTERNAL extern JConfiguration* jd_configuration; 39 | 40 | G_GNUC_INTERNAL gboolean jd_handle_message(JMessage*, GSocketConnection*, JMemoryChunk*, guint64, JStatistics*); 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /include/core/jconfiguration-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2021-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_CONFIGURATION_INTERNAL_H 24 | #define JULEA_CONFIGURATION_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | /** 34 | * \addtogroup JConfiguration Configuration 35 | * 36 | * @{ 37 | **/ 38 | 39 | /** 40 | * Initializes the configuration. 41 | * 42 | * \private 43 | */ 44 | G_GNUC_INTERNAL void j_configuration_init(void); 45 | 46 | /** 47 | * Shuts down the configuration. 48 | * 49 | * \private 50 | */ 51 | G_GNUC_INTERNAL void j_configuration_fini(void); 52 | 53 | /** 54 | * @} 55 | **/ 56 | 57 | G_END_DECLS 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /scripts/benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2017-2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | SELF_BASE="${SELF_PATH##*/}" 24 | 25 | # shellcheck source=scripts/common 26 | . "${SELF_DIR}/common" 27 | # shellcheck source=scripts/spack 28 | . "${SELF_DIR}/spack" 29 | 30 | usage () 31 | { 32 | echo "Usage: ${SELF_BASE} [arguments]" 33 | exit 1 34 | } 35 | 36 | set_path 37 | set_library_path 38 | set_backend_path 39 | set_hdf_path 40 | 41 | run_benchmark () 42 | { 43 | local ret 44 | 45 | ret=0 46 | 47 | julea-benchmark "$@" || ret=$? 48 | 49 | return ${ret} 50 | } 51 | 52 | if test -z "${JULEA_SPACK_DIR}" 53 | then 54 | JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" 55 | fi 56 | 57 | spack_load_dependencies 58 | 59 | run_benchmark "$@" 60 | -------------------------------------------------------------------------------- /benchmark/cache.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "benchmark.h" 28 | 29 | static void 30 | benchmark_cache_get_release(BenchmarkRun* run) 31 | { 32 | guint const n = 100000; 33 | 34 | JCache* cache; 35 | 36 | cache = j_cache_new(n); 37 | 38 | j_benchmark_timer_start(run); 39 | 40 | while (j_benchmark_iterate(run)) 41 | { 42 | for (guint i = 0; i < n; i++) 43 | { 44 | gpointer buf; 45 | 46 | buf = j_cache_get(cache, 1); 47 | j_cache_release(cache, buf); 48 | } 49 | } 50 | 51 | j_benchmark_timer_stop(run); 52 | 53 | j_cache_free(cache); 54 | 55 | run->operations = n * 2; 56 | } 57 | 58 | void 59 | benchmark_cache(void) 60 | { 61 | j_benchmark_add("/cache/get-release", benchmark_cache_get_release); 62 | } 63 | -------------------------------------------------------------------------------- /lib/core/joperation.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | /** 32 | * \addtogroup JOperation Operation 33 | * 34 | * @{ 35 | **/ 36 | 37 | JOperation* 38 | j_operation_new(void) 39 | { 40 | J_TRACE_FUNCTION(NULL); 41 | 42 | JOperation* operation; 43 | 44 | operation = g_new(JOperation, 1); 45 | operation->key = NULL; 46 | operation->data = NULL; 47 | operation->exec_func = NULL; 48 | operation->free_func = NULL; 49 | 50 | return operation; 51 | } 52 | 53 | /// \todo 54 | void 55 | j_operation_free(JOperation* operation) 56 | { 57 | J_TRACE_FUNCTION(NULL); 58 | 59 | if (operation->free_func != NULL) 60 | { 61 | operation->free_func(operation->data); 62 | } 63 | 64 | g_free(operation); 65 | } 66 | 67 | /** 68 | * @} 69 | **/ 70 | -------------------------------------------------------------------------------- /benchmark/memory-chunk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "benchmark.h" 28 | 29 | static void 30 | benchmark_memory_chunk_get(BenchmarkRun* run) 31 | { 32 | guint const n = 10000000; 33 | 34 | JMemoryChunk* memory_chunk; 35 | 36 | memory_chunk = j_memory_chunk_new(n); 37 | 38 | j_benchmark_timer_start(run); 39 | 40 | while (j_benchmark_iterate(run)) 41 | { 42 | for (guint i = 0; i < n; i++) 43 | { 44 | j_memory_chunk_get(memory_chunk, 1); 45 | } 46 | 47 | j_memory_chunk_reset(memory_chunk); 48 | } 49 | 50 | j_benchmark_timer_stop(run); 51 | 52 | j_memory_chunk_free(memory_chunk); 53 | 54 | run->operations = n; 55 | } 56 | 57 | void 58 | benchmark_memory_chunk(void) 59 | { 60 | j_benchmark_add("/memory-chunk/get-reset", benchmark_memory_chunk_get); 61 | } 62 | -------------------------------------------------------------------------------- /fuse/julea-fuse.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | #include 26 | 27 | struct fuse_operations jfs_vtable = { 28 | .access = jfs_access, 29 | .chmod = jfs_chmod, 30 | .chown = jfs_chown, 31 | .create = jfs_create, 32 | .destroy = jfs_destroy, 33 | .getattr = jfs_getattr, 34 | .init = jfs_init, 35 | .mkdir = jfs_mkdir, 36 | .read = jfs_read, 37 | .readdir = jfs_readdir, 38 | .rmdir = jfs_rmdir, 39 | .truncate = jfs_truncate, 40 | .unlink = jfs_unlink, 41 | .utimens = jfs_utimens, 42 | .write = jfs_write, 43 | }; 44 | 45 | int 46 | main(int argc, char** argv) 47 | { 48 | gint ret; 49 | 50 | // Explicitly enable UTF-8 since functions such as g_format_size might return UTF-8 characters. 51 | setlocale(LC_ALL, "C.UTF-8"); 52 | 53 | ret = fuse_main(argc, argv, &jfs_vtable, NULL); 54 | 55 | return ret; 56 | } 57 | -------------------------------------------------------------------------------- /fuse/truncate.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_truncate(char const* path, off_t size, struct fuse_file_info* fi) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | gpointer value; 33 | guint32 len; 34 | 35 | (void)size; 36 | (void)fi; 37 | 38 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 39 | kv = j_kv_new("posix", path); 40 | 41 | j_kv_get(kv, &value, &len, batch); 42 | 43 | if (j_batch_execute(batch)) 44 | { 45 | bson_t file[1]; 46 | bson_iter_t iter; 47 | 48 | bson_init_static(file, value, len); 49 | 50 | if (bson_iter_init_find(&iter, file, "file") && bson_iter_type(&iter) == BSON_TYPE_BOOL && bson_iter_bool(&iter)) 51 | { 52 | /// \todo 53 | ret = 0; 54 | } 55 | 56 | bson_destroy(file); 57 | g_free(value); 58 | } 59 | 60 | return ret; 61 | } 62 | -------------------------------------------------------------------------------- /test/hdf5/hdf-helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2021 Timm Leon Erxleben 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include "test.h" 30 | 31 | #ifdef HAVE_HDF5 32 | 33 | #include 34 | 35 | #include 36 | 37 | void j_expect_vol_db_fail(void); 38 | void j_expect_vol_kv_fail(void); 39 | 40 | /** 41 | * Create a simple HDF5 file to test groups, datasets, ... 42 | * 43 | * \internal 44 | * 45 | * \param file File to be created. 46 | * \param udata User specified data. Will be ignored. 47 | **/ 48 | void j_test_hdf_file_fixture_setup(hid_t* file, gconstpointer udata); 49 | 50 | /** 51 | * Teardown function for \ref j_test_hdf_file_fixture_setup 52 | * 53 | * \internal 54 | * 55 | * \param file File to delete. 56 | * \param udata User specified data. Will be ignored. 57 | **/ 58 | void j_test_hdf_file_fixture_teardown(hid_t* file, gconstpointer udata); 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /fuse/mkdir.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_mkdir(char const* path, mode_t mode) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | bson_t* tmp; 33 | gpointer value; 34 | guint32 len; 35 | g_autofree gchar* basename = NULL; 36 | 37 | (void)mode; 38 | 39 | basename = g_path_get_basename(path); 40 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 41 | kv = j_kv_new("posix", path); 42 | 43 | tmp = bson_new(); 44 | 45 | bson_append_utf8(tmp, "name", -1, basename, -1); 46 | bson_append_bool(tmp, "file", -1, FALSE); 47 | bson_append_int64(tmp, "time", -1, g_get_real_time()); 48 | 49 | value = bson_destroy_with_steal(tmp, TRUE, &len); 50 | 51 | j_kv_put(kv, value, len, bson_free, batch); 52 | 53 | if (j_batch_execute(batch)) 54 | { 55 | ret = 0; 56 | } 57 | 58 | return ret; 59 | } 60 | -------------------------------------------------------------------------------- /scripts/ci/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | MODE="$1" 22 | DEPS="$2" 23 | 24 | GDBM_PREFIX='' 25 | 26 | . scripts/environment.sh 27 | 28 | if test "${DEPS}" = 'spack' 29 | then 30 | GDBM_PREFIX="-Dgdbm_prefix=$(spack location --install-dir gdbm)" 31 | fi 32 | 33 | case "${MODE}" in 34 | release) 35 | # shellcheck disable=SC2086 36 | meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror ${GDBM_PREFIX} bld 37 | ninja -C bld 38 | ninja -C bld install 39 | ;; 40 | debug) 41 | CLANG_LUNDEF='' 42 | if test "${CC}" = 'clang' 43 | then 44 | CLANG_LUNDEF='-Db_lundef=false' 45 | fi 46 | # shellcheck disable=SC2086 47 | meson setup -Db_sanitize=address,undefined ${CLANG_LUNDEF} ${GDBM_PREFIX} bld 48 | ninja -C bld 49 | ;; 50 | coverage) 51 | # shellcheck disable=SC2086 52 | meson setup -Db_coverage=true -Db_sanitize=address,undefined ${GDBM_PREFIX} bld 53 | ninja -C bld 54 | ;; 55 | *) 56 | exit 1 57 | ;; 58 | esac 59 | -------------------------------------------------------------------------------- /scripts/check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2019-2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | SELF_BASE="${SELF_PATH##*/}" 24 | 25 | # shellcheck source=scripts/common 26 | . "${SELF_DIR}/common" 27 | 28 | usage () 29 | { 30 | echo "Usage: ${SELF_BASE}" 31 | exit 1 32 | } 33 | 34 | check_shellcheck () 35 | { 36 | if ! command -v shellcheck > /dev/null 2>&1 37 | then 38 | printf '%s: shellcheck is not available.\n' "${SELF_BASE}" >&2 39 | exit 1 40 | fi 41 | } 42 | 43 | get_files () 44 | { 45 | git ls-files '*.sh' 46 | } 47 | 48 | check_shellcheck 49 | 50 | # Make sure to exit with an error if at least one file has problems 51 | get_files | ( ret=0 52 | while read -r file 53 | do 54 | shell_arg='' 55 | 56 | if test "${file##*/}" = 'environment.sh' 57 | then 58 | shell_arg='--shell=dash' 59 | fi 60 | 61 | shellcheck --external-sources --check-sourced ${shell_arg} "${file}" || ret=$? 62 | done 63 | exit ${ret} ) || exit $? 64 | -------------------------------------------------------------------------------- /include/core/jbackground-operation-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_BACKGROUND_OPERATION_INTERNAL_H 24 | #define JULEA_BACKGROUND_OPERATION_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * Initializes the background operation framework. 36 | * 37 | * \code 38 | * j_background_operation_init(); 39 | * \endcode 40 | **/ 41 | G_GNUC_INTERNAL void j_background_operation_init(guint count); 42 | 43 | /** 44 | * Shuts down the background operation framework. 45 | * 46 | * \code 47 | * j_background_operation_fini(); 48 | * \endcode 49 | **/ 50 | G_GNUC_INTERNAL void j_background_operation_fini(void); 51 | 52 | /** 53 | * Gets the number of threads used. 54 | **/ 55 | G_GNUC_INTERNAL guint j_background_operation_get_num_threads(void); 56 | 57 | G_END_DECLS 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/db/jdb-error.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2019 Benjamin Warnke 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DB_ERROR_H 24 | #define JULEA_DB_ERROR_H 25 | 26 | #if !defined(JULEA_DB_H) && !defined(JULEA_DB_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | #define J_DB_ERROR j_db_error_quark() 35 | 36 | enum JDBError 37 | { 38 | J_DB_ERROR_DUPLICATE_INDEX, 39 | J_DB_ERROR_ITERATOR_NO_MORE_ELEMENTS, 40 | J_DB_ERROR_MODE_INVALID, 41 | J_DB_ERROR_OPERATOR_INVALID, 42 | J_DB_ERROR_SCHEMA_INITIALIZED, 43 | J_DB_ERROR_SCHEMA_NOT_INITIALIZED, 44 | J_DB_ERROR_SCHEMA_SERVER, 45 | J_DB_ERROR_SELECTOR_EMPTY, 46 | J_DB_ERROR_SELECTOR_MUST_NOT_EQUAL, 47 | J_DB_ERROR_SELECTOR_TOO_COMPLEX, 48 | J_DB_ERROR_TYPE_INVALID, 49 | J_DB_ERROR_VARIABLE_ALREADY_SET, 50 | J_DB_ERROR_VARIABLE_NOT_FOUND 51 | }; 52 | 53 | typedef enum JDBError JDBError; 54 | 55 | GQuark j_db_error_quark(void); 56 | 57 | G_END_DECLS 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/db/jdb-type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2019 Benjamin Warnke 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DB_TYPE_H 24 | #define JULEA_DB_TYPE_H 25 | 26 | #if !defined(JULEA_DB_H) && !defined(JULEA_DB_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | enum JDBType 35 | { 36 | J_DB_TYPE_SINT32, 37 | J_DB_TYPE_UINT32, 38 | J_DB_TYPE_FLOAT32, 39 | J_DB_TYPE_SINT64, 40 | J_DB_TYPE_UINT64, 41 | J_DB_TYPE_FLOAT64, 42 | J_DB_TYPE_STRING, 43 | J_DB_TYPE_BLOB, 44 | J_DB_TYPE_ID 45 | }; 46 | 47 | typedef enum JDBType JDBType; 48 | 49 | union JDBTypeValue 50 | { 51 | guint32 val_uint32; 52 | gint32 val_sint32; 53 | guint64 val_uint64; 54 | gint64 val_sint64; 55 | gdouble val_float64; 56 | gfloat val_float32; 57 | gchar const* val_string; 58 | 59 | struct 60 | { 61 | gchar const* val_blob; 62 | guint32 val_blob_length; 63 | }; 64 | }; 65 | 66 | typedef union JDBTypeValue JDBTypeValue; 67 | 68 | G_END_DECLS 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /include/julea.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_H 24 | #define JULEA_H 25 | 26 | /** 27 | * JULEA is a distributed file system with a newly designed application programming interface. 28 | * It also allows to change the file system semantics at runtime. 29 | * 30 | * The name JULEA is a play on ROMIO. 31 | **/ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | 54 | #undef JULEA_H 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /include/core/jlist-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_LIST_INTERNAL_H 24 | #define JULEA_LIST_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \addtogroup JList List 36 | * 37 | * @{ 38 | **/ 39 | 40 | /** 41 | * A JList element. 42 | **/ 43 | struct JListElement 44 | { 45 | /** 46 | * Pointer to the next element. 47 | **/ 48 | struct JListElement* next; 49 | /** 50 | * Pointer to data. 51 | **/ 52 | gpointer data; 53 | }; 54 | 55 | typedef struct JListElement JListElement; 56 | 57 | G_END_DECLS 58 | 59 | #include 60 | 61 | G_BEGIN_DECLS 62 | 63 | /** 64 | * Returns the list's first element. 65 | * 66 | * \private 67 | * 68 | * \code 69 | * \endcode 70 | * 71 | * \param list A JList. 72 | * 73 | * \return A JListElement. 74 | **/ 75 | G_GNUC_INTERNAL JListElement* j_list_head(JList* list); 76 | 77 | /** 78 | * @} 79 | **/ 80 | 81 | G_END_DECLS 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2017-2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | SELF_BASE="${SELF_PATH##*/}" 24 | 25 | # shellcheck source=scripts/common 26 | . "${SELF_DIR}/common" 27 | # shellcheck source=scripts/spack 28 | . "${SELF_DIR}/spack" 29 | 30 | usage () 31 | { 32 | echo "Usage: ${SELF_BASE} [arguments]" 33 | exit 1 34 | } 35 | 36 | set_path 37 | set_library_path 38 | set_backend_path 39 | set_hdf_path 40 | 41 | run_test () 42 | { 43 | local build_dir 44 | local ret 45 | 46 | build_dir="$(get_directory "${SELF_DIR}/..")/bld" 47 | ret=0 48 | 49 | test -d "${build_dir}" || error_build_directory_not_found 50 | 51 | meson test -C "${build_dir}" --no-rebuild --print-errorlogs --verbose --test-args "$*" || ret=$? 52 | 53 | if test -n "${CI}" -a ${ret} -ne 0 54 | then 55 | journalctl --boot GLIB_DOMAIN=JULEA || true 56 | fi 57 | 58 | return ${ret} 59 | } 60 | 61 | if test -z "${JULEA_SPACK_DIR}" 62 | then 63 | JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" 64 | fi 65 | 66 | spack_load_dependencies 67 | 68 | run_test "$@" 69 | -------------------------------------------------------------------------------- /fuse/create.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_create(char const* path, mode_t mode, struct fuse_file_info* fi) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | g_autoptr(JObject) object = NULL; 33 | bson_t* tmp; 34 | gpointer value; 35 | guint32 len; 36 | g_autofree gchar* basename = NULL; 37 | 38 | (void)mode; 39 | (void)fi; 40 | 41 | basename = g_path_get_basename(path); 42 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 43 | kv = j_kv_new("posix", path); 44 | object = j_object_new("posix", path); 45 | 46 | tmp = bson_new(); 47 | 48 | bson_append_utf8(tmp, "name", -1, basename, -1); 49 | bson_append_bool(tmp, "file", -1, TRUE); 50 | bson_append_int64(tmp, "size", -1, 0); 51 | bson_append_int64(tmp, "time", -1, g_get_real_time()); 52 | 53 | value = bson_destroy_with_steal(tmp, TRUE, &len); 54 | 55 | j_kv_put(kv, value, len, bson_free, batch); 56 | j_object_create(object, batch); 57 | 58 | if (j_batch_execute(batch)) 59 | { 60 | ret = 0; 61 | } 62 | 63 | /// \todo does not return 0 on success 64 | return ret; 65 | } 66 | -------------------------------------------------------------------------------- /lib/core/distribution/distribution.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DISTRIBUTION_DISTRIBUTION_H 24 | #define JULEA_DISTRIBUTION_DISTRIBUTION_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | 36 | struct JDistributionVTable 37 | { 38 | gpointer (*distribution_new)(guint, guint64); 39 | void (*distribution_free)(gpointer); 40 | 41 | void (*distribution_set)(gpointer, gchar const*, guint64); 42 | void (*distribution_set2)(gpointer, gchar const*, guint64, guint64); 43 | 44 | void (*distribution_serialize)(gpointer, bson_t*); 45 | void (*distribution_deserialize)(gpointer, bson_t const*); 46 | 47 | void (*distribution_reset)(gpointer, guint64, guint64); 48 | gboolean (*distribution_distribute)(gpointer, guint*, guint64*, guint64*, guint64*); 49 | }; 50 | 51 | typedef struct JDistributionVTable JDistributionVTable; 52 | 53 | void j_distribution_round_robin_get_vtable(JDistributionVTable*); 54 | void j_distribution_single_server_get_vtable(JDistributionVTable*); 55 | void j_distribution_weighted_get_vtable(JDistributionVTable*); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /fuse/readdir.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | #include 25 | 26 | int 27 | jfs_readdir(char const* path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi, enum fuse_readdir_flags flags) 28 | { 29 | int ret = -ENOENT; 30 | 31 | JKVIterator* it; 32 | g_autofree gchar* prefix = NULL; 33 | 34 | (void)offset; 35 | (void)fi; 36 | (void)flags; 37 | 38 | if (g_strcmp0(path, "/") == 0) 39 | { 40 | prefix = g_strdup(path); 41 | } 42 | else 43 | { 44 | prefix = g_strdup_printf("%s/", path); 45 | } 46 | 47 | it = j_kv_iterator_new("posix", prefix); 48 | 49 | while (j_kv_iterator_next(it)) 50 | { 51 | gconstpointer value; 52 | guint32 len; 53 | bson_t tmp[1]; 54 | bson_iter_t iter; 55 | 56 | j_kv_iterator_get(it, &value, &len); 57 | bson_init_static(tmp, value, len); 58 | 59 | if (bson_iter_init_find(&iter, tmp, "name") && bson_iter_type(&iter) == BSON_TYPE_UTF8) 60 | { 61 | gchar const* name; 62 | 63 | name = bson_iter_utf8(&iter, NULL); 64 | filler(buf, name, NULL, 0, 0); 65 | } 66 | else 67 | { 68 | filler(buf, "???", NULL, 0, 0); 69 | } 70 | } 71 | 72 | j_kv_iterator_free(it); 73 | 74 | ret = 0; 75 | 76 | return ret; 77 | } 78 | -------------------------------------------------------------------------------- /fuse/julea-fuse.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #define FUSE_USE_VERSION 35 20 | #define _XOPEN_SOURCE 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | int jfs_access(char const*, int); 31 | int jfs_chmod(char const*, mode_t, struct fuse_file_info*); 32 | int jfs_chown(char const*, uid_t, gid_t, struct fuse_file_info*); 33 | int jfs_create(char const*, mode_t, struct fuse_file_info*); 34 | void jfs_destroy(void*); 35 | int jfs_getattr(char const*, struct stat*, struct fuse_file_info*); 36 | void* jfs_init(struct fuse_conn_info*, struct fuse_config*); 37 | int jfs_link(char const*, char const*); 38 | int jfs_mkdir(char const*, mode_t); 39 | int jfs_open(char const*, struct fuse_file_info*); 40 | int jfs_read(char const*, char*, size_t, off_t, struct fuse_file_info*); 41 | int jfs_readdir(char const*, void*, fuse_fill_dir_t, off_t, struct fuse_file_info*, enum fuse_readdir_flags); 42 | int jfs_rmdir(char const*); 43 | int jfs_statfs(char const*, struct statvfs*); 44 | int jfs_truncate(char const*, off_t, struct fuse_file_info*); 45 | int jfs_unlink(char const*); 46 | int jfs_utimens(char const*, const struct timespec[2], struct fuse_file_info*); 47 | int jfs_write(char const*, char const*, size_t, off_t, struct fuse_file_info*); 48 | -------------------------------------------------------------------------------- /benchmark/benchmark.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #ifndef JULEA_BENCHMARK_H 20 | #define JULEA_BENCHMARK_H 21 | 22 | #include 23 | 24 | struct BenchmarkRun 25 | { 26 | gchar* name; 27 | void (*func)(struct BenchmarkRun*); 28 | GTimer* timer; 29 | gboolean timer_started; 30 | guint iterations; 31 | guint64 operations; 32 | guint64 bytes; 33 | }; 34 | 35 | typedef struct BenchmarkRun BenchmarkRun; 36 | 37 | #include 38 | 39 | typedef void (*BenchmarkFunc)(BenchmarkRun*); 40 | 41 | JSemantics* j_benchmark_get_semantics(void); 42 | 43 | void j_benchmark_timer_start(BenchmarkRun*); 44 | void j_benchmark_timer_stop(BenchmarkRun*); 45 | 46 | gboolean j_benchmark_iterate(BenchmarkRun*); 47 | 48 | void j_benchmark_add(gchar const*, BenchmarkFunc); 49 | 50 | void benchmark_background_operation(void); 51 | void benchmark_cache(void); 52 | void benchmark_memory_chunk(void); 53 | void benchmark_message(void); 54 | 55 | void benchmark_kv(void); 56 | 57 | void benchmark_distributed_object(void); 58 | void benchmark_object(void); 59 | 60 | void benchmark_db_entry(void); 61 | void benchmark_db_iterator(void); 62 | void benchmark_db_schema(void); 63 | 64 | void benchmark_collection(void); 65 | void benchmark_item(void); 66 | 67 | void benchmark_hdf(void); 68 | void benchmark_hdf_dai(void); 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /include/core/jbatch-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_BATCH_INTERNAL_H 24 | #define JULEA_BATCH_INTERNAL_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | /** 37 | * \addtogroup JBatch Batch 38 | * 39 | * @{ 40 | **/ 41 | 42 | /** 43 | * Copies a batch. 44 | * 45 | * \private 46 | * 47 | * \code 48 | * \endcode 49 | * 50 | * \param old_batch A batch. 51 | * 52 | * \return A new batch. 53 | */ 54 | G_GNUC_INTERNAL JBatch* j_batch_new_from_batch(JBatch* batch); 55 | 56 | /** 57 | * Returns a batch's parts. 58 | * 59 | * \private 60 | * 61 | * \code 62 | * \endcode 63 | * 64 | * \param batch A batch. 65 | * 66 | * \return A list of parts. 67 | **/ 68 | G_GNUC_INTERNAL JList* j_batch_get_operations(JBatch* batch); 69 | 70 | /** 71 | * Executes the batch. 72 | * 73 | * \private 74 | * 75 | * \code 76 | * \endcode 77 | * 78 | * \param batch A batch. 79 | * 80 | * \return TRUE on success, FALSE if an error occurred. 81 | **/ 82 | G_GNUC_INTERNAL gboolean j_batch_execute_internal(JBatch* batch); 83 | 84 | /** 85 | * @} 86 | **/ 87 | 88 | G_END_DECLS 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /test/hdf5/hdf-helper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2021 Timm Leon Erxleben 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include "hdf-helper.h" 24 | 25 | #ifdef HAVE_HDF5 26 | 27 | void 28 | j_expect_vol_db_fail(void) 29 | { 30 | const gchar* conn_name = NULL; 31 | conn_name = g_getenv("HDF5_VOL_CONNECTOR"); 32 | 33 | if (!g_strcmp0(conn_name, "julea-db")) 34 | { 35 | J_TEST_EXPECT_FAIL("This test is currently expected to fail for the DB VOL plugin."); 36 | } 37 | } 38 | 39 | void 40 | j_expect_vol_kv_fail(void) 41 | { 42 | const gchar* conn_name = NULL; 43 | conn_name = g_getenv("HDF5_VOL_CONNECTOR"); 44 | 45 | if (!g_strcmp0(conn_name, "julea-kv")) 46 | { 47 | J_TEST_EXPECT_FAIL("This test is currently expected to fail for the KV VOL plugin."); 48 | } 49 | } 50 | 51 | void 52 | j_test_hdf_file_fixture_setup(hid_t* file, gconstpointer udata) 53 | { 54 | if (g_test_subprocess()) 55 | { 56 | const gchar* name = udata; 57 | 58 | *file = H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); 59 | } 60 | } 61 | 62 | void 63 | j_test_hdf_file_fixture_teardown(hid_t* file, gconstpointer udata) 64 | { 65 | if (g_test_subprocess()) 66 | { 67 | (void)udata; 68 | H5Fclose(*file); 69 | /// \todo Delete is not yet implemented for KV version 70 | // const gchar* name = udata; 71 | // H5Fdelete(name, H5P_DEFAULT); 72 | } 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /include/core/joperation.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_OPERATION_H 24 | #define JULEA_OPERATION_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | 35 | G_BEGIN_DECLS 36 | 37 | /** 38 | * \defgroup JOperation Operation 39 | * 40 | * @{ 41 | **/ 42 | 43 | typedef gboolean (*JOperationExecFunc)(JList*, JSemantics*); 44 | typedef void (*JOperationFreeFunc)(gpointer); 45 | 46 | /** 47 | * An operation. 48 | **/ 49 | struct JOperation 50 | { 51 | gconstpointer key; 52 | gpointer data; 53 | 54 | JOperationExecFunc exec_func; 55 | JOperationFreeFunc free_func; 56 | }; 57 | 58 | typedef struct JOperation JOperation; 59 | 60 | /** 61 | * Creates a new operation. 62 | * 63 | * \code 64 | * \endcode 65 | * 66 | * \return A new operation. Should be freed with j_operation_free(). 67 | **/ 68 | JOperation* j_operation_new(void); 69 | 70 | /** 71 | * Frees the memory allocated by an operation. 72 | * 73 | * \private 74 | * 75 | * \code 76 | * \endcode 77 | * 78 | * \param data An operation. 79 | **/ 80 | void j_operation_free(JOperation*); 81 | 82 | /** 83 | * @} 84 | **/ 85 | 86 | G_END_DECLS 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /.github/workflows/containers.yml: -------------------------------------------------------------------------------- 1 | # JULEA - Flexible storage framework 2 | # Copyright (C) 2023-2024 Michael Kuhn 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Lesser General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Lesser General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this program. If not, see . 16 | 17 | name: Containers 18 | on: 19 | push: 20 | branches: 21 | - master 22 | paths: 23 | - '.github/workflows/containers.yml' 24 | - 'containers/*' 25 | pull_request: 26 | branches: 27 | - master 28 | paths: 29 | - '.github/workflows/containers.yml' 30 | - 'containers/*' 31 | schedule: 32 | - cron: '0 0 * * 0' 33 | jobs: 34 | ubuntu-dev: 35 | name: Ubuntu Development Container 36 | runs-on: ubuntu-24.04 37 | permissions: 38 | packages: write 39 | timeout-minutes: 60 40 | steps: 41 | - name: Checkout 42 | uses: actions/checkout@v5 43 | with: 44 | persist-credentials: false 45 | show-progress: false 46 | - name: Login to GitHub Container Registry 47 | if: ${{ github.event_name != 'pull_request' }} 48 | uses: docker/login-action@v3 49 | with: 50 | registry: ghcr.io 51 | username: ${{ github.actor }} 52 | password: ${{ secrets.GITHUB_TOKEN }} 53 | - name: Build and Push to GitHub Container Registry 54 | uses: docker/build-push-action@v6 55 | with: 56 | context: containers 57 | file: containers/ubuntu-22.04-dev 58 | push: ${{ github.event_name != 'pull_request' }} 59 | tags: ghcr.io/parcio/ubuntu-dev:22.04 60 | -------------------------------------------------------------------------------- /doc/debugging.md: -------------------------------------------------------------------------------- 1 | # Debugging 2 | 3 | There are several ways to debug JULEA. 4 | Moreover, developers should be aware of the fact that debug builds of JULEA can behave differently than release builds: 5 | 1. The `JULEA_BACKEND_PATH` environment variable can be used to override JULEA's internal backend path only in debug builds. 6 | 2. Tracing via `JULEA_TRACE` is only available in debug builds. 7 | 8 | ## Debug Output 9 | 10 | Debug prints from `g_debug()` can be enabled by setting the environment variable `G_MESSAGES_DEBUG` to `JULEA`. 11 | 12 | ## Tracing 13 | 14 | JULEA contains a tracing component that can be used to record information about various aspects of JULEA's behavior. 15 | The overall tracing can be influenced using the `JULEA_TRACE` environment variable. 16 | If the variable is set to `echo`, all tracing information will be printed to stderr. 17 | If JULEA has been built with OTF support, a value of `otf` will cause JULEA to produce traces via OTF. 18 | It is also possible to specify multiple values separated by commas. 19 | 20 | By default, all functions are traced. 21 | If this produces too much output, a filter can be set using the `JULEA_TRACE_FUNCTION` environment variable. 22 | The variable can contain a list of function wildcards that are separated by commas. 23 | The wildcards support `*` and `?`. 24 | 25 | ## Coverage 26 | 27 | Generating a coverage report requires the `gcovr` tool to be installed. 28 | If it is not available on your system, you can install it as an [additional dependency](dependencies.md#additional-dependencies). 29 | The following steps set up everything to generate a coverage report: 30 | 31 | ```console 32 | $ . scripts/environment.sh 33 | $ spack install py-gcovr 34 | $ spack load py-gcovr 35 | $ meson setup -Db_coverage=true bld 36 | $ ninja -C bld 37 | $ ./scripts/setup.sh start 38 | $ ./scripts/test.sh 39 | $ ./scripts/setup.sh stop 40 | $ ninja -C bld coverage 41 | ``` 42 | 43 | ## Tests 44 | 45 | By default, tests that are marked as expected fail will not cause the test suite to fail as a whole. 46 | While debugging, it sometimes makes sense to disable this behavior, which can be achieved by setting the `JULEA_TEST_FATAL_EXPECTED_FAIL` environment variable. 47 | -------------------------------------------------------------------------------- /benchmark/background-operation.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "benchmark.h" 28 | 29 | gint benchmark_background_operation_counter; 30 | 31 | static gpointer 32 | on_background_operation_completed(gpointer data) 33 | { 34 | (void)data; 35 | 36 | g_atomic_int_inc(&benchmark_background_operation_counter); 37 | 38 | return NULL; 39 | } 40 | 41 | static void 42 | benchmark_background_operation_new_ref_unref(BenchmarkRun* run) 43 | { 44 | guint const n = 100000; 45 | 46 | JBackgroundOperation* background_operation; 47 | 48 | j_benchmark_timer_start(run); 49 | 50 | while (j_benchmark_iterate(run)) 51 | { 52 | g_atomic_int_set(&benchmark_background_operation_counter, 0); 53 | 54 | for (guint i = 0; i < n; i++) 55 | { 56 | background_operation = j_background_operation_new(on_background_operation_completed, NULL); 57 | j_background_operation_unref(background_operation); 58 | } 59 | 60 | /// \todo overhead? 61 | while ((guint64)g_atomic_int_get(&benchmark_background_operation_counter) != n) 62 | { 63 | } 64 | } 65 | 66 | j_benchmark_timer_stop(run); 67 | 68 | run->operations = n; 69 | } 70 | 71 | void 72 | benchmark_background_operation(void) 73 | { 74 | j_benchmark_add("/background-operation/new", benchmark_background_operation_new_ref_unref); 75 | } 76 | -------------------------------------------------------------------------------- /test/core/background-operation.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "test.h" 28 | 29 | static gpointer 30 | on_background_operation_completed(gpointer data) 31 | { 32 | (void)data; 33 | 34 | return NULL; 35 | } 36 | 37 | static void 38 | test_background_operation_new_ref_unref(void) 39 | { 40 | JBackgroundOperation* background_operation; 41 | 42 | J_TEST_TRAP_START; 43 | background_operation = j_background_operation_new(on_background_operation_completed, NULL); 44 | j_background_operation_ref(background_operation); 45 | j_background_operation_unref(background_operation); 46 | j_background_operation_unref(background_operation); 47 | J_TEST_TRAP_END; 48 | } 49 | 50 | static void 51 | test_background_operation_wait(void) 52 | { 53 | JBackgroundOperation* background_operation; 54 | 55 | J_TEST_TRAP_START; 56 | background_operation = j_background_operation_new(on_background_operation_completed, NULL); 57 | 58 | j_background_operation_wait(background_operation); 59 | j_background_operation_unref(background_operation); 60 | J_TEST_TRAP_END; 61 | } 62 | 63 | void 64 | test_core_background_operation(void) 65 | { 66 | g_test_add_func("/core/background_operation/new_ref_unref", test_background_operation_new_ref_unref); 67 | g_test_add_func("/core/background_operation/wait", test_background_operation_wait); 68 | } 69 | -------------------------------------------------------------------------------- /.github/workflows/dependencies.yml: -------------------------------------------------------------------------------- 1 | # JULEA - Flexible storage framework 2 | # Copyright (C) 2021-2024 Michael Kuhn 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Lesser General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Lesser General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this program. If not, see . 16 | 17 | name: Dependencies 18 | on: 19 | schedule: 20 | - cron: '0 0 * * 0' 21 | jobs: 22 | dependencies: 23 | name: Dependencies 24 | runs-on: ${{ matrix.os.dist }} 25 | timeout-minutes: 180 26 | strategy: 27 | fail-fast: false 28 | matrix: 29 | os: 30 | - dist: ubuntu-24.04 31 | compiler: gcc 32 | compiler_spack: gcc 33 | compiler_version: 13.3.0 34 | - dist: ubuntu-24.04 35 | compiler: clang 36 | compiler_spack: llvm 37 | compiler_version: 18.1.3 38 | - dist: ubuntu-22.04 39 | compiler: gcc 40 | compiler_spack: gcc 41 | compiler_version: 11.4.0 42 | - dist: ubuntu-22.04 43 | compiler: clang 44 | compiler_spack: llvm 45 | compiler_version: 14.0.0 46 | steps: 47 | - name: Checkout 48 | uses: actions/checkout@v5 49 | with: 50 | persist-credentials: false 51 | show-progress: false 52 | - name: Cache dependencies 53 | uses: actions/cache@v4 54 | with: 55 | path: dependencies 56 | key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} 57 | - name: Install dependencies 58 | env: 59 | JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }} 60 | run: | 61 | rm --recursive --force dependencies 62 | ./scripts/install-dependencies.sh 63 | -------------------------------------------------------------------------------- /scripts/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2019-2024 Michael Kuhn 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with this program. If not, see . 18 | 19 | set -e 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | SELF_BASE="${SELF_PATH##*/}" 24 | 25 | # shellcheck source=scripts/common 26 | . "${SELF_DIR}/common" 27 | 28 | usage () 29 | { 30 | echo "Usage: ${SELF_BASE} [-f]" 31 | exit 1 32 | } 33 | 34 | check_clang_format () 35 | { 36 | local output 37 | local ret 38 | 39 | if ! command -v clang-format > /dev/null 2>&1 40 | then 41 | printf '%s: clang-format is not available.\n' "${SELF_BASE}" >&2 42 | exit 1 43 | fi 44 | 45 | ret=0 46 | output="$(printf 'int main (void) { return 0; }\n' | clang-format --style=file 2>&1)" || ret=$? 47 | 48 | if test ${ret} -ne 0 49 | then 50 | printf '%s: clang-format does not work correctly:\n%s\n' "${SELF_BASE}" "${output}" >&2 51 | exit 1 52 | fi 53 | } 54 | 55 | get_files () 56 | { 57 | git ls-files '*.c' '*.h' 58 | } 59 | 60 | MODE='diff' 61 | 62 | test -n "$1" -a "$1" = '-f' && MODE='format' 63 | 64 | check_clang_format 65 | 66 | case "${MODE}" in 67 | diff) 68 | # Make sure to exit with an error if at least one file differs 69 | get_files | ( ret=0 70 | while read -r file 71 | do 72 | diff --color --unified "${file}" <(clang-format --style=file "${file}") || ret=$? 73 | diff --color --unified "${file}" <(sed -E 's/[[:blank:]]+$//' "${file}") || ret=$? 74 | done 75 | exit ${ret} ) || exit $? 76 | ;; 77 | format) 78 | get_files | while read -r file 79 | do 80 | clang-format -i --style=file "${file}" 81 | sed -E --in-place 's/[[:blank:]]+$//' "${file}" 82 | done 83 | ;; 84 | *) 85 | usage 86 | ;; 87 | esac 88 | -------------------------------------------------------------------------------- /lib/item/jcollection-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | /** 37 | * \addtogroup JCollectionIterator 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JCollectionIterator 43 | { 44 | JKVIterator* iterator; 45 | }; 46 | 47 | JCollectionIterator* 48 | j_collection_iterator_new(void) 49 | { 50 | JCollectionIterator* iterator; 51 | 52 | iterator = g_new(JCollectionIterator, 1); 53 | iterator->iterator = j_kv_iterator_new("collections", NULL); 54 | 55 | return iterator; 56 | } 57 | 58 | void 59 | j_collection_iterator_free(JCollectionIterator* iterator) 60 | { 61 | g_return_if_fail(iterator != NULL); 62 | 63 | j_kv_iterator_free(iterator->iterator); 64 | 65 | g_free(iterator); 66 | } 67 | 68 | gboolean 69 | j_collection_iterator_next(JCollectionIterator* iterator) 70 | { 71 | g_return_val_if_fail(iterator != NULL, FALSE); 72 | 73 | return j_kv_iterator_next(iterator->iterator); 74 | } 75 | 76 | JCollection* 77 | j_collection_iterator_get(JCollectionIterator* iterator) 78 | { 79 | JCollection* collection; 80 | bson_t tmp[1]; 81 | gconstpointer value; 82 | guint32 len; 83 | 84 | g_return_val_if_fail(iterator != NULL, NULL); 85 | 86 | j_kv_iterator_get(iterator->iterator, &value, &len); 87 | bson_init_static(tmp, value, len); 88 | collection = j_collection_new_from_bson(tmp); 89 | 90 | return collection; 91 | } 92 | 93 | /** 94 | * @} 95 | **/ 96 | -------------------------------------------------------------------------------- /fuse/write.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | 25 | int 26 | jfs_write(char const* path, char const* buf, size_t size, off_t offset, struct fuse_file_info* fi) 27 | { 28 | int ret = -ENOENT; 29 | 30 | g_autoptr(JBatch) batch = NULL; 31 | g_autoptr(JKV) kv = NULL; 32 | g_autoptr(JObject) object = NULL; 33 | guint64 bytes_written; 34 | gpointer value; 35 | guint32 len; 36 | 37 | (void)fi; 38 | 39 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 40 | kv = j_kv_new("posix", path); 41 | object = j_object_new("posix", path); 42 | 43 | j_kv_get(kv, &value, &len, batch); 44 | j_object_write(object, buf, size, offset, &bytes_written, batch); 45 | 46 | if (j_batch_execute(batch)) 47 | { 48 | bson_t file[1]; 49 | bson_iter_t iter; 50 | gint64 old_size = 0; 51 | 52 | ret = bytes_written; 53 | 54 | bson_init_static(file, value, len); 55 | 56 | if (bson_iter_init_find(&iter, file, "size") && bson_iter_type(&iter) == BSON_TYPE_INT64) 57 | { 58 | old_size = bson_iter_int64(&iter); 59 | 60 | if ((guint64)old_size < offset + size) 61 | { 62 | gpointer copy; 63 | 64 | bson_iter_overwrite_int64(&iter, offset + size); 65 | 66 | #if GLIB_CHECK_VERSION(2, 68, 0) 67 | copy = g_memdup2(bson_get_data(file), file->len); 68 | #else 69 | copy = g_memdup(bson_get_data(file), file->len); 70 | #endif 71 | 72 | j_kv_put(kv, copy, file->len, g_free, batch); 73 | 74 | if (!j_batch_execute(batch)) 75 | { 76 | ret = -EIO; 77 | } 78 | } 79 | } 80 | 81 | bson_destroy(file); 82 | g_free(value); 83 | } 84 | 85 | return ret; 86 | } 87 | -------------------------------------------------------------------------------- /cli/list.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "cli.h" 22 | 23 | gboolean 24 | j_cmd_list(gchar const** arguments) 25 | { 26 | gboolean ret = TRUE; 27 | g_autoptr(JURI) uri = NULL; 28 | GError* error = NULL; 29 | 30 | if (j_cmd_arguments_length(arguments) != 1) 31 | { 32 | ret = FALSE; 33 | j_cmd_usage(); 34 | goto end; 35 | } 36 | 37 | if ((uri = j_uri_new(arguments[0])) == NULL) 38 | { 39 | ret = FALSE; 40 | g_print("Error: Invalid argument “%s”.\n", arguments[0]); 41 | goto end; 42 | } 43 | 44 | if (!j_uri_get(uri, &error)) 45 | { 46 | ret = FALSE; 47 | g_print("Error: %s\n", error->message); 48 | g_error_free(error); 49 | goto end; 50 | } 51 | 52 | if (j_uri_get_item(uri) != NULL) 53 | { 54 | ret = FALSE; 55 | j_cmd_usage(); 56 | } 57 | else if (j_uri_get_collection(uri) != NULL) 58 | { 59 | JItemIterator* iterator; 60 | 61 | iterator = j_item_iterator_new(j_uri_get_collection(uri)); 62 | 63 | while (j_item_iterator_next(iterator)) 64 | { 65 | JItem* item_ = j_item_iterator_get(iterator); 66 | 67 | g_print("%s\n", j_item_get_name(item_)); 68 | 69 | j_item_unref(item_); 70 | } 71 | 72 | j_item_iterator_free(iterator); 73 | } 74 | else 75 | { 76 | JCollectionIterator* iterator; 77 | 78 | iterator = j_collection_iterator_new(); 79 | 80 | while (j_collection_iterator_next(iterator)) 81 | { 82 | JCollection* collection_ = j_collection_iterator_get(iterator); 83 | 84 | g_print("%s\n", j_collection_get_name(collection_)); 85 | 86 | j_collection_unref(collection_); 87 | } 88 | 89 | j_collection_iterator_free(iterator); 90 | } 91 | 92 | end: 93 | return ret; 94 | } 95 | -------------------------------------------------------------------------------- /test/item/collection.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "test.h" 27 | 28 | static void 29 | test_collection_fixture_setup(JCollection** collection, gconstpointer data) 30 | { 31 | g_autoptr(JBatch) batch = NULL; 32 | 33 | (void)data; 34 | 35 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 36 | *collection = j_collection_create("test-collection", batch); 37 | } 38 | 39 | static void 40 | test_collection_fixture_teardown(JCollection** collection, gconstpointer data) 41 | { 42 | (void)data; 43 | 44 | j_collection_unref(*collection); 45 | } 46 | 47 | static void 48 | test_collection_new_free(void) 49 | { 50 | guint const n = 100000; 51 | 52 | J_TEST_TRAP_START; 53 | for (guint i = 0; i < n; i++) 54 | { 55 | g_autoptr(JBatch) batch = NULL; 56 | g_autoptr(JCollection) collection = NULL; 57 | 58 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 59 | collection = j_collection_create("test-collection", batch); 60 | 61 | g_assert_true(collection != NULL); 62 | } 63 | J_TEST_TRAP_END; 64 | } 65 | 66 | static void 67 | test_collection_name(JCollection** collection, gconstpointer data) 68 | { 69 | (void)data; 70 | 71 | J_TEST_TRAP_START; 72 | g_assert_cmpstr(j_collection_get_name(*collection), ==, "test-collection"); 73 | J_TEST_TRAP_END; 74 | } 75 | 76 | void 77 | test_item_collection(void) 78 | { 79 | g_test_add_func("/item/collection/new_free", test_collection_new_free); 80 | g_test_add("/item/collection/name", JCollection*, NULL, test_collection_fixture_setup, test_collection_name, test_collection_fixture_teardown); 81 | } 82 | -------------------------------------------------------------------------------- /test/test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #ifndef JULEA_TEST_T 20 | #define JULEA_TEST_T 21 | 22 | #define J_TEST_TRAP_START \ 23 | if (g_test_subprocess()) \ 24 | { 25 | #define J_TEST_TRAP_END \ 26 | } \ 27 | else \ 28 | { \ 29 | g_test_trap_subprocess(NULL, 0, G_TEST_SUBPROCESS_INHERIT_STDIN | G_TEST_SUBPROCESS_INHERIT_STDOUT | G_TEST_SUBPROCESS_INHERIT_STDERR); \ 30 | g_test_trap_assert_passed(); \ 31 | } 32 | 33 | #define J_TEST_EXPECT_FAIL(msg) \ 34 | if (!g_getenv("JULEA_TEST_FATAL_EXPECTED_FAIL")) \ 35 | { \ 36 | g_test_incomplete(msg); \ 37 | } 38 | 39 | void test_core_background_operation(void); 40 | void test_core_batch(void); 41 | void test_core_cache(void); 42 | void test_core_configuration(void); 43 | void test_core_credentials(void); 44 | void test_core_dir_iterator(void); 45 | void test_core_distribution(void); 46 | void test_core_list(void); 47 | void test_core_list_iterator(void); 48 | void test_core_memory_chunk(void); 49 | void test_core_message(void); 50 | void test_core_semantics(void); 51 | 52 | void test_object_distributed_object(void); 53 | void test_object_object(void); 54 | void test_object_object_iterator(void); 55 | 56 | void test_kv_kv(void); 57 | void test_kv_kv_iterator(void); 58 | void test_kv_parallel(void); 59 | 60 | void test_db_db(void); 61 | 62 | void test_item_collection(void); 63 | void test_item_collection_iterator(void); 64 | void test_item_item(void); 65 | void test_item_item_iterator(void); 66 | void test_item_uri(void); 67 | 68 | void test_hdf_hdf(void); 69 | void test_hdf_attribute(void); 70 | void test_hdf_dataset(void); 71 | void test_hdf_datatype(void); 72 | void test_hdf_file(void); 73 | void test_hdf_group_link(void); 74 | void test_hdf_object(void); 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /test/core/cache.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "test.h" 28 | 29 | static void 30 | test_cache_new_free(void) 31 | { 32 | JCache* cache; 33 | 34 | J_TEST_TRAP_START; 35 | cache = j_cache_new(42); 36 | g_assert_true(cache != NULL); 37 | 38 | j_cache_free(cache); 39 | J_TEST_TRAP_END; 40 | } 41 | 42 | static void 43 | test_cache_get(void) 44 | { 45 | JCache* cache; 46 | gpointer ret; 47 | 48 | J_TEST_TRAP_START; 49 | cache = j_cache_new(3); 50 | 51 | ret = j_cache_get(cache, 1); 52 | g_assert_true(ret != NULL); 53 | ret = j_cache_get(cache, 1); 54 | g_assert_true(ret != NULL); 55 | ret = j_cache_get(cache, 1); 56 | g_assert_true(ret != NULL); 57 | ret = j_cache_get(cache, 1); 58 | g_assert_true(ret == NULL); 59 | 60 | j_cache_free(cache); 61 | J_TEST_TRAP_END; 62 | } 63 | 64 | static void 65 | test_cache_release(void) 66 | { 67 | JCache* cache; 68 | gpointer ret1; 69 | gpointer ret2; 70 | 71 | J_TEST_TRAP_START; 72 | cache = j_cache_new(1); 73 | 74 | ret1 = j_cache_get(cache, 1); 75 | g_assert_true(ret1 != NULL); 76 | ret2 = j_cache_get(cache, 1); 77 | g_assert_true(ret2 == NULL); 78 | 79 | j_cache_release(cache, ret1); 80 | 81 | ret1 = j_cache_get(cache, 1); 82 | g_assert_true(ret1 != NULL); 83 | ret2 = j_cache_get(cache, 1); 84 | g_assert_true(ret2 == NULL); 85 | 86 | j_cache_free(cache); 87 | J_TEST_TRAP_END; 88 | } 89 | 90 | void 91 | test_core_cache(void) 92 | { 93 | g_test_add_func("/core/cache/new_free", test_cache_new_free); 94 | g_test_add_func("/core/cache/get", test_cache_get); 95 | g_test_add_func("/core/cache/release", test_cache_release); 96 | } 97 | -------------------------------------------------------------------------------- /test/test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "test.h" 28 | 29 | int 30 | main(int argc, char** argv) 31 | { 32 | gint ret; 33 | 34 | // Explicitly enable UTF-8 since functions such as g_format_size might return UTF-8 characters. 35 | setlocale(LC_ALL, "C.UTF-8"); 36 | 37 | g_test_init(&argc, &argv, NULL); 38 | 39 | // Failing assertions will not abort all coming tests but call g_test_fail and continue 40 | // This is necessary because we call g_test_trap_assert_passed() in J_TEST_TRAP_END 41 | g_test_set_nonfatal_assertions(); 42 | 43 | // Core 44 | test_core_background_operation(); 45 | test_core_batch(); 46 | test_core_cache(); 47 | test_core_configuration(); 48 | test_core_credentials(); 49 | test_core_dir_iterator(); 50 | test_core_distribution(); 51 | test_core_list(); 52 | test_core_list_iterator(); 53 | test_core_memory_chunk(); 54 | test_core_message(); 55 | test_core_semantics(); 56 | 57 | // Object client 58 | test_object_distributed_object(); 59 | test_object_object(); 60 | test_object_object_iterator(); 61 | 62 | // KV client 63 | test_kv_kv(); 64 | test_kv_kv_iterator(); 65 | test_kv_parallel(); 66 | 67 | // DB client 68 | test_db_db(); 69 | 70 | // Item client 71 | test_item_collection(); 72 | test_item_collection_iterator(); 73 | test_item_item(); 74 | test_item_item_iterator(); 75 | test_item_uri(); 76 | 77 | // HDF5 client 78 | test_hdf_hdf(); 79 | test_hdf_attribute(); 80 | test_hdf_dataset(); 81 | test_hdf_datatype(); 82 | test_hdf_file(); 83 | test_hdf_group_link(); 84 | test_hdf_object(); 85 | 86 | ret = g_test_run(); 87 | 88 | return ret; 89 | } 90 | -------------------------------------------------------------------------------- /include/core/jlist-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_LIST_ITERATOR_H 24 | #define JULEA_LIST_ITERATOR_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JListIterator List Iterator 36 | * 37 | * @{ 38 | **/ 39 | 40 | struct JListIterator; 41 | 42 | typedef struct JListIterator JListIterator; 43 | 44 | G_END_DECLS 45 | 46 | #include 47 | 48 | G_BEGIN_DECLS 49 | 50 | /** 51 | * Creates a new list iterator. 52 | * 53 | * \code 54 | * \endcode 55 | * 56 | * \param list A list. 57 | * 58 | * \return A new list iterator. 59 | **/ 60 | JListIterator* j_list_iterator_new(JList* list); 61 | 62 | /** 63 | * Frees the memory allocated by a list iterator. 64 | * 65 | * \code 66 | * \endcode 67 | * 68 | * \param iterator A list iterator. 69 | **/ 70 | void j_list_iterator_free(JListIterator* iterator); 71 | 72 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JListIterator, j_list_iterator_free) 73 | 74 | /** 75 | * Checks whether another list element is available. 76 | * 77 | * \code 78 | * \endcode 79 | * 80 | * \param iterator A list iterator. 81 | * 82 | * \return TRUE on success, FALSE if the end of the list is reached. 83 | **/ 84 | gboolean j_list_iterator_next(JListIterator* iterator); 85 | 86 | /** 87 | * Returns the current list element. 88 | * 89 | * \code 90 | * \endcode 91 | * 92 | * \param iterator A list iterator. 93 | * 94 | * \return A list element. 95 | **/ 96 | gpointer j_list_iterator_get(JListIterator* iterator); 97 | 98 | /** 99 | * @} 100 | **/ 101 | 102 | G_END_DECLS 103 | 104 | #endif 105 | -------------------------------------------------------------------------------- /scripts/environment.sh: -------------------------------------------------------------------------------- 1 | # JULEA - Flexible storage framework 2 | # Copyright (C) 2017-2024 Michael Kuhn 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Lesser General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Lesser General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Lesser General Public License 15 | # along with this program. If not, see . 16 | 17 | SELF_ZERO="$0" 18 | # shellcheck disable=SC2169,SC3028,SC3054 19 | test -n "${BASH_VERSION}" && SELF_ZERO="${BASH_SOURCE[0]}" 20 | 21 | SELF_PATH="$(readlink --canonicalize-existing -- "${SELF_ZERO}")" 22 | SELF_DIR="${SELF_PATH%/*}" 23 | # shellcheck disable=SC2034 24 | SELF_BASE="${SELF_PATH##*/}" 25 | 26 | # shellcheck source=scripts/common 27 | . "${SELF_DIR}/common" 28 | # shellcheck source=scripts/spack 29 | . "${SELF_DIR}/spack" 30 | 31 | JULEA_ENVIRONMENT_SOURCED=1 32 | 33 | # See https://stackoverflow.com/a/28776166 34 | if test -n "${BASH_VERSION}" 35 | then 36 | (return 0 2>/dev/null) || JULEA_ENVIRONMENT_SOURCED=0 37 | elif test -n "${ZSH_EVAL_CONTEXT}" 38 | then 39 | case "${ZSH_EVAL_CONTEXT}" in 40 | *:file) 41 | ;; 42 | *) 43 | JULEA_ENVIRONMENT_SOURCED=0 44 | ;; 45 | esac 46 | fi 47 | 48 | if test "${JULEA_ENVIRONMENT_SOURCED}" -eq 0 49 | then 50 | printf 'Warning: This script should be sourced using ". %s", otherwise changes to the environment will not persist.\n' "${SELF_PATH}" >&2 51 | fi 52 | 53 | JULEA_ENVIRONMENT=1 54 | 55 | set_path 56 | set_library_path 57 | set_pkg_config_path 58 | set_backend_path 59 | set_hdf_path 60 | 61 | if test -z "${JULEA_SPACK_DIR}" 62 | then 63 | JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" 64 | fi 65 | 66 | spack_load_dependencies 67 | 68 | # Do not filter out paths contained in CPATH and LIBRARY_PATH. 69 | PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 70 | PKG_CONFIG_ALLOW_SYSTEM_LIBS=1 71 | 72 | export PKG_CONFIG_ALLOW_SYSTEM_CFLAGS 73 | export PKG_CONFIG_ALLOW_SYSTEM_LIBS 74 | 75 | # FIXME The Spack pkg-config does not search in global directories and Meson does not provide a way to override this 76 | PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/lib64/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig" 77 | 78 | export PKG_CONFIG_PATH 79 | -------------------------------------------------------------------------------- /include/core/jdir-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2021-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_DIR_ITERATOR_H 24 | #define JULEA_DIR_ITERATOR_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JDirIterator Directory Iterator 36 | * 37 | * @{ 38 | **/ 39 | 40 | struct JDirIterator; 41 | 42 | typedef struct JDirIterator JDirIterator; 43 | 44 | G_END_DECLS 45 | 46 | #include 47 | 48 | G_BEGIN_DECLS 49 | 50 | /** 51 | * Creates a new directory iterator. 52 | * 53 | * \code 54 | * \endcode 55 | * 56 | * \param path A directory path. 57 | * 58 | * \return A new directory iterator. 59 | **/ 60 | JDirIterator* j_dir_iterator_new(gchar const* path); 61 | 62 | /** 63 | * Frees the memory allocated by a directory iterator. 64 | * 65 | * \code 66 | * \endcode 67 | * 68 | * \param iterator A directory iterator. 69 | **/ 70 | void j_dir_iterator_free(JDirIterator* iterator); 71 | 72 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JDirIterator, j_dir_iterator_free) 73 | 74 | /** 75 | * Checks whether another file is available. 76 | * 77 | * \code 78 | * \endcode 79 | * 80 | * \param iterator A directory iterator. 81 | * 82 | * \return TRUE on success, FALSE if the end of the iterator is reached. 83 | **/ 84 | gboolean j_dir_iterator_next(JDirIterator* iterator); 85 | 86 | /** 87 | * Returns the current file. 88 | * 89 | * \code 90 | * \endcode 91 | * 92 | * \param iterator A directory iterator. 93 | * 94 | * \return A list element. 95 | **/ 96 | gchar const* j_dir_iterator_get(JDirIterator* iterator); 97 | 98 | /** 99 | * @} 100 | **/ 101 | 102 | G_END_DECLS 103 | 104 | #endif 105 | -------------------------------------------------------------------------------- /doc/implementing-backend.md: -------------------------------------------------------------------------------- 1 | # Implementing a Backend 2 | 3 | Implementing an additional JULEA backend is done by providing its implementation within the `backend` directory. 4 | Depending on the type of backend, it has to be placed in the `object` or `kv` subdirectory. 5 | 6 | The `null` object and key-value backends can serve as starting points for a new backend. 7 | 8 | The following headers should be included by all backends: 9 | 10 | ```c 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | #include 17 | ``` 18 | 19 | The headers are followed by the actual implementation of the backend, which is divided into multiple functions. 20 | Among others, the backend can define initialization and finalization functions: 21 | 22 | ```c 23 | static 24 | gboolean 25 | backend_init (gchar const* path) 26 | { 27 | (void)path; 28 | 29 | return TRUE; 30 | } 31 | 32 | static 33 | void 34 | backend_fini (void) 35 | { 36 | } 37 | ``` 38 | 39 | Finally, a `JBackend` structure has to be defined and returned to make the backend known to JULEA. 40 | 41 | ```c 42 | static 43 | JBackend null_backend = { 44 | .type = J_BACKEND_TYPE_OBJECT, 45 | .component = J_BACKEND_COMPONENT_SERVER, 46 | .flags = 0, 47 | .object = { 48 | .backend_init = backend_init, 49 | .backend_fini = backend_fini, 50 | .backend_create = backend_create, 51 | .backend_delete = backend_delete, 52 | .backend_open = backend_open, 53 | .backend_close = backend_close, 54 | .backend_status = backend_status, 55 | .backend_sync = backend_sync, 56 | .backend_read = backend_read, 57 | .backend_write = backend_write 58 | } 59 | }; 60 | 61 | G_MODULE_EXPORT 62 | JBackend* 63 | backend_info (void) 64 | { 65 | return &null_backend; 66 | } 67 | ``` 68 | 69 | ## Build System 70 | 71 | JULEA uses the [Meson](https://mesonbuild.com/) build system. 72 | In case the new backend depends on additional libraries, these dependencies have to be checked first. 73 | Specifically, the dependency's existence is checked using `pkg-config`: 74 | 75 | ```python 76 | # Ubuntu 18.04 has LevelDB 1.20 77 | leveldb_version = '1.20' 78 | 79 | leveldb_dep = dependency('leveldb', 80 | version: '>= @0@'.format(leveldb_version), 81 | required: false, 82 | ) 83 | ``` 84 | 85 | Last, the backend has to be added to the list of backend targets. 86 | 87 | ```python 88 | ... 89 | 90 | if leveldb_dep.found() 91 | julea_backends += 'kv/leveldb' 92 | endif 93 | 94 | ... 95 | 96 | foreach backend: julea_backends 97 | ... 98 | 99 | if backend == 'kv/leveldb' 100 | extra_deps += leveldb_dep 101 | 102 | ... 103 | ``` 104 | -------------------------------------------------------------------------------- /include/core/jcache.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_CACHE_H 24 | #define JULEA_CACHE_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JCache Cache 36 | * 37 | * @{ 38 | **/ 39 | 40 | struct JCache; 41 | 42 | typedef struct JCache JCache; 43 | 44 | /** 45 | * Creates a new cache. 46 | * 47 | * \code 48 | * JCache* cache; 49 | * 50 | * cache = j_cache_new(1024); 51 | * \endcode 52 | * 53 | * \param size A size. 54 | * 55 | * \return A new cache. Should be freed with j_cache_free(). 56 | **/ 57 | JCache* j_cache_new(guint64 size); 58 | 59 | /** 60 | * Frees the memory allocated for the cache. 61 | * 62 | * \code 63 | * JCache* cache; 64 | * 65 | * ... 66 | * 67 | * j_cache_free(cache); 68 | * \endcode 69 | * 70 | * \param cache A cache. 71 | **/ 72 | void j_cache_free(JCache* cache); 73 | 74 | /** 75 | * Gets a new segment from the cache. 76 | * 77 | * \code 78 | * JCache* cache; 79 | * 80 | * ... 81 | * 82 | * j_cache_get(cache, 1024); 83 | * \endcode 84 | * 85 | * \param cache A cache. 86 | * \param length A length. 87 | * 88 | * \return A pointer to a segment of the cache, NULL if not enough space is available. 89 | **/ 90 | gpointer j_cache_get(JCache* cache, guint64 length); 91 | 92 | /** 93 | * Frees cache memory 94 | * 95 | * \code 96 | * JCache* cache; 97 | * 98 | * cache = j_cache_new(1024); 99 | * \endcode 100 | * 101 | * \param cache A cache. 102 | * \param data A pointer to cached data. 103 | * 104 | **/ 105 | void j_cache_release(JCache* cache, gpointer data); 106 | 107 | /** 108 | * @} 109 | **/ 110 | 111 | G_END_DECLS 112 | 113 | #endif 114 | -------------------------------------------------------------------------------- /include/item/jcollection-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_ITEM_COLLECTION_ITERATOR_H 24 | #define JULEA_ITEM_COLLECTION_ITERATOR_H 25 | 26 | #if !defined(JULEA_ITEM_H) && !defined(JULEA_ITEM_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JCollectionIterator Collection Iterator 36 | * 37 | * Data structures and functions for iterating over collections. 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JCollectionIterator; 43 | 44 | typedef struct JCollectionIterator JCollectionIterator; 45 | 46 | G_END_DECLS 47 | 48 | #include 49 | 50 | G_BEGIN_DECLS 51 | 52 | /** 53 | * Creates a new JCollectionIterator. 54 | * 55 | * \return A new JCollectionIterator. 56 | **/ 57 | JCollectionIterator* j_collection_iterator_new(void); 58 | 59 | /** 60 | * Frees the memory allocated by the JCollectionIterator. 61 | * 62 | * \param iterator A JCollectionIterator. 63 | **/ 64 | void j_collection_iterator_free(JCollectionIterator* iterator); 65 | 66 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JCollectionIterator, j_collection_iterator_free) 67 | 68 | /** 69 | * Checks whether another collection is available. 70 | * 71 | * \code 72 | * \endcode 73 | * 74 | * \param iterator A store iterator. 75 | * 76 | * \return TRUE on success, FALSE if the end of the store is reached. 77 | **/ 78 | gboolean j_collection_iterator_next(JCollectionIterator* iterator); 79 | 80 | /** 81 | * Returns the current collection. 82 | * 83 | * \code 84 | * \endcode 85 | * 86 | * \param iterator A store iterator. 87 | * 88 | * \return A new collection. Should be freed with j_collection_unref(). 89 | **/ 90 | JCollection* j_collection_iterator_get(JCollectionIterator* iterator); 91 | 92 | /** 93 | * @} 94 | **/ 95 | 96 | G_END_DECLS 97 | 98 | #endif 99 | -------------------------------------------------------------------------------- /test/core/credentials.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2020-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | #include "test.h" 29 | 30 | static void 31 | test_credentials_new_ref_unref(void) 32 | { 33 | guint const n = 1000; 34 | 35 | J_TEST_TRAP_START; 36 | for (guint i = 0; i < n; i++) 37 | { 38 | g_autoptr(JCredentials) credentials = NULL; 39 | 40 | credentials = j_credentials_new(); 41 | g_assert_nonnull(credentials); 42 | credentials = j_credentials_ref(credentials); 43 | g_assert_nonnull(credentials); 44 | j_credentials_unref(credentials); 45 | } 46 | J_TEST_TRAP_END; 47 | } 48 | 49 | static void 50 | test_credentials_get(void) 51 | { 52 | g_autoptr(JCredentials) credentials = NULL; 53 | 54 | guint32 user; 55 | guint32 group; 56 | 57 | J_TEST_TRAP_START; 58 | credentials = j_credentials_new(); 59 | user = j_credentials_get_user(credentials); 60 | group = j_credentials_get_group(credentials); 61 | 62 | g_assert_cmpuint(user, ==, getuid()); 63 | g_assert_cmpuint(group, ==, getgid()); 64 | J_TEST_TRAP_END; 65 | } 66 | 67 | static void 68 | test_credentials_serialize(void) 69 | { 70 | g_autoptr(JCredentials) credentials = NULL; 71 | g_autoptr(JCredentials) credentials2 = NULL; 72 | 73 | bson_t* bson = NULL; 74 | 75 | J_TEST_TRAP_START; 76 | credentials = j_credentials_new(); 77 | credentials2 = j_credentials_new(); 78 | 79 | bson = j_credentials_serialize(credentials); 80 | g_assert_nonnull(bson); 81 | 82 | j_credentials_deserialize(credentials2, bson); 83 | 84 | bson_destroy(bson); 85 | J_TEST_TRAP_END; 86 | } 87 | 88 | void 89 | test_core_credentials(void) 90 | { 91 | g_test_add_func("/core/credentials/new_ref_unref", test_credentials_new_ref_unref); 92 | g_test_add_func("/core/credentials/get", test_credentials_get); 93 | g_test_add_func("/core/credentials/serialize", test_credentials_serialize); 94 | } 95 | -------------------------------------------------------------------------------- /include/item/jitem-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_ITEM_ITEM_ITERATOR_H 24 | #define JULEA_ITEM_ITEM_ITERATOR_H 25 | 26 | #if !defined(JULEA_ITEM_H) && !defined(JULEA_ITEM_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JItemIterator Item Iterator 36 | * 37 | * Data structures and functions for iterating over items. 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JItemIterator; 43 | 44 | typedef struct JItemIterator JItemIterator; 45 | G_END_DECLS 46 | 47 | #include 48 | #include 49 | 50 | G_BEGIN_DECLS 51 | 52 | /** 53 | * Creates a new JItemIterator. 54 | * 55 | * \param collection A JCollection. 56 | * 57 | * \return A new JItemIterator. 58 | **/ 59 | JItemIterator* j_item_iterator_new(JCollection* collection); 60 | 61 | /** 62 | * Frees the memory allocated by the JItemIterator. 63 | * 64 | * \param iterator A JItemIterator. 65 | **/ 66 | void j_item_iterator_free(JItemIterator* iterator); 67 | 68 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JItemIterator, j_item_iterator_free) 69 | 70 | /** 71 | * Checks whether another item is available. 72 | * 73 | * \code 74 | * \endcode 75 | * 76 | * \param iterator A collection iterator. 77 | * 78 | * \return TRUE on success, FALSE if the end of the collection is reached. 79 | **/ 80 | gboolean j_item_iterator_next(JItemIterator* iterator); 81 | 82 | /** 83 | * Returns the current item. 84 | * 85 | * \code 86 | * \endcode 87 | * 88 | * \param iterator A collection iterator. 89 | * 90 | * \return A new item. Should be freed with j_item_unref(). 91 | **/ 92 | JItem* j_item_iterator_get(JItemIterator* iterator); 93 | 94 | /** 95 | * @} 96 | **/ 97 | 98 | G_END_DECLS 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /lib/core/jmemory-chunk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | 33 | /** 34 | * \addtogroup JMemoryChunk 35 | * 36 | * @{ 37 | **/ 38 | 39 | /** 40 | * A cache. 41 | */ 42 | struct JMemoryChunk 43 | { 44 | /** 45 | * The size. 46 | */ 47 | guint64 size; 48 | 49 | /** 50 | * The data. 51 | */ 52 | gchar* data; 53 | 54 | /** 55 | * The current position within #data. 56 | */ 57 | gchar* current; 58 | }; 59 | 60 | JMemoryChunk* 61 | j_memory_chunk_new(guint64 size) 62 | { 63 | J_TRACE_FUNCTION(NULL); 64 | 65 | JMemoryChunk* cache; 66 | 67 | g_return_val_if_fail(size > 0, NULL); 68 | 69 | cache = g_new(JMemoryChunk, 1); 70 | cache->size = size; 71 | cache->data = g_malloc(cache->size); 72 | cache->current = cache->data; 73 | 74 | return cache; 75 | } 76 | 77 | void 78 | j_memory_chunk_free(JMemoryChunk* cache) 79 | { 80 | J_TRACE_FUNCTION(NULL); 81 | 82 | g_return_if_fail(cache != NULL); 83 | 84 | if (cache->data != NULL) 85 | { 86 | g_free(cache->data); 87 | } 88 | 89 | g_free(cache); 90 | } 91 | 92 | gpointer 93 | j_memory_chunk_get(JMemoryChunk* cache, guint64 length) 94 | { 95 | J_TRACE_FUNCTION(NULL); 96 | 97 | gpointer ret = NULL; 98 | 99 | g_return_val_if_fail(cache != NULL, NULL); 100 | 101 | if (cache->current + length > cache->data + cache->size) 102 | { 103 | return NULL; 104 | } 105 | 106 | ret = cache->current; 107 | cache->current += length; 108 | 109 | return ret; 110 | } 111 | 112 | void 113 | j_memory_chunk_reset(JMemoryChunk* cache) 114 | { 115 | J_TRACE_FUNCTION(NULL); 116 | 117 | g_return_if_fail(cache != NULL); 118 | 119 | cache->current = cache->data; 120 | } 121 | 122 | /** 123 | * @} 124 | **/ 125 | -------------------------------------------------------------------------------- /doc/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | To configure JULEA, the `julea-config` tool can be used. 4 | Configuration files are typically saved in `~/.config/julea` (if `--user` is specified) or `/etc/xdg/julea` (if `--system` is specified). 5 | For an example invocation of `julea-config`, please refer to the [Quick Start](../README.md#quick-start). 6 | 7 | JULEA supports multiple configurations that can be selected at runtime using the `JULEA_CONFIG` environment variable. 8 | They can be created using the `--name` parameter when calling `julea-config`. 9 | If no name is specified, the default (`julea`) is used. 10 | 11 | ## Backends 12 | 13 | JULEA supports multiple backends that can be used for object, key-value or database storage. 14 | It is possible to use them either on the client or on the server. 15 | The following tables visualize all supported types and special configuration parameters. 16 | Some backends might require third-party servers to be usable. 17 | Please also refer to the documentation about [containers](dependencies.md#containers). 18 | 19 | The backend paths can contain the special string `{PORT}`, which will be replaced with the server's port at runtime. 20 | This can be used to run two servers on the same machine, as sharing backend paths among multiple instances will typically lead to problems. 21 | 22 | ### Object Backends 23 | 24 | | Backend | Client | Server | Path format | 25 | |---------|:------:|:------:|--------------| 26 | | gio | ❌ | ✔ | Path to a directory (`/var/storage/gio`) | 27 | | null | ❌ | ✔ | | 28 | | posix | ❌ | ✔ | Path to a directory (`/var/storage/posix`) | 29 | | rados | ✔ | ❌ | Path to a configuration file and pool name (`/etc/ceph/ceph.conf:data`) | 30 | 31 | ## Key-Value Backends 32 | 33 | | Backend | Client | Server | Path format | 34 | |---------|:------:|:------:|--------------| 35 | | gdbm | ❌ | ✔ | Path to a file (`/var/storage/gdbm`) | 36 | | leveldb | ❌ | ✔ | Path to a directory (`/var/storage/leveldb`) | 37 | | lmdb | ❌ | ✔ | Path to a directory (`/var/storage/lmdb`) | 38 | | mongodb | ✔ | ❌ | Host and database (`127.0.0.1:julea_db`) | 39 | | null | ❌ | ✔ | | 40 | | rocksdb | ❌ | ✔ | Path to a directory (`/var/storage/rocksdb`) | 41 | | sqlite | ❌ | ✔ | Path to a file (`/var/storage/sqlite.db`) | 42 | 43 | ## Database Backends 44 | 45 | | Backend | Client | Server | Path format | 46 | |---------|:------:|:------:|--------------| 47 | | mysql | ✔ | ❌ | Host, database, user and password (`127.0.0.1:julea_db:julea_user:julea_pw`) | 48 | | null | ❌ | ✔ | | 49 | | sqlite | ❌ | ✔ | Path to a file (`/var/storage/sqlite.db`) or `:memory:` for an in-memory database | 50 | -------------------------------------------------------------------------------- /doc/installation-usage.md: -------------------------------------------------------------------------------- 1 | # Installation and Usage 2 | 3 | By default, JULEA is built as a debug version that can be used for development and debugging purposes. 4 | To build and install a release version, some additional arguments and variables are necessary. 5 | 6 | First of all, you will have to make sure that JULEA's [dependencies](dependencies.md) have been installed. 7 | Afterwards, JULEA's environment has to be loaded with the environment variable `JULEA_PREFIX`: 8 | 9 | ```console 10 | $ export JULEA_PREFIX="${HOME}/julea-install" 11 | $ . scripts/environment.sh 12 | ``` 13 | 14 | You can then build and install JULEA in release mode: 15 | 16 | ```console 17 | $ meson setup --prefix="${HOME}/julea-install" --buildtype=release bld-release 18 | $ ninja -C bld-release 19 | $ ninja -C bld-release install 20 | ``` 21 | 22 | Finally, you have to create a [configuration](configuration.md) if you do not have one already. 23 | 24 | ## Benchmarks 25 | 26 | JULEA provides a script that executes benchmarks to measure the performance of JULEA's different components. 27 | 28 | ```console 29 | $ ./scripts/setup.sh start 30 | $ ./scripts/benchmark.sh 31 | $ ./scripts/setup.sh stop 32 | ``` 33 | 34 | By default, the individual benchmarks will run for approximately one second. 35 | This can be changed using the `-d` or `--duration` option. 36 | Additional options can be shown using `-h` or `--help`. 37 | 38 | ```console 39 | $ ./scripts/benchmark.sh -d 60 40 | ``` 41 | 42 | ## Using Specific Compilers 43 | 44 | JULEA and its dependencies can also be built using specific compilers instead of the default ones. 45 | The `install-dependencies.sh` script supports a `JULEA_SPACK_COMPILER` variable that can be used to set the compiler: 46 | 47 | ```console 48 | $ export JULEA_SPACK_COMPILER=llvm 49 | $ ./scripts/install-dependencies.sh 50 | ``` 51 | 52 | The compiler to use for JULEA can be configured using Meson by setting the `CC` variable: 53 | 54 | ```console 55 | $ export CC=clang 56 | $ meson setup --prefix="${HOME}/julea-install" bld 57 | ``` 58 | 59 | It is important to note that the `JULEA_SPACK_COMPILER` variable refers to the compiler name used within Spack, while the `CC` variable refers to the compiler binary that should be used. 60 | 61 | ## Installation via Spack 62 | 63 | Alternatively, you can install JULEA using Spack, which will install JULEA and all of its dependencies: 64 | 65 | ```console 66 | $ git clone https://github.com/spack/spack.git 67 | $ . spack/share/spack/setup-env.sh 68 | $ spack install julea 69 | $ spack load julea 70 | ``` 71 | 72 | The second and fourth steps will have to be repeated every time you want to use JULEA. 73 | Since `spack load` will take care of setting up the environment, you do not need to use JULEA's environment script in this case. 74 | -------------------------------------------------------------------------------- /test/core/memory-chunk.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "test.h" 28 | 29 | static void 30 | test_memory_chunk_new_free(void) 31 | { 32 | JMemoryChunk* memory_chunk; 33 | 34 | J_TEST_TRAP_START; 35 | memory_chunk = j_memory_chunk_new(42); 36 | g_assert_true(memory_chunk != NULL); 37 | 38 | j_memory_chunk_free(memory_chunk); 39 | J_TEST_TRAP_END; 40 | } 41 | 42 | static void 43 | test_memory_chunk_get(void) 44 | { 45 | JMemoryChunk* memory_chunk; 46 | gpointer ret; 47 | 48 | J_TEST_TRAP_START; 49 | memory_chunk = j_memory_chunk_new(3); 50 | 51 | ret = j_memory_chunk_get(memory_chunk, 1); 52 | g_assert_true(ret != NULL); 53 | ret = j_memory_chunk_get(memory_chunk, 1); 54 | g_assert_true(ret != NULL); 55 | ret = j_memory_chunk_get(memory_chunk, 1); 56 | g_assert_true(ret != NULL); 57 | ret = j_memory_chunk_get(memory_chunk, 1); 58 | g_assert_true(ret == NULL); 59 | 60 | j_memory_chunk_free(memory_chunk); 61 | J_TEST_TRAP_END; 62 | } 63 | 64 | static void 65 | test_memory_chunk_reset(void) 66 | { 67 | JMemoryChunk* memory_chunk; 68 | gpointer ret; 69 | 70 | J_TEST_TRAP_START; 71 | memory_chunk = j_memory_chunk_new(1); 72 | 73 | ret = j_memory_chunk_get(memory_chunk, 1); 74 | g_assert_true(ret != NULL); 75 | ret = j_memory_chunk_get(memory_chunk, 1); 76 | g_assert_true(ret == NULL); 77 | 78 | j_memory_chunk_reset(memory_chunk); 79 | 80 | ret = j_memory_chunk_get(memory_chunk, 1); 81 | g_assert_true(ret != NULL); 82 | ret = j_memory_chunk_get(memory_chunk, 1); 83 | g_assert_true(ret == NULL); 84 | 85 | j_memory_chunk_free(memory_chunk); 86 | J_TEST_TRAP_END; 87 | } 88 | 89 | void 90 | test_core_memory_chunk(void) 91 | { 92 | g_test_add_func("/core/memory-chunk/new_free", test_memory_chunk_new_free); 93 | g_test_add_func("/core/memory-chunk/get", test_memory_chunk_get); 94 | g_test_add_func("/core/memory-chunk/reset", test_memory_chunk_reset); 95 | } 96 | -------------------------------------------------------------------------------- /include/core/jmemory-chunk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_MEMORY_CHUNK_H 24 | #define JULEA_MEMORY_CHUNK_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JMemoryChunk Memory Chunk 36 | * 37 | * A chunk of memory intended for short time cashing of larger data chunks. 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JMemoryChunk; 43 | 44 | typedef struct JMemoryChunk JMemoryChunk; 45 | 46 | /** 47 | * Creates a new chunk. 48 | * 49 | * \code 50 | * JMemoryChunk* chunk; 51 | * 52 | * chunk = j_memory_chunk_new(1024); 53 | * \endcode 54 | * 55 | * \param size A size. 56 | * 57 | * \return A new chunk. Should be freed with j_memory_chunk_free(). 58 | **/ 59 | JMemoryChunk* j_memory_chunk_new(guint64 size); 60 | 61 | /** 62 | * Frees the memory allocated for the chunk. 63 | * 64 | * \code 65 | * JMemoryChunk* chunk; 66 | * 67 | * ... 68 | * 69 | * j_memory_chunk_free(chunk); 70 | * \endcode 71 | * 72 | * \param chunk A chunk. 73 | **/ 74 | void j_memory_chunk_free(JMemoryChunk* chunk); 75 | 76 | /** 77 | * Gets a new segment from the chunk. 78 | * 79 | * \code 80 | * JMemoryChunk* chunk; 81 | * 82 | * ... 83 | * 84 | * j_memory_chunk_get(chunk, 1024); 85 | * \endcode 86 | * 87 | * \param chunk A chunk. 88 | * \param length A length. 89 | * 90 | * \return A pointer to a segment of the chunk, NULL if not enough space is available. 91 | **/ 92 | gpointer j_memory_chunk_get(JMemoryChunk* chunk, guint64 length); 93 | 94 | /** 95 | * Resets the given chunk. All data inside should be considered lost. 96 | * 97 | * \param chunk A chunk. 98 | * 99 | **/ 100 | void j_memory_chunk_reset(JMemoryChunk* chunk); 101 | 102 | /** 103 | * @} 104 | **/ 105 | 106 | G_END_DECLS 107 | 108 | #endif 109 | -------------------------------------------------------------------------------- /lib/item/jitem-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #include 37 | 38 | /** 39 | * \addtogroup JItemIterator 40 | * 41 | * @{ 42 | **/ 43 | 44 | struct JItemIterator 45 | { 46 | JCollection* collection; 47 | JKVIterator* iterator; 48 | }; 49 | 50 | JItemIterator* 51 | j_item_iterator_new(JCollection* collection) 52 | { 53 | JItemIterator* iterator; 54 | g_autofree gchar* prefix = NULL; 55 | 56 | g_return_val_if_fail(collection != NULL, NULL); 57 | 58 | prefix = g_strdup_printf("%s/", j_collection_get_name(collection)); 59 | 60 | iterator = g_new(JItemIterator, 1); 61 | iterator->collection = j_collection_ref(collection); 62 | iterator->iterator = j_kv_iterator_new("items", prefix); 63 | 64 | return iterator; 65 | } 66 | 67 | void 68 | j_item_iterator_free(JItemIterator* iterator) 69 | { 70 | g_return_if_fail(iterator != NULL); 71 | 72 | j_kv_iterator_free(iterator->iterator); 73 | j_collection_unref(iterator->collection); 74 | 75 | g_free(iterator); 76 | } 77 | 78 | gboolean 79 | j_item_iterator_next(JItemIterator* iterator) 80 | { 81 | g_return_val_if_fail(iterator != NULL, FALSE); 82 | 83 | return j_kv_iterator_next(iterator->iterator); 84 | } 85 | 86 | JItem* 87 | j_item_iterator_get(JItemIterator* iterator) 88 | { 89 | JItem* item; 90 | bson_t tmp[1]; 91 | gconstpointer value; 92 | guint32 len; 93 | 94 | g_return_val_if_fail(iterator != NULL, NULL); 95 | 96 | j_kv_iterator_get(iterator->iterator, &value, &len); 97 | bson_init_static(tmp, value, len); 98 | item = j_item_new_from_bson(iterator->collection, tmp); 99 | 100 | return item; 101 | } 102 | 103 | /** 104 | * @} 105 | **/ 106 | -------------------------------------------------------------------------------- /scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # JULEA - Flexible storage framework 4 | # Copyright (C) 2013-2024 Michael Kuhn 5 | # Copyright (C) 2013 Anna Fuchs 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Lesser General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Lesser General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Lesser General Public License 18 | # along with this program. If not, see . 19 | 20 | set -e 21 | 22 | SELF_PATH="$(readlink --canonicalize-existing -- "$0")" 23 | SELF_DIR="${SELF_PATH%/*}" 24 | SELF_BASE="${SELF_PATH##*/}" 25 | 26 | # shellcheck source=scripts/common 27 | . "${SELF_DIR}/common" 28 | # shellcheck source=scripts/setup 29 | . "${SELF_DIR}/setup" 30 | # shellcheck source=scripts/spack 31 | . "${SELF_DIR}/spack" 32 | 33 | usage () 34 | { 35 | echo "Usage: ${SELF_BASE} start|stop|restart" 36 | exit 1 37 | } 38 | 39 | setup_slurm () 40 | { 41 | local mode 42 | local nodes 43 | 44 | mode="$1" 45 | 46 | test -n "${mode}" || return 1 47 | 48 | if test -n "${SLURM_JOB_NODELIST}" 49 | then 50 | nodes="$(scontrol show hostnames)" 51 | 52 | for node in ${nodes} 53 | do 54 | # shellcheck disable=SC2029 55 | ssh "${node}" "${SELF_PATH}" "${mode}-local" 56 | done 57 | 58 | return 0 59 | fi 60 | 61 | return 1 62 | } 63 | 64 | test -n "$1" || usage 65 | 66 | MODE="$1" 67 | 68 | #export G_MESSAGES_DEBUG=JULEA 69 | 70 | set_path 71 | set_library_path 72 | set_backend_path 73 | 74 | if test -z "${JULEA_SPACK_DIR}" 75 | then 76 | JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" 77 | fi 78 | 79 | spack_load_dependencies 80 | 81 | setup_init 82 | 83 | case "${MODE}" in 84 | start) 85 | if ! setup_slurm "${MODE}" 86 | then 87 | setup_start 88 | fi 89 | ;; 90 | start-local) 91 | setup_start 92 | ;; 93 | stop) 94 | if ! setup_slurm "${MODE}" 95 | then 96 | setup_stop 97 | fi 98 | ;; 99 | stop-local) 100 | setup_stop 101 | ;; 102 | clean) 103 | if ! setup_slurm "${MODE}" 104 | then 105 | setup_clean 106 | fi 107 | ;; 108 | clean-local) 109 | setup_clean 110 | ;; 111 | restart) 112 | if ! setup_slurm stop 113 | then 114 | setup_stop 115 | fi 116 | 117 | sleep 10 118 | 119 | if ! setup_slurm start 120 | then 121 | setup_start 122 | fi 123 | ;; 124 | *) 125 | usage 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /lib/core/jlist-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | /** 34 | * \addtogroup JListIterator List Iterator 35 | * 36 | * @{ 37 | **/ 38 | 39 | /** 40 | * 41 | **/ 42 | struct JListIterator 43 | { 44 | /** 45 | * The associated list. 46 | **/ 47 | JList* list; 48 | /** 49 | * The current list element. 50 | **/ 51 | JListElement* current; 52 | /** 53 | * Whether the current element is the first one. 54 | **/ 55 | gboolean first; 56 | }; 57 | 58 | JListIterator* 59 | j_list_iterator_new(JList* list) 60 | { 61 | J_TRACE_FUNCTION(NULL); 62 | 63 | JListIterator* iterator; 64 | 65 | g_return_val_if_fail(list != NULL, NULL); 66 | 67 | iterator = g_new(JListIterator, 1); 68 | iterator->list = j_list_ref(list); 69 | iterator->current = j_list_head(iterator->list); 70 | iterator->first = TRUE; 71 | 72 | return iterator; 73 | } 74 | 75 | void 76 | j_list_iterator_free(JListIterator* iterator) 77 | { 78 | J_TRACE_FUNCTION(NULL); 79 | 80 | g_return_if_fail(iterator != NULL); 81 | 82 | j_list_unref(iterator->list); 83 | 84 | g_free(iterator); 85 | } 86 | 87 | gboolean 88 | j_list_iterator_next(JListIterator* iterator) 89 | { 90 | J_TRACE_FUNCTION(NULL); 91 | 92 | g_return_val_if_fail(iterator != NULL, FALSE); 93 | 94 | if (G_UNLIKELY(iterator->first)) 95 | { 96 | iterator->first = FALSE; 97 | } 98 | else 99 | { 100 | iterator->current = iterator->current->next; 101 | } 102 | 103 | return (iterator->current != NULL); 104 | } 105 | 106 | gpointer 107 | j_list_iterator_get(JListIterator* iterator) 108 | { 109 | J_TRACE_FUNCTION(NULL); 110 | 111 | g_return_val_if_fail(iterator != NULL, NULL); 112 | g_return_val_if_fail(iterator->current != NULL, NULL); 113 | 114 | return iterator->current->data; 115 | } 116 | 117 | /** 118 | * @} 119 | **/ 120 | -------------------------------------------------------------------------------- /lib/db/jdb.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2019-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | /** 32 | * \defgroup JDB DB 33 | * 34 | * Data structures and functions for managing databases. 35 | * 36 | * @{ 37 | **/ 38 | 39 | static JBackend* j_db_backend = NULL; 40 | static GModule* j_db_module = NULL; 41 | 42 | /// \todo copy and use GLib's G_DEFINE_CONSTRUCTOR/DESTRUCTOR 43 | static void __attribute__((constructor)) j_db_init(void); 44 | static void __attribute__((destructor)) j_db_fini(void); 45 | 46 | /** 47 | * Initializes the db client. 48 | */ 49 | static void 50 | j_db_init(void) 51 | { 52 | gchar const* db_backend; 53 | gchar const* db_path; 54 | 55 | if (j_db_backend != NULL && j_db_module != NULL) 56 | { 57 | return; 58 | } 59 | 60 | db_backend = j_configuration_get_backend(j_configuration(), J_BACKEND_TYPE_DB); 61 | db_path = j_configuration_get_backend_path(j_configuration(), J_BACKEND_TYPE_DB); 62 | 63 | if (j_backend_load(db_backend, J_BACKEND_COMPONENT_CLIENT, J_BACKEND_TYPE_DB, &j_db_module, &j_db_backend)) 64 | { 65 | if (j_db_backend != NULL && !j_backend_db_init(j_db_backend, db_path)) 66 | { 67 | g_critical("Could not initialize db backend %s.\n", db_backend); 68 | } 69 | } 70 | } 71 | 72 | /** 73 | * Shuts down the db client. 74 | */ 75 | static void 76 | j_db_fini(void) 77 | { 78 | if (j_db_backend == NULL && j_db_module == NULL) 79 | { 80 | return; 81 | } 82 | 83 | if (j_db_backend != NULL) 84 | { 85 | j_backend_db_fini(j_db_backend); 86 | } 87 | 88 | if (j_db_module != NULL) 89 | { 90 | j_backend_unload(j_db_backend, j_db_module); 91 | } 92 | 93 | j_db_backend = NULL; 94 | j_db_module = NULL; 95 | } 96 | 97 | /** 98 | * Returns the db backend. 99 | * 100 | * \return The db backend. 101 | */ 102 | JBackend* 103 | j_db_get_backend(void) 104 | { 105 | return j_db_backend; 106 | } 107 | 108 | /** 109 | * @} 110 | **/ 111 | -------------------------------------------------------------------------------- /cli/delete.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "cli.h" 22 | 23 | gboolean 24 | j_cmd_delete(gchar const** arguments) 25 | { 26 | gboolean ret = TRUE; 27 | g_autoptr(JBatch) batch = NULL; 28 | g_autoptr(JObjectURI) duri = NULL; 29 | g_autoptr(JObjectURI) ouri = NULL; 30 | g_autoptr(JURI) uri = NULL; 31 | GError* error = NULL; 32 | 33 | if (j_cmd_arguments_length(arguments) != 1) 34 | { 35 | ret = FALSE; 36 | j_cmd_usage(); 37 | goto end; 38 | } 39 | 40 | ouri = j_object_uri_new(arguments[0], J_OBJECT_URI_SCHEME_OBJECT); 41 | 42 | if (ouri != NULL) 43 | { 44 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 45 | j_object_delete(j_object_uri_get_object(ouri), batch); 46 | ret = j_batch_execute(batch); 47 | 48 | goto end; 49 | } 50 | 51 | duri = j_object_uri_new(arguments[0], J_OBJECT_URI_SCHEME_DISTRIBUTED_OBJECT); 52 | 53 | if (duri != NULL) 54 | { 55 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 56 | j_distributed_object_delete(j_object_uri_get_distributed_object(duri), batch); 57 | ret = j_batch_execute(batch); 58 | 59 | goto end; 60 | } 61 | 62 | uri = j_uri_new(arguments[0]); 63 | 64 | if (uri != NULL) 65 | { 66 | if (!j_uri_get(uri, &error)) 67 | { 68 | ret = FALSE; 69 | g_print("Error: %s\n", error->message); 70 | g_error_free(error); 71 | 72 | goto end; 73 | } 74 | 75 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 76 | 77 | if (j_uri_get_item(uri) != NULL) 78 | { 79 | j_item_delete(j_uri_get_item(uri), batch); 80 | ret = j_batch_execute(batch); 81 | } 82 | else if (j_uri_get_collection(uri) != NULL) 83 | { 84 | j_collection_delete(j_uri_get_collection(uri), batch); 85 | ret = j_batch_execute(batch); 86 | } 87 | else 88 | { 89 | ret = FALSE; 90 | j_cmd_usage(); 91 | } 92 | 93 | goto end; 94 | } 95 | 96 | ret = FALSE; 97 | g_print("Error: Invalid argument “%s”.\n", arguments[0]); 98 | 99 | end: 100 | return ret; 101 | } 102 | -------------------------------------------------------------------------------- /cli/create.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "cli.h" 22 | 23 | #include 24 | 25 | gboolean 26 | j_cmd_create(gchar const** arguments, gboolean with_parents) 27 | { 28 | gboolean ret = TRUE; 29 | g_autoptr(JObjectURI) duri = NULL; 30 | g_autoptr(JObjectURI) ouri = NULL; 31 | g_autoptr(JKVURI) kuri = NULL; 32 | g_autoptr(JURI) uri = NULL; 33 | GError* error = NULL; 34 | 35 | if (j_cmd_arguments_length(arguments) != 1) 36 | { 37 | ret = FALSE; 38 | j_cmd_usage(); 39 | goto end; 40 | } 41 | 42 | ouri = j_object_uri_new(arguments[0], J_OBJECT_URI_SCHEME_OBJECT); 43 | 44 | if (ouri != NULL) 45 | { 46 | g_autoptr(JBatch) batch = NULL; 47 | 48 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 49 | j_object_create(j_object_uri_get_object(ouri), batch); 50 | ret = j_batch_execute(batch); 51 | 52 | goto end; 53 | } 54 | 55 | duri = j_object_uri_new(arguments[0], J_OBJECT_URI_SCHEME_DISTRIBUTED_OBJECT); 56 | 57 | if (duri != NULL) 58 | { 59 | g_autoptr(JBatch) batch = NULL; 60 | 61 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 62 | j_distributed_object_create(j_object_uri_get_distributed_object(duri), batch); 63 | ret = j_batch_execute(batch); 64 | 65 | goto end; 66 | } 67 | 68 | kuri = j_kv_uri_new(arguments[0], J_KV_URI_SCHEME_KV); 69 | 70 | if (kuri != NULL) 71 | { 72 | g_autoptr(JBatch) batch = NULL; 73 | 74 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 75 | j_kv_put(j_kv_uri_get_kv(kuri), g_strdup("empty"), 6, g_free, batch); 76 | ret = j_batch_execute(batch); 77 | 78 | goto end; 79 | } 80 | 81 | uri = j_uri_new(arguments[0]); 82 | 83 | if (uri != NULL) 84 | { 85 | if (!j_uri_create(uri, with_parents, &error)) 86 | { 87 | ret = FALSE; 88 | g_print("Error: %s\n", error->message); 89 | g_error_free(error); 90 | } 91 | 92 | goto end; 93 | } 94 | 95 | ret = FALSE; 96 | g_print("Error: Invalid argument “%s”.\n", arguments[0]); 97 | 98 | end: 99 | return ret; 100 | } 101 | -------------------------------------------------------------------------------- /test/core/semantics.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "test.h" 26 | 27 | static void 28 | test_semantics_fixture_setup(JSemantics** semantics, G_GNUC_UNUSED gconstpointer data) 29 | { 30 | *semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); 31 | } 32 | 33 | static void 34 | test_semantics_fixture_teardown(JSemantics** semantics, G_GNUC_UNUSED gconstpointer data) 35 | { 36 | j_semantics_unref(*semantics); 37 | } 38 | 39 | static void 40 | test_semantics_new_ref_unref(void) 41 | { 42 | guint const n = 100000; 43 | 44 | J_TEST_TRAP_START; 45 | for (guint i = 0; i < n; i++) 46 | { 47 | g_autoptr(JSemantics) semantics = NULL; 48 | 49 | semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); 50 | g_assert_true(semantics != NULL); 51 | semantics = j_semantics_ref(semantics); 52 | g_assert_true(semantics != NULL); 53 | j_semantics_unref(semantics); 54 | } 55 | J_TEST_TRAP_END; 56 | } 57 | 58 | static void 59 | test_semantics_set_get(JSemantics** semantics, G_GNUC_UNUSED gconstpointer data) 60 | { 61 | gint s; 62 | 63 | J_TEST_TRAP_START; 64 | j_semantics_set(*semantics, J_SEMANTICS_ATOMICITY, J_SEMANTICS_ATOMICITY_OPERATION); 65 | s = j_semantics_get(*semantics, J_SEMANTICS_ATOMICITY); 66 | g_assert_cmpint(s, ==, J_SEMANTICS_ATOMICITY_OPERATION); 67 | 68 | j_semantics_set(*semantics, J_SEMANTICS_CONSISTENCY, J_SEMANTICS_CONSISTENCY_IMMEDIATE); 69 | s = j_semantics_get(*semantics, J_SEMANTICS_CONSISTENCY); 70 | g_assert_cmpint(s, ==, J_SEMANTICS_CONSISTENCY_IMMEDIATE); 71 | 72 | j_semantics_set(*semantics, J_SEMANTICS_PERSISTENCY, J_SEMANTICS_PERSISTENCY_STORAGE); 73 | s = j_semantics_get(*semantics, J_SEMANTICS_PERSISTENCY); 74 | g_assert_cmpint(s, ==, J_SEMANTICS_PERSISTENCY_STORAGE); 75 | 76 | j_semantics_set(*semantics, J_SEMANTICS_SECURITY, J_SEMANTICS_SECURITY_STRICT); 77 | s = j_semantics_get(*semantics, J_SEMANTICS_SECURITY); 78 | g_assert_cmpint(s, ==, J_SEMANTICS_SECURITY_STRICT); 79 | J_TEST_TRAP_END; 80 | } 81 | 82 | void 83 | test_core_semantics(void) 84 | { 85 | g_test_add_func("/core/semantics/new_ref_unref", test_semantics_new_ref_unref); 86 | g_test_add("/core/semantics/set_get", JSemantics*, NULL, test_semantics_fixture_setup, test_semantics_set_get, test_semantics_fixture_teardown); 87 | } 88 | -------------------------------------------------------------------------------- /include/item/jcollection-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_ITEM_COLLECTION_INTERNAL_H 24 | #define JULEA_ITEM_COLLECTION_INTERNAL_H 25 | 26 | #if !defined(JULEA_ITEM_H) && !defined(JULEA_ITEM_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | 36 | #include 37 | 38 | G_BEGIN_DECLS 39 | 40 | /** 41 | * \addtogroup JCollection 42 | * 43 | * @{ 44 | **/ 45 | 46 | /** 47 | * Creates a new collection. 48 | * 49 | * \code 50 | * JCollection* c; 51 | * 52 | * c = j_collection_new("JULEA"); 53 | * \endcode 54 | * 55 | * \param name A collection name. 56 | * 57 | * \return A new collection. Should be freed with j_collection_unref(). 58 | **/ 59 | G_GNUC_INTERNAL JCollection* j_collection_new(gchar const* name); 60 | 61 | /** 62 | * Creates a new collection from a BSON object. 63 | * 64 | * \private 65 | * 66 | * \code 67 | * \endcode 68 | * 69 | * \param b A BSON object. 70 | * 71 | * \return A new collection. Should be freed with j_collection_unref(). 72 | **/ 73 | G_GNUC_INTERNAL JCollection* j_collection_new_from_bson(bson_t const* b); 74 | 75 | /** 76 | * Serializes a collection. 77 | * 78 | * \private 79 | * 80 | * \code 81 | * \endcode 82 | * 83 | * \param collection A collection. 84 | * 85 | * \return A new BSON object. Should be freed with bson_destroy(). 86 | **/ 87 | G_GNUC_INTERNAL bson_t* j_collection_serialize(JCollection* collection); 88 | 89 | /** 90 | * Deserializes a collection. 91 | * 92 | * \private 93 | * 94 | * \code 95 | * \endcode 96 | * 97 | * \param collection A collection. 98 | * \param b A BSON object. 99 | **/ 100 | G_GNUC_INTERNAL void j_collection_deserialize(JCollection* collection, bson_t const* b); 101 | 102 | /** 103 | * Returns a collection's ID. 104 | * 105 | * \private 106 | * 107 | * \code 108 | * \endcode 109 | * 110 | * \param collection A collection. 111 | * 112 | * \return An ID. 113 | **/ 114 | G_GNUC_INTERNAL bson_oid_t const* j_collection_get_id(JCollection* collection); 115 | 116 | /** 117 | * @} 118 | **/ 119 | 120 | G_END_DECLS 121 | 122 | #endif 123 | -------------------------------------------------------------------------------- /include/core/jstatistics.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_STATISTICS_H 24 | #define JULEA_STATISTICS_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JStatistics Statistics 36 | * 37 | * @{ 38 | **/ 39 | 40 | enum JStatisticsType 41 | { 42 | J_STATISTICS_FILES_CREATED, 43 | J_STATISTICS_FILES_DELETED, 44 | J_STATISTICS_FILES_STATED, 45 | J_STATISTICS_SYNC, 46 | J_STATISTICS_BYTES_READ, 47 | J_STATISTICS_BYTES_WRITTEN, 48 | J_STATISTICS_BYTES_RECEIVED, 49 | J_STATISTICS_BYTES_SENT 50 | }; 51 | 52 | typedef enum JStatisticsType JStatisticsType; 53 | 54 | struct JStatistics; 55 | 56 | typedef struct JStatistics JStatistics; 57 | 58 | /** 59 | * Creates a new statistics. 60 | * 61 | * \private 62 | * 63 | * \code 64 | * JStatistics* s; 65 | * 66 | * s = j_statistics_new(FALSE); 67 | * \endcode 68 | * 69 | * \param trace Specify wether tracing was on or off while gathering data. 70 | * 71 | * \return A new statistics. Should be freed with j_statistics_free(). 72 | **/ 73 | JStatistics* j_statistics_new(gboolean trace); 74 | 75 | /** 76 | * Frees the memory allocated for the statistics. 77 | * 78 | * \private 79 | * 80 | * \code 81 | * \endcode 82 | * 83 | * \param statistics A statistics. 84 | **/ 85 | void j_statistics_free(JStatistics* statistics); 86 | 87 | /** 88 | * Get a value from a statistic. 89 | * 90 | * \private 91 | * 92 | * \code 93 | * \endcode 94 | * 95 | * \param statistics A statistics. 96 | * \param type A statistics type. 97 | * 98 | * \return The requested value. 99 | **/ 100 | guint64 j_statistics_get(JStatistics* statistics, JStatisticsType type); 101 | 102 | /** 103 | * Adds a value to a statistic. 104 | * 105 | * \private 106 | * 107 | * \code 108 | * \endcode 109 | * 110 | * \param statistics A statistics. 111 | * \param type A statistics type. 112 | * \param value A value to add to the statistics. 113 | * 114 | **/ 115 | void j_statistics_add(JStatistics* statistics, JStatisticsType type, guint64 value); 116 | 117 | /** 118 | * @} 119 | **/ 120 | 121 | G_END_DECLS 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /test/core/list-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "test.h" 26 | 27 | static void 28 | test_list_iterator_fixture_setup(JListIterator** iterator, gconstpointer data) 29 | { 30 | g_autoptr(JList) list = NULL; 31 | 32 | (void)data; 33 | 34 | list = j_list_new(g_free); 35 | 36 | j_list_append(list, g_strdup("0")); 37 | j_list_append(list, g_strdup("1")); 38 | j_list_append(list, g_strdup("2")); 39 | 40 | *iterator = j_list_iterator_new(list); 41 | } 42 | 43 | static void 44 | test_list_iterator_fixture_teardown(JListIterator** iterator, gconstpointer data) 45 | { 46 | (void)data; 47 | 48 | j_list_iterator_free(*iterator); 49 | } 50 | 51 | static void 52 | test_list_iterator_new_free(void) 53 | { 54 | guint const n = 100000; 55 | 56 | J_TEST_TRAP_START; 57 | for (guint i = 0; i < n; i++) 58 | { 59 | g_autoptr(JList) list = NULL; 60 | g_autoptr(JListIterator) iterator = NULL; 61 | 62 | list = j_list_new(NULL); 63 | g_assert_true(list != NULL); 64 | 65 | iterator = j_list_iterator_new(list); 66 | g_assert_true(iterator != NULL); 67 | } 68 | J_TEST_TRAP_END; 69 | } 70 | 71 | static void 72 | test_list_iterator_next_get(JListIterator** iterator, gconstpointer data) 73 | { 74 | gchar const* s; 75 | gboolean next; 76 | 77 | (void)data; 78 | 79 | J_TEST_TRAP_START; 80 | next = j_list_iterator_next(*iterator); 81 | g_assert_true(next); 82 | 83 | s = j_list_iterator_get(*iterator); 84 | g_assert_cmpstr(s, ==, "0"); 85 | 86 | next = j_list_iterator_next(*iterator); 87 | g_assert_true(next); 88 | 89 | s = j_list_iterator_get(*iterator); 90 | g_assert_cmpstr(s, ==, "1"); 91 | 92 | next = j_list_iterator_next(*iterator); 93 | g_assert_true(next); 94 | 95 | s = j_list_iterator_get(*iterator); 96 | g_assert_cmpstr(s, ==, "2"); 97 | 98 | next = j_list_iterator_next(*iterator); 99 | g_assert_true(!next); 100 | J_TEST_TRAP_END; 101 | } 102 | 103 | void 104 | test_core_list_iterator(void) 105 | { 106 | g_test_add_func("/core/list-iterator/new_free", test_list_iterator_new_free); 107 | g_test_add("/core/list-iterator/next_get", JListIterator*, NULL, test_list_iterator_fixture_setup, test_list_iterator_next_get, test_list_iterator_fixture_teardown); 108 | } 109 | -------------------------------------------------------------------------------- /test/item/item-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "test.h" 27 | 28 | static void 29 | test_item_iterator_new_free(void) 30 | { 31 | guint const n = 100000; 32 | 33 | J_TEST_TRAP_START; 34 | for (guint i = 0; i < n; i++) 35 | { 36 | g_autoptr(JBatch) batch = NULL; 37 | g_autoptr(JCollection) collection = NULL; 38 | g_autoptr(JItem) item = NULL; 39 | 40 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 41 | collection = j_collection_create("test-collection", batch); 42 | item = j_item_create(collection, "test-item", NULL, batch); 43 | 44 | g_assert_true(item != NULL); 45 | } 46 | J_TEST_TRAP_END; 47 | } 48 | 49 | static void 50 | test_item_iterator_next_get(void) 51 | { 52 | guint const n = 1000; 53 | 54 | g_autoptr(JBatch) batch = NULL; 55 | g_autoptr(JBatch) delete_batch = NULL; 56 | g_autoptr(JCollection) collection = NULL; 57 | g_autoptr(JItemIterator) item_iterator = NULL; 58 | gboolean ret; 59 | 60 | guint items = 0; 61 | 62 | J_TEST_TRAP_START; 63 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 64 | delete_batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 65 | collection = j_collection_create("test-collection", batch); 66 | 67 | for (guint i = 0; i < n; i++) 68 | { 69 | g_autoptr(JItem) item = NULL; 70 | 71 | g_autofree gchar* name = NULL; 72 | 73 | name = g_strdup_printf("test-item-%d", i); 74 | item = j_item_create(collection, name, NULL, batch); 75 | j_item_delete(item, delete_batch); 76 | 77 | g_assert_true(item != NULL); 78 | } 79 | 80 | ret = j_batch_execute(batch); 81 | g_assert_true(ret); 82 | 83 | item_iterator = j_item_iterator_new(collection); 84 | 85 | while (j_item_iterator_next(item_iterator)) 86 | { 87 | g_autoptr(JItem) item = j_item_iterator_get(item_iterator); 88 | (void)item; 89 | items++; 90 | } 91 | 92 | ret = j_batch_execute(delete_batch); 93 | g_assert_true(ret); 94 | 95 | g_assert_cmpuint(items, ==, n); 96 | J_TEST_TRAP_END; 97 | } 98 | 99 | void 100 | test_item_item_iterator(void) 101 | { 102 | g_test_add_func("/item/item-iterator/new_free", test_item_iterator_new_free); 103 | g_test_add_func("/item/item-iterator/next_get", test_item_iterator_next_get); 104 | } 105 | -------------------------------------------------------------------------------- /include/core/jcredentials.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_CREDENTIALS_H 24 | #define JULEA_CREDENTIALS_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | /** 37 | * \defgroup JCredentials Credentials 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JCredentials; 43 | 44 | typedef struct JCredentials JCredentials; 45 | 46 | /** 47 | * Create new credential for the current user. 48 | * 49 | * \return A new JCredentials object. Should be freed with j_credentials_unref(). 50 | **/ 51 | JCredentials* j_credentials_new(void); 52 | 53 | /** 54 | * Increases the credential's reference count. 55 | * 56 | * \param credentials A credentials object. 57 | * 58 | * \return The credentials object. 59 | **/ 60 | JCredentials* j_credentials_ref(JCredentials* credentials); 61 | 62 | /** 63 | * Decreases the credential's reference count. 64 | * 65 | * \param credentials A credentials object. 66 | * 67 | **/ 68 | void j_credentials_unref(JCredentials* credentials); 69 | 70 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JCredentials, j_credentials_unref) 71 | 72 | /** 73 | * Get the user id associated with the credentials 74 | * 75 | * \param credentials A credentials object. 76 | * 77 | * \return The user id. 78 | **/ 79 | guint32 j_credentials_get_user(JCredentials* credentials); 80 | 81 | /** 82 | * Get the group id associated with the credentials 83 | * 84 | * \param credentials A credentials object. 85 | * 86 | * \return The group id. 87 | **/ 88 | guint32 j_credentials_get_group(JCredentials* credentials); 89 | 90 | /** 91 | * Serializes credentials. 92 | * 93 | * \code 94 | * \endcode 95 | * 96 | * \param credentials Credentials. 97 | * 98 | * \return A new BSON object. Should be freed with bson_destroy(). 99 | **/ 100 | bson_t* j_credentials_serialize(JCredentials* credentials); 101 | 102 | /** 103 | * Deserializes credentials. 104 | * 105 | * \code 106 | * \endcode 107 | * 108 | * \param credentials credentials. 109 | * \param b A BSON object. 110 | **/ 111 | void j_credentials_deserialize(JCredentials* credentials, bson_t const* b); 112 | 113 | /** 114 | * @} 115 | **/ 116 | 117 | G_END_DECLS 118 | 119 | #endif 120 | -------------------------------------------------------------------------------- /test/item/collection-iterator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "test.h" 27 | 28 | static void 29 | test_collection_iterator_new_free(void) 30 | { 31 | guint const n = 100000; 32 | 33 | J_TEST_TRAP_START; 34 | for (guint i = 0; i < n; i++) 35 | { 36 | g_autoptr(JBatch) batch = NULL; 37 | g_autoptr(JCollection) collection = NULL; 38 | 39 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 40 | collection = j_collection_create("test-collection-iterator", batch); 41 | 42 | g_assert_true(collection != NULL); 43 | } 44 | J_TEST_TRAP_END; 45 | } 46 | 47 | static void 48 | test_collection_iterator_next_get(void) 49 | { 50 | guint const n = 1000; 51 | 52 | g_autoptr(JBatch) batch = NULL; 53 | g_autoptr(JBatch) delete_batch = NULL; 54 | g_autoptr(JCollectionIterator) collection_iterator = NULL; 55 | gboolean ret; 56 | 57 | guint collections = 0; 58 | 59 | J_TEST_TRAP_START; 60 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 61 | delete_batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_DEFAULT); 62 | 63 | for (guint i = 0; i < n; i++) 64 | { 65 | g_autoptr(JCollection) collection = NULL; 66 | 67 | g_autofree gchar* name = NULL; 68 | 69 | name = g_strdup_printf("test-collection-iterator-%d", i); 70 | collection = j_collection_create(name, batch); 71 | j_collection_delete(collection, delete_batch); 72 | 73 | g_assert_true(collection != NULL); 74 | } 75 | 76 | ret = j_batch_execute(batch); 77 | g_assert_true(ret); 78 | 79 | collection_iterator = j_collection_iterator_new(); 80 | 81 | while (j_collection_iterator_next(collection_iterator)) 82 | { 83 | g_autoptr(JCollection) collection = j_collection_iterator_get(collection_iterator); 84 | 85 | if (!g_str_has_prefix(j_collection_get_name(collection), "test-collection-iterator-")) 86 | { 87 | continue; 88 | } 89 | 90 | collections++; 91 | } 92 | 93 | ret = j_batch_execute(delete_batch); 94 | g_assert_true(ret); 95 | 96 | g_assert_cmpuint(collections, ==, n); 97 | J_TEST_TRAP_END; 98 | } 99 | 100 | void 101 | test_item_collection_iterator(void) 102 | { 103 | g_test_add_func("/item/collection-iterator/new_free", test_collection_iterator_new_free); 104 | g_test_add_func("/item/collection-iterator/next_get", test_collection_iterator_next_get); 105 | } 106 | -------------------------------------------------------------------------------- /include/object/jobject-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2017-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_OBJECT_OBJECT_ITERATOR_H 24 | #define JULEA_OBJECT_OBJECT_ITERATOR_H 25 | 26 | #if !defined(JULEA_OBJECT_H) && !defined(JULEA_OBJECT_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JObjectIterator Object Iterator 36 | * 37 | * Data structures and functions for iterating over objects. 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JObjectIterator; 43 | 44 | typedef struct JObjectIterator JObjectIterator; 45 | 46 | /** 47 | * Creates a new JObjectIterator. 48 | * 49 | * \param namespace The namespace to iterate over. 50 | * \param prefix Prefix of names to iterate over. Set to NULL to iterate over all objects in the namespace. 51 | * 52 | * \return A new JObjectIterator. 53 | **/ 54 | JObjectIterator* j_object_iterator_new(gchar const* namespace, gchar const* prefix); 55 | 56 | /** 57 | * Creates a new JObjectIterator on a specific object server. 58 | * 59 | * \param index Server to query. 60 | * \param namespace JKV namespace to iterate over. 61 | * \param prefix Prefix of names to iterate over. Set to NULL to iterate over all objects in the namespace. 62 | * 63 | * \return A new JObjectIterator. 64 | **/ 65 | JObjectIterator* j_object_iterator_new_for_index(guint32 index, gchar const* namespace, gchar const* prefix); 66 | 67 | /** 68 | * Frees the memory allocated by the JObjectIterator. 69 | * 70 | * \param iterator A JObjectIterator. 71 | **/ 72 | void j_object_iterator_free(JObjectIterator* iterator); 73 | 74 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JObjectIterator, j_object_iterator_free) 75 | 76 | /** 77 | * Checks whether another collection is available. 78 | * 79 | * \code 80 | * \endcode 81 | * 82 | * \param iterator A store iterator. 83 | * 84 | * \return TRUE on success, FALSE if the end of the store is reached. 85 | **/ 86 | gboolean j_object_iterator_next(JObjectIterator* iterator); 87 | 88 | /** 89 | * Returns the current collection. 90 | * 91 | * \code 92 | * \endcode 93 | * 94 | * \param iterator A store iterator. 95 | * 96 | * \return A new collection. Should be freed with j_object_unref(). 97 | **/ 98 | gchar const* j_object_iterator_get(JObjectIterator* iterator); 99 | 100 | /** 101 | * @} 102 | **/ 103 | 104 | G_END_DECLS 105 | 106 | #endif 107 | -------------------------------------------------------------------------------- /include/kv/jkv-iterator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2017-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_KV_KV_ITERATOR_H 24 | #define JULEA_KV_KV_ITERATOR_H 25 | 26 | #if !defined(JULEA_KV_H) && !defined(JULEA_KV_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JKVIterator KV Iterator 36 | * 37 | * Data structures and functions for iterating over key-value pairs. 38 | * 39 | * @{ 40 | **/ 41 | 42 | struct JKVIterator; 43 | 44 | typedef struct JKVIterator JKVIterator; 45 | 46 | G_END_DECLS 47 | 48 | #include 49 | 50 | G_BEGIN_DECLS 51 | 52 | /** 53 | * Creates a new JKVIterator. 54 | * 55 | * \param namespace JKV namespace to iterate over. 56 | * \param prefix Prefix of keys to iterate over. Set to NULL to iterate over all KVs. 57 | * 58 | * \return A new JKVIterator. 59 | **/ 60 | JKVIterator* j_kv_iterator_new(gchar const* namespace, gchar const* prefix); 61 | 62 | /** 63 | * Creates a new JKVIterator on a specific KV server. 64 | * 65 | * \param index Server to query. 66 | * \param namespace JKV namespace to iterate over. 67 | * \param prefix Prefix of keys to iterate over. Set to NULL to iterate over all KVs. 68 | * 69 | * \return A new JKVIterator. 70 | **/ 71 | JKVIterator* j_kv_iterator_new_for_index(guint32 index, gchar const* namespace, gchar const* prefix); 72 | 73 | /** 74 | * Frees the memory allocated by the JKVIterator. 75 | * 76 | * \param iterator A JKVIterator. 77 | **/ 78 | void j_kv_iterator_free(JKVIterator* iterator); 79 | 80 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JKVIterator, j_kv_iterator_free) 81 | 82 | /** 83 | * Checks whether another collection is available. 84 | * 85 | * \code 86 | * \endcode 87 | * 88 | * \param iterator A store iterator. 89 | * 90 | * \return TRUE on success, FALSE if the end of the store is reached. 91 | **/ 92 | gboolean j_kv_iterator_next(JKVIterator* iterator); 93 | 94 | /** 95 | * Returns the current collection. 96 | * 97 | * \code 98 | * \endcode 99 | * 100 | * \param iterator A store iterator. 101 | * \param value A pointer to be set to the current value. 102 | * \param len Will be set to the length of the current value. 103 | * 104 | * \return The current key. 105 | **/ 106 | gchar const* j_kv_iterator_get(JKVIterator* iterator, gconstpointer* value, guint32* len); 107 | 108 | /** 109 | * @} 110 | **/ 111 | 112 | G_END_DECLS 113 | 114 | #endif 115 | -------------------------------------------------------------------------------- /include/db-util/jbson.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2019 Benjamin Warnke 4 | * Copyright (C) 2022 Timm Leon Erxleben 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU Lesser General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU Lesser General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU Lesser General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /** 21 | * \file 22 | **/ 23 | 24 | #ifndef JULEA_DB_UTIL_BSON_H 25 | #define JULEA_DB_UTIL_BSON_H 26 | 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | 33 | gboolean j_bson_init(bson_t* bson, GError** error); 34 | void j_bson_destroy(bson_t* bson); 35 | 36 | gboolean j_bson_has_field(bson_t* bson, gchar const* name, gboolean* has_field, GError** error); 37 | gboolean j_bson_has_enough_keys(const bson_t* bson, guint32 min_keys, GError** error); 38 | gboolean j_bson_count_keys(bson_t* bson, guint32* count, GError** error); 39 | gboolean j_bson_array_generate_key(guint32 index, const char** key, char* buf, guint buf_length, GError** error); 40 | gboolean j_bson_append_array(bson_t* bson, const char* key, bson_t* bson_child, GError** error); 41 | gboolean j_bson_append_array_begin(bson_t* bson, const char* key, bson_t* bson_child, GError** error); 42 | gboolean j_bson_append_array_end(bson_t* bson, bson_t* bson_child, GError** error); 43 | gboolean j_bson_append_document(bson_t* bson, const char* key, bson_t* bson_child, GError** error); 44 | gboolean j_bson_append_document_begin(bson_t* bson, const char* key, bson_t* bson_child, GError** error); 45 | gboolean j_bson_append_document_end(bson_t* bson, bson_t* bson_child, GError** error); 46 | gboolean j_bson_append_value(bson_t* bson, const char* name, JDBType type, JDBTypeValue* value, GError** error); 47 | 48 | gboolean j_bson_iter_init(bson_iter_t* iter, const bson_t* bson, GError** error); 49 | gboolean j_bson_iter_next(bson_iter_t* iter, gboolean* has_next, GError** error); 50 | gboolean j_bson_iter_key_equals(bson_iter_t* iter, const char* key, gboolean* equals, GError** error); 51 | gboolean j_bson_iter_skip_key(bson_iter_t* iter, const char* key, GError** error); 52 | const char* j_bson_iter_key(bson_iter_t* iter, GError** error); 53 | gboolean j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** error); 54 | gboolean j_bson_iter_find(bson_iter_t* iter, const char* key, GError** error); 55 | gboolean j_bson_iter_not_find(bson_iter_t* iter, const char* key, GError** error); 56 | gboolean j_bson_iter_recurse_array(bson_iter_t* iter, bson_iter_t* iter_child, GError** error); 57 | gboolean j_bson_iter_recurse_document(bson_iter_t* iter, bson_iter_t* iter_child, GError** error); 58 | gboolean j_bson_iter_copy_document(bson_iter_t* iter, bson_t* bson, GError** error); 59 | 60 | // enables the use of g_autoptr(bson_t) 61 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(bson_t, j_bson_destroy) 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /fuse/getattr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include "julea-fuse.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | int 29 | jfs_getattr(char const* path, struct stat* stbuf, struct fuse_file_info* fi) 30 | { 31 | int ret = -ENOENT; 32 | 33 | g_autoptr(JBatch) batch = NULL; 34 | g_autoptr(JKV) kv = NULL; 35 | gpointer value; 36 | guint32 len; 37 | 38 | (void)fi; 39 | 40 | if (g_strcmp0(path, "/") == 0) 41 | { 42 | stbuf->st_mode = S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; 43 | stbuf->st_nlink = 1; 44 | stbuf->st_uid = 0; 45 | stbuf->st_gid = 0; 46 | stbuf->st_size = 0; 47 | stbuf->st_atime = stbuf->st_ctime = stbuf->st_mtime = g_get_real_time() / G_USEC_PER_SEC; 48 | 49 | return 0; 50 | } 51 | 52 | batch = j_batch_new_for_template(J_SEMANTICS_TEMPLATE_POSIX); 53 | kv = j_kv_new("posix", path); 54 | 55 | j_kv_get(kv, &value, &len, batch); 56 | 57 | if (j_batch_execute(batch)) 58 | { 59 | bson_t file[1]; 60 | bson_iter_t iter; 61 | gboolean is_file = TRUE; 62 | gint64 size = 0; 63 | gint64 time = 0; 64 | 65 | bson_init_static(file, value, len); 66 | bson_iter_init(&iter, file); 67 | 68 | while (bson_iter_next(&iter)) 69 | { 70 | gchar const* key; 71 | 72 | key = bson_iter_key(&iter); 73 | 74 | if (g_strcmp0(key, "file") == 0) 75 | { 76 | is_file = bson_iter_bool(&iter); 77 | } 78 | else if (g_strcmp0(key, "size") == 0) 79 | { 80 | size = bson_iter_int64(&iter); 81 | } 82 | else if (g_strcmp0(key, "time") == 0) 83 | { 84 | time = bson_iter_int64(&iter); 85 | } 86 | } 87 | 88 | if (is_file) 89 | { 90 | stbuf->st_mode = S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; 91 | stbuf->st_nlink = 1; 92 | stbuf->st_uid = getuid(); 93 | stbuf->st_gid = getgid(); 94 | stbuf->st_size = size; 95 | stbuf->st_atime = stbuf->st_ctime = stbuf->st_mtime = time / G_USEC_PER_SEC; 96 | 97 | ret = 0; 98 | } 99 | else 100 | { 101 | stbuf->st_mode = S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; 102 | stbuf->st_nlink = 1; 103 | stbuf->st_uid = getuid(); 104 | stbuf->st_gid = getgid(); 105 | stbuf->st_size = 0; 106 | stbuf->st_atime = stbuf->st_ctime = stbuf->st_mtime = time / G_USEC_PER_SEC; 107 | 108 | ret = 0; 109 | } 110 | 111 | bson_destroy(file); 112 | g_free(value); 113 | } 114 | 115 | return ret; 116 | } 117 | -------------------------------------------------------------------------------- /include/core/jlist.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_LIST_H 24 | #define JULEA_LIST_H 25 | 26 | #if !defined(JULEA_H) && !defined(JULEA_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | G_BEGIN_DECLS 33 | 34 | /** 35 | * \defgroup JList List 36 | * 37 | * @{ 38 | **/ 39 | 40 | struct JList; 41 | 42 | typedef struct JList JList; 43 | 44 | typedef void (*JListFreeFunc)(gpointer); 45 | 46 | /** 47 | * Creates a new list. 48 | * 49 | * \code 50 | * \endcode 51 | * 52 | * \param free_func A function to free the element data, or NULL. 53 | * 54 | * \return A new list. 55 | **/ 56 | JList* j_list_new(JListFreeFunc free_func); 57 | 58 | /** 59 | * Increases the list's reference count. 60 | * 61 | * \param list A list. 62 | * 63 | * \return The list. 64 | **/ 65 | JList* j_list_ref(JList* list); 66 | 67 | /** 68 | * Decreases the list's reference count. 69 | * When the reference count reaches zero, frees the memory allocated for the list. 70 | * 71 | * \code 72 | * \endcode 73 | * 74 | * \param list A list. 75 | **/ 76 | void j_list_unref(JList* list); 77 | 78 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JList, j_list_unref) 79 | 80 | /** 81 | * Returns the list's length. 82 | * 83 | * \code 84 | * \endcode 85 | * 86 | * \param list A list. 87 | * 88 | * \return The list's length. 89 | **/ 90 | guint j_list_length(JList* list); 91 | 92 | /** 93 | * Appends a new list element to a list. 94 | * 95 | * \code 96 | * \endcode 97 | * 98 | * \param list A list. 99 | * \param data A list element. 100 | **/ 101 | void j_list_append(JList* list, gpointer data); 102 | 103 | /** 104 | * Prepends a new list element to a list. 105 | * 106 | * \code 107 | * \endcode 108 | * 109 | * \param list A list. 110 | * \param data A list element. 111 | **/ 112 | void j_list_prepend(JList* list, gpointer data); 113 | 114 | /** 115 | * Returns the first list element. 116 | * 117 | * \param list A list. 118 | * 119 | * \return A list element, or NULL. 120 | **/ 121 | gpointer j_list_get_first(JList* list); 122 | 123 | /** 124 | * Returns the last list element. 125 | * 126 | * \param list A list. 127 | * 128 | * \return A list element, or NULL. 129 | **/ 130 | gpointer j_list_get_last(JList* list); 131 | 132 | /** 133 | * Deletes all list elements. 134 | * 135 | * \param list A list. 136 | **/ 137 | void j_list_delete_all(JList* list); 138 | 139 | /** 140 | * @} 141 | **/ 142 | 143 | G_END_DECLS 144 | 145 | #endif 146 | -------------------------------------------------------------------------------- /include/item/jcollection.h: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #ifndef JULEA_ITEM_COLLECTION_H 24 | #define JULEA_ITEM_COLLECTION_H 25 | 26 | #if !defined(JULEA_ITEM_H) && !defined(JULEA_ITEM_COMPILATION) 27 | #error "Only can be included directly." 28 | #endif 29 | 30 | #include 31 | 32 | #include 33 | 34 | G_BEGIN_DECLS 35 | 36 | struct JCollection; 37 | 38 | typedef struct JCollection JCollection; 39 | 40 | G_END_DECLS 41 | 42 | #include 43 | 44 | G_BEGIN_DECLS 45 | 46 | /** 47 | * \defgroup JCollection Collection 48 | * 49 | * Data structures and functions for managing collections. 50 | * 51 | * @{ 52 | **/ 53 | 54 | /** 55 | * Increases a collection's reference count. 56 | * 57 | * \code 58 | * JCollection* c; 59 | * 60 | * j_collection_ref(c); 61 | * \endcode 62 | * 63 | * \param collection A collection. 64 | * 65 | * \return \p collection. 66 | **/ 67 | JCollection* j_collection_ref(JCollection* collection); 68 | 69 | /** 70 | * Decreases a collection's reference count. 71 | * When the reference count reaches zero, frees the memory allocated for the collection. 72 | * 73 | * \code 74 | * \endcode 75 | * 76 | * \param collection A collection. 77 | **/ 78 | void j_collection_unref(JCollection* collection); 79 | 80 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(JCollection, j_collection_unref) 81 | 82 | /** 83 | * Returns a collection's name. 84 | * 85 | * \code 86 | * \endcode 87 | * 88 | * \param collection A collection. 89 | * 90 | * \return A collection name. 91 | **/ 92 | gchar const* j_collection_get_name(JCollection* collection); 93 | 94 | /** 95 | * Creates a collection. 96 | * 97 | * \code 98 | * \endcode 99 | * 100 | * \param name A name for the new collection. 101 | * \param batch A batch. 102 | **/ 103 | JCollection* j_collection_create(gchar const* name, JBatch* batch); 104 | 105 | /** 106 | * Gets a collection. 107 | * 108 | * \code 109 | * \endcode 110 | * 111 | * \param collection A pointer to a collection. 112 | * \param name A name. 113 | * \param batch A batch. 114 | **/ 115 | void j_collection_get(JCollection** collection, gchar const* name, JBatch* batch); 116 | 117 | /** 118 | * Deletes a collection. 119 | * 120 | * \code 121 | * \endcode 122 | * 123 | * \param collection A collection. 124 | * \param batch A batch. 125 | **/ 126 | void j_collection_delete(JCollection* collection, JBatch* batch); 127 | 128 | /** 129 | * @} 130 | **/ 131 | 132 | G_END_DECLS 133 | 134 | #endif 135 | -------------------------------------------------------------------------------- /lib/core/jcache.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | 31 | #include 32 | 33 | /** 34 | * \addtogroup JCache Cache 35 | * 36 | * @{ 37 | **/ 38 | 39 | /** 40 | * A cache. 41 | */ 42 | struct JCache 43 | { 44 | /** 45 | * The size. 46 | */ 47 | guint64 size; 48 | 49 | GHashTable* buffers; 50 | 51 | guint64 used; 52 | 53 | GMutex mutex[1]; 54 | }; 55 | 56 | JCache* 57 | j_cache_new(guint64 size) 58 | { 59 | J_TRACE_FUNCTION(NULL); 60 | 61 | JCache* cache; 62 | 63 | g_return_val_if_fail(size > 0, NULL); 64 | 65 | cache = g_new(JCache, 1); 66 | cache->size = size; 67 | cache->buffers = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); 68 | cache->used = 0; 69 | 70 | g_mutex_init(cache->mutex); 71 | 72 | return cache; 73 | } 74 | 75 | void 76 | j_cache_free(JCache* cache) 77 | { 78 | J_TRACE_FUNCTION(NULL); 79 | 80 | GHashTableIter iter[1]; 81 | gpointer key; 82 | gpointer value; 83 | 84 | g_return_if_fail(cache != NULL); 85 | 86 | g_hash_table_iter_init(iter, cache->buffers); 87 | 88 | while (g_hash_table_iter_next(iter, &key, &value)) 89 | { 90 | g_free(key); 91 | } 92 | 93 | g_hash_table_unref(cache->buffers); 94 | 95 | g_mutex_clear(cache->mutex); 96 | 97 | g_free(cache); 98 | } 99 | 100 | gpointer 101 | j_cache_get(JCache* cache, guint64 length) 102 | { 103 | J_TRACE_FUNCTION(NULL); 104 | 105 | gpointer ret = NULL; 106 | 107 | g_return_val_if_fail(cache != NULL, NULL); 108 | 109 | g_mutex_lock(cache->mutex); 110 | 111 | if (cache->used + length > cache->size) 112 | { 113 | goto end; 114 | } 115 | 116 | ret = g_malloc(length); 117 | cache->used += length; 118 | 119 | g_hash_table_insert(cache->buffers, ret, GSIZE_TO_POINTER(length)); 120 | 121 | end: 122 | g_mutex_unlock(cache->mutex); 123 | 124 | return ret; 125 | } 126 | 127 | void 128 | j_cache_release(JCache* cache, gpointer data) 129 | { 130 | J_TRACE_FUNCTION(NULL); 131 | 132 | gpointer size; 133 | 134 | g_return_if_fail(cache != NULL); 135 | g_return_if_fail(data != NULL); 136 | 137 | g_mutex_lock(cache->mutex); 138 | 139 | if ((size = g_hash_table_lookup(cache->buffers, data)) == NULL) 140 | { 141 | g_warn_if_reached(); 142 | return; 143 | } 144 | 145 | g_hash_table_remove(cache->buffers, data); 146 | 147 | cache->used -= GPOINTER_TO_SIZE(size); 148 | g_free(data); 149 | 150 | g_mutex_unlock(cache->mutex); 151 | } 152 | 153 | /** 154 | * @} 155 | **/ 156 | -------------------------------------------------------------------------------- /benchmark/message.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2010-2024 Michael Kuhn 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "benchmark.h" 28 | 29 | static void 30 | _benchmark_message_new(BenchmarkRun* run, gboolean append) 31 | { 32 | guint const n = 100000; 33 | guint const m = 100; 34 | guint64 const dummy = 42; 35 | gsize const size = m * sizeof(guint64); 36 | 37 | j_benchmark_timer_start(run); 38 | 39 | while (j_benchmark_iterate(run)) 40 | { 41 | for (guint i = 0; i < n; i++) 42 | { 43 | g_autoptr(JMessage) message = NULL; 44 | 45 | message = j_message_new(J_MESSAGE_NONE, (append) ? size : 0); 46 | 47 | if (append) 48 | { 49 | for (guint j = 0; j < m; j++) 50 | { 51 | j_message_append_8(message, &dummy); 52 | } 53 | } 54 | } 55 | } 56 | 57 | j_benchmark_timer_stop(run); 58 | 59 | run->operations = n; 60 | } 61 | 62 | static void 63 | benchmark_message_new(BenchmarkRun* run) 64 | { 65 | _benchmark_message_new(run, FALSE); 66 | } 67 | 68 | static void 69 | benchmark_message_new_append(BenchmarkRun* run) 70 | { 71 | _benchmark_message_new(run, TRUE); 72 | } 73 | 74 | static void 75 | _benchmark_message_add_operation(BenchmarkRun* run, gboolean large) 76 | { 77 | guint const n = (large) ? 100 : 10000; 78 | guint const m = (large) ? 10000 : 100; 79 | guint64 const dummy = 42; 80 | 81 | j_benchmark_timer_start(run); 82 | 83 | while (j_benchmark_iterate(run)) 84 | { 85 | for (guint i = 0; i < n; i++) 86 | { 87 | g_autoptr(JMessage) message = NULL; 88 | 89 | message = j_message_new(J_MESSAGE_NONE, 0); 90 | 91 | for (guint j = 0; j < m; j++) 92 | { 93 | j_message_add_operation(message, sizeof(guint64)); 94 | j_message_append_8(message, &dummy); 95 | } 96 | } 97 | } 98 | 99 | j_benchmark_timer_stop(run); 100 | 101 | run->operations = n; 102 | run->bytes = n * m * sizeof(guint64); 103 | } 104 | 105 | static void 106 | benchmark_message_add_operation_small(BenchmarkRun* run) 107 | { 108 | _benchmark_message_add_operation(run, FALSE); 109 | } 110 | 111 | static void 112 | benchmark_message_add_operation_large(BenchmarkRun* run) 113 | { 114 | _benchmark_message_add_operation(run, TRUE); 115 | } 116 | 117 | void 118 | benchmark_message(void) 119 | { 120 | j_benchmark_add("/message/new", benchmark_message_new); 121 | j_benchmark_add("/message/new-append", benchmark_message_new_append); 122 | j_benchmark_add("/message/add-operation-small", benchmark_message_add_operation_small); 123 | j_benchmark_add("/message/add-operation-large", benchmark_message_add_operation_large); 124 | } 125 | -------------------------------------------------------------------------------- /lib/hdf5-db/jhdf5-db-object.c: -------------------------------------------------------------------------------- 1 | /* 2 | * JULEA - Flexible storage framework 3 | * Copyright (C) 2021-2022 Timm Leon Erxleben 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Lesser General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Lesser General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Lesser General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | /** 20 | * \file 21 | **/ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include 38 | 39 | #include 40 | #include 41 | #include 42 | 43 | #include "jhdf5-db.h" 44 | 45 | void* 46 | H5VL_julea_db_object_open(void* obj, const H5VL_loc_params_t* loc_params, H5I_type_t* opened_type, hid_t dxpl, void** req) 47 | { 48 | JHDF5Object_t* parent = obj; 49 | JHDF5Object_t* file; 50 | JHDF5Object_t* child = H5VL_julea_db_object_new(0); // type is not relevant here and will be set by link get 51 | 52 | (void)dxpl; 53 | (void)req; 54 | 55 | g_return_val_if_fail(loc_params->type == H5VL_OBJECT_BY_NAME, NULL); 56 | 57 | switch (parent->type) 58 | { 59 | case J_HDF5_OBJECT_TYPE_FILE: 60 | file = parent; 61 | break; 62 | case J_HDF5_OBJECT_TYPE_GROUP: 63 | file = parent->group.file; 64 | break; 65 | case J_HDF5_OBJECT_TYPE_DATASET: 66 | case J_HDF5_OBJECT_TYPE_ATTR: 67 | case J_HDF5_OBJECT_TYPE_DATATYPE: 68 | case J_HDF5_OBJECT_TYPE_SPACE: 69 | case _J_HDF5_OBJECT_TYPE_COUNT: 70 | default: 71 | g_debug("%s: Unsupported parent type for object open!", G_STRFUNC); 72 | j_goto_error(); 73 | } 74 | 75 | if (!H5VL_julea_db_link_get_helper(obj, child, loc_params->loc_data.loc_by_name.name)) 76 | { 77 | g_debug("%s: Could not find object!", G_STRFUNC); 78 | j_goto_error(); 79 | } 80 | 81 | switch (child->type) 82 | { 83 | case J_HDF5_OBJECT_TYPE_GROUP: 84 | *opened_type = H5I_GROUP; 85 | child->group.file = H5VL_julea_db_object_ref(file); 86 | child->group.name = g_strdup(loc_params->loc_data.loc_by_name.name); 87 | break; 88 | 89 | case J_HDF5_OBJECT_TYPE_DATASET: 90 | *opened_type = H5I_DATASET; 91 | child->dataset.file = H5VL_julea_db_object_ref(file); 92 | child->dataset.name = g_strdup(loc_params->loc_data.loc_by_name.name); 93 | if (!H5VL_julea_db_dataset_set_info(child, NULL)) 94 | { 95 | g_debug("%s: Could not set dataset metadata!", G_STRFUNC); 96 | j_goto_error(); 97 | } 98 | break; 99 | 100 | case J_HDF5_OBJECT_TYPE_DATATYPE: 101 | /// \todo implement when committed datatypes are supported 102 | case J_HDF5_OBJECT_TYPE_FILE: 103 | case J_HDF5_OBJECT_TYPE_ATTR: 104 | case J_HDF5_OBJECT_TYPE_SPACE: 105 | case _J_HDF5_OBJECT_TYPE_COUNT: 106 | default: 107 | g_debug("Unsupported type for H5Open()"); 108 | j_goto_error(); 109 | } 110 | 111 | return child; 112 | 113 | _error: 114 | H5VL_julea_db_object_unref(child); 115 | return NULL; 116 | } 117 | --------------------------------------------------------------------------------