├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Doxyfile ├── LICENSES └── GPL-2.0-only.txt ├── Makefile ├── README.md ├── SBOM.json ├── doc ├── errors.txt ├── scanoss-logo.png └── style.css ├── package.sh ├── run_test.sh ├── scripts ├── debpkg │ └── DEBIAN │ │ └── control └── rpmpkg │ └── ldb.spec ├── src ├── bsort.c ├── bsort.h ├── collate.c ├── command.c ├── command.h ├── config.c ├── decode.c ├── decode.h ├── dump.c ├── file.c ├── hex.c ├── ignored.h ├── import.c ├── import.h ├── join.c ├── join.h ├── keys.c ├── ldb.c ├── ldb.h ├── ldb │ ├── collate.h │ ├── definitions.h │ ├── mz.h │ └── types.h ├── ldb_error.h ├── ldb_string.c ├── ldb_string.h ├── ldb_wrapper.c ├── ldb_wrapper.h ├── lock.c ├── logger.c ├── logger.h ├── md5.c ├── mz.c ├── mz_optimise.c ├── node.c ├── pointer.c ├── recordset.c ├── sector.c └── shell.c ├── test ├── bash_unit ├── source │ ├── mined │ │ ├── attribution │ │ │ └── 10.csv │ │ ├── file │ │ │ ├── 00.csv │ │ │ ├── 39.csv │ │ │ ├── 74.csv │ │ │ ├── 85.csv │ │ │ └── b3.csv │ │ ├── license.csv │ │ ├── pivot │ │ │ └── ba.csv │ │ ├── quality.csv │ │ ├── sources │ │ │ ├── 8f2b.mz │ │ │ └── b377.mz │ │ ├── url.csv │ │ ├── version.json │ │ └── wfp │ │ │ ├── 05.bin │ │ │ └── 20.bin │ ├── mined2 │ │ ├── quality.csv │ │ └── version.json │ ├── mined3 │ │ ├── quality.csv │ │ └── version.json │ ├── test_file_commands.txt │ └── test_remove_record.txt └── test_kb └── version.sh /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: [ 'main' ] 7 | push: 8 | branches: 9 | - '*' 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Install third party dependencies 18 | run: sudo apt update && sudo apt install build-essential libgcrypt-dev zlib1g-dev libsqlite3-dev libz-dev curl gem ruby unzip p7zip-full unrar-free 19 | 20 | - uses: actions/checkout@v3 21 | 22 | - name: Install ldb 23 | run: make all 24 | 25 | - name: version 26 | run: echo "::set-output name=version::$(./ldb -v)" 27 | id: version 28 | 29 | - name: Test output 30 | run: echo ${{ steps.version.outputs.version }} 31 | 32 | - name: 'Tar files' 33 | run: tar czvf ldb-${{ steps.version.outputs.version }}-amd64.tar.gz ldb libldb.so 34 | 35 | - name: Upload a Build Artifact 36 | uses: actions/upload-artifact@v4 37 | with: 38 | name: ldb 39 | path: | 40 | ldb-${{ steps.version.outputs.version }}-amd64.tar.gz 41 | retention-days: 5 -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: tagged-release 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | tags: 7 | - "v*.*.*" 8 | 9 | jobs: 10 | tagged-release: 11 | name: "Tagged Release" 12 | runs-on: ubuntu-20.04 13 | #container: 14 | # image: debian:buster 15 | 16 | steps: 17 | 18 | - name: Install third party dependencies 19 | run: sudo apt install build-essential libgcrypt-dev zlib1g-dev 20 | 21 | - uses: actions/checkout@v3 22 | with: 23 | fetch-depth: 0 24 | 25 | - name: Build 26 | run: | 27 | make all 28 | mkdir -p ./artifacts/ 29 | cp ldb ./artifacts/ldb 30 | cp libldb.so ./artifacts/libldb.so 31 | echo "Produced artifact at ${PWD}/artifacts/ldb" 32 | 33 | - name: 'Tar files' 34 | run: tar czvf ldb-${{ github.ref_name }}-linux-amd64.tar.gz -C artifacts/ . 35 | 36 | - name: Prepare deb package 37 | id: build_deb 38 | run: | 39 | make prepare_deb_package 40 | echo "debpkg_file_name=$(ls *.deb)" >> $GITHUB_OUTPUT 41 | 42 | - name: Prepare rpm package 43 | run: | 44 | make prepare_rpm_package 45 | echo "rpmpkg_file_name=$(ls dist/rpm/RPMS/x86_64/*.rpm)" >> $GITHUB_OUTPUT 46 | 47 | - name: Show the artifacts 48 | # Items placed in /artifacts in the container will be in 49 | # ${PWD}/artifacts on the host. 50 | run: | 51 | ls -al "${PWD}/artifacts" 52 | ls -al "${PWD}" 53 | ls -al "${PWD}/dist/rpm/RPMS/x86_64/" 54 | 55 | - name: Create Draft Release ${{ github.ref_type }} - ${{ github.ref_name }} 56 | if: github.ref_type == 'tag' 57 | uses: softprops/action-gh-release@v1 58 | with: 59 | draft: true 60 | files: | 61 | ./*.tar.gz 62 | ./*.deb 63 | ./dist/rpm/RPMS/x86_64/*.rpm 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | doc 4 | .vscode 5 | dist/ 6 | *.deb 7 | /ldb -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | - Using welcoming and inclusive language 18 | - Being respectful of differing viewpoints and experiences 19 | - Gracefully accepting constructive criticism 20 | - Focusing on what is best for the community 21 | - Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | - The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | - Trolling, insulting/derogatory comments, and personal or political attacks 28 | - Public or private harassment 29 | - Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | - Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at support@scanoss.co.uk. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for considering contributing to the SCANOSS LDB. It's people like you that make the SCANOSS LDB such a great tool. Feel welcome and read the following sections in order to know how to get involved, ask questions and more importantly how to work on something. 2 | 3 | The SCANOSS LDB is an open source project and we love to receive contributions from our community. There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests, or writing code. 4 | A welcome addition to the project is an integration with a new source code repository. 5 | 6 | ### Submitting bugs 7 | 8 | If you are submitting a bug, please tell us: 9 | 10 | - Version of SCANOSS LDB you are using 11 | - Linux OS Version 12 | - GCC version 13 | - how to reproduce the bug. 14 | 15 | ### Pull requests 16 | 17 | Want to submit a pull request? Great! But please follow some basic rules: 18 | 19 | - Write a brief description that help us understand what you are trying to accomplish: what the change does, link to any relevant issue 20 | - If you are changing a source file please make sure that you only include in the changeset the lines changed by you (beware of your editor reformatting the file) 21 | - If you are adding functionality, please write a unit test. 22 | 23 | When reviewing your pull request, we will follow a checklist similar to this one: https://gist.github.com/audreyr/4feef90445b9680475f2 24 | 25 | We will also verify that the functionality implemented change serves the general public and not a particular interest group. 26 | 27 | ### Licensing 28 | 29 | The SCANOSS Platform is released under the GPL-2.0 license. If you wish to contribute, you must accept that you are aware of the license under which the project is released, and that your contribution will be released under the same license. Sometimes the GPL-2.0 license is incompatible with other licenses chosen by other projects. Therefore, you must accept that your contribution can also be released under the MIT license, which is the license we choose for those situations. Unless you expressly request otherwise, we may use your name, email address, username or URL for your attribution notice text. The submission of your contribution implies that you agree with these licensing terms. 30 | -------------------------------------------------------------------------------- /Doxyfile: -------------------------------------------------------------------------------- 1 | # Doxyfile 1.9.2 2 | 3 | #--------------------------------------------------------------------------- 4 | # Project related configuration options 5 | #--------------------------------------------------------------------------- 6 | DOXYFILE_ENCODING = UTF-8 7 | PROJECT_NAME = "LDB" 8 | PROJECT_NUMBER = 3.1.1 9 | PROJECT_BRIEF = "Linked-list database" 10 | PROJECT_LOGO = doc/scanoss-logo.png 11 | OUTPUT_DIRECTORY = doc 12 | CREATE_SUBDIRS = NO 13 | ALLOW_UNICODE_NAMES = NO 14 | OUTPUT_LANGUAGE = English 15 | BRIEF_MEMBER_DESC = YES 16 | REPEAT_BRIEF = YES 17 | ABBREVIATE_BRIEF = "The $name class" \ 18 | "The $name widget" \ 19 | "The $name file" \ 20 | is \ 21 | provides \ 22 | specifies \ 23 | contains \ 24 | represents \ 25 | a \ 26 | an \ 27 | the 28 | ALWAYS_DETAILED_SEC = NO 29 | INLINE_INHERITED_MEMB = NO 30 | FULL_PATH_NAMES = YES 31 | STRIP_FROM_PATH = 32 | STRIP_FROM_INC_PATH = 33 | SHORT_NAMES = NO 34 | JAVADOC_AUTOBRIEF = NO 35 | JAVADOC_BANNER = NO 36 | QT_AUTOBRIEF = NO 37 | MULTILINE_CPP_IS_BRIEF = NO 38 | PYTHON_DOCSTRING = YES 39 | INHERIT_DOCS = YES 40 | SEPARATE_MEMBER_PAGES = NO 41 | TAB_SIZE = 4 42 | ALIASES = 43 | OPTIMIZE_OUTPUT_FOR_C = YES 44 | OPTIMIZE_OUTPUT_JAVA = NO 45 | OPTIMIZE_FOR_FORTRAN = NO 46 | OPTIMIZE_OUTPUT_VHDL = NO 47 | OPTIMIZE_OUTPUT_SLICE = NO 48 | EXTENSION_MAPPING = 49 | MARKDOWN_SUPPORT = YES 50 | TOC_INCLUDE_HEADINGS = 5 51 | AUTOLINK_SUPPORT = YES 52 | BUILTIN_STL_SUPPORT = NO 53 | CPP_CLI_SUPPORT = NO 54 | SIP_SUPPORT = NO 55 | IDL_PROPERTY_SUPPORT = YES 56 | DISTRIBUTE_GROUP_DOC = NO 57 | GROUP_NESTED_COMPOUNDS = NO 58 | SUBGROUPING = YES 59 | INLINE_GROUPED_CLASSES = NO 60 | INLINE_SIMPLE_STRUCTS = NO 61 | TYPEDEF_HIDES_STRUCT = NO 62 | LOOKUP_CACHE_SIZE = 0 63 | NUM_PROC_THREADS = 1 64 | #--------------------------------------------------------------------------- 65 | # Build related configuration options 66 | #--------------------------------------------------------------------------- 67 | EXTRACT_ALL = YES 68 | EXTRACT_PRIVATE = YES 69 | EXTRACT_PRIV_VIRTUAL = NO 70 | EXTRACT_PACKAGE = NO 71 | EXTRACT_STATIC = YES 72 | EXTRACT_LOCAL_CLASSES = YES 73 | EXTRACT_LOCAL_METHODS = NO 74 | EXTRACT_ANON_NSPACES = NO 75 | RESOLVE_UNNAMED_PARAMS = YES 76 | HIDE_UNDOC_MEMBERS = NO 77 | HIDE_UNDOC_CLASSES = NO 78 | HIDE_FRIEND_COMPOUNDS = NO 79 | HIDE_IN_BODY_DOCS = NO 80 | INTERNAL_DOCS = NO 81 | CASE_SENSE_NAMES = NO 82 | HIDE_SCOPE_NAMES = YES 83 | HIDE_COMPOUND_REFERENCE= NO 84 | SHOW_HEADERFILE = YES 85 | SHOW_INCLUDE_FILES = YES 86 | SHOW_GROUPED_MEMB_INC = NO 87 | FORCE_LOCAL_INCLUDES = NO 88 | INLINE_INFO = YES 89 | SORT_MEMBER_DOCS = YES 90 | SORT_BRIEF_DOCS = NO 91 | SORT_MEMBERS_CTORS_1ST = NO 92 | SORT_GROUP_NAMES = NO 93 | SORT_BY_SCOPE_NAME = NO 94 | STRICT_PROTO_MATCHING = NO 95 | GENERATE_TODOLIST = YES 96 | GENERATE_TESTLIST = YES 97 | GENERATE_BUGLIST = YES 98 | GENERATE_DEPRECATEDLIST= YES 99 | ENABLED_SECTIONS = 100 | MAX_INITIALIZER_LINES = 30 101 | SHOW_USED_FILES = YES 102 | SHOW_FILES = YES 103 | SHOW_NAMESPACES = YES 104 | FILE_VERSION_FILTER = 105 | LAYOUT_FILE = 106 | CITE_BIB_FILES = 107 | #--------------------------------------------------------------------------- 108 | # Configuration options related to warning and progress messages 109 | #--------------------------------------------------------------------------- 110 | QUIET = NO 111 | WARNINGS = YES 112 | WARN_IF_UNDOCUMENTED = YES 113 | WARN_IF_DOC_ERROR = YES 114 | WARN_IF_INCOMPLETE_DOC = YES 115 | WARN_NO_PARAMDOC = NO 116 | WARN_AS_ERROR = NO 117 | WARN_FORMAT = "$file:$line: $text" 118 | WARN_LOGFILE = 119 | #--------------------------------------------------------------------------- 120 | # Configuration options related to the input files 121 | #--------------------------------------------------------------------------- 122 | INPUT = 123 | INPUT_ENCODING = UTF-8 124 | FILE_PATTERNS = *.c \ 125 | *.cc \ 126 | *.cxx \ 127 | *.cpp \ 128 | *.c++ \ 129 | *.java \ 130 | *.ii \ 131 | *.ixx \ 132 | *.ipp \ 133 | *.i++ \ 134 | *.inl \ 135 | *.idl \ 136 | *.ddl \ 137 | *.odl \ 138 | *.h \ 139 | *.hh \ 140 | *.hxx \ 141 | *.hpp \ 142 | *.h++ \ 143 | *.l \ 144 | *.cs \ 145 | *.d \ 146 | *.php \ 147 | *.php4 \ 148 | *.php5 \ 149 | *.phtml \ 150 | *.inc \ 151 | *.m \ 152 | *.markdown \ 153 | *.md \ 154 | *.mm \ 155 | *.dox \ 156 | *.py \ 157 | *.pyw \ 158 | *.f90 \ 159 | *.f95 \ 160 | *.f03 \ 161 | *.f08 \ 162 | *.f18 \ 163 | *.f \ 164 | *.for \ 165 | *.vhd \ 166 | *.vhdl \ 167 | *.ucf \ 168 | *.qsf \ 169 | *.ice 170 | RECURSIVE = YES 171 | EXCLUDE = 172 | EXCLUDE_SYMLINKS = NO 173 | EXCLUDE_PATTERNS = 174 | EXCLUDE_SYMBOLS = 175 | EXAMPLE_PATH = 176 | EXAMPLE_PATTERNS = * 177 | EXAMPLE_RECURSIVE = NO 178 | IMAGE_PATH = 179 | INPUT_FILTER = 180 | FILTER_PATTERNS = 181 | FILTER_SOURCE_FILES = NO 182 | FILTER_SOURCE_PATTERNS = 183 | USE_MDFILE_AS_MAINPAGE = README.md 184 | #--------------------------------------------------------------------------- 185 | # Configuration options related to source browsing 186 | #--------------------------------------------------------------------------- 187 | SOURCE_BROWSER = NO 188 | INLINE_SOURCES = NO 189 | STRIP_CODE_COMMENTS = YES 190 | REFERENCED_BY_RELATION = NO 191 | REFERENCES_RELATION = NO 192 | REFERENCES_LINK_SOURCE = YES 193 | SOURCE_TOOLTIPS = YES 194 | USE_HTAGS = NO 195 | VERBATIM_HEADERS = YES 196 | CLANG_ASSISTED_PARSING = NO 197 | CLANG_ADD_INC_PATHS = YES 198 | CLANG_OPTIONS = 199 | CLANG_DATABASE_PATH = 200 | #--------------------------------------------------------------------------- 201 | # Configuration options related to the alphabetical class index 202 | #--------------------------------------------------------------------------- 203 | ALPHABETICAL_INDEX = YES 204 | IGNORE_PREFIX = 205 | #--------------------------------------------------------------------------- 206 | # Configuration options related to the HTML output 207 | #--------------------------------------------------------------------------- 208 | GENERATE_HTML = YES 209 | HTML_OUTPUT = html 210 | HTML_FILE_EXTENSION = .html 211 | HTML_HEADER = 212 | HTML_FOOTER = 213 | HTML_STYLESHEET = 214 | HTML_EXTRA_STYLESHEET = doc/style.css 215 | HTML_EXTRA_FILES = 216 | HTML_COLORSTYLE_HUE = 220 217 | HTML_COLORSTYLE_SAT = 100 218 | HTML_COLORSTYLE_GAMMA = 80 219 | HTML_TIMESTAMP = NO 220 | HTML_DYNAMIC_MENUS = YES 221 | HTML_DYNAMIC_SECTIONS = NO 222 | HTML_INDEX_NUM_ENTRIES = 100 223 | GENERATE_DOCSET = NO 224 | DOCSET_FEEDNAME = "Doxygen generated docs" 225 | DOCSET_BUNDLE_ID = org.doxygen.Project 226 | DOCSET_PUBLISHER_ID = org.doxygen.Publisher 227 | DOCSET_PUBLISHER_NAME = Publisher 228 | GENERATE_HTMLHELP = NO 229 | CHM_FILE = 230 | HHC_LOCATION = 231 | GENERATE_CHI = NO 232 | CHM_INDEX_ENCODING = 233 | BINARY_TOC = NO 234 | TOC_EXPAND = NO 235 | GENERATE_QHP = NO 236 | QCH_FILE = 237 | QHP_NAMESPACE = org.doxygen.Project 238 | QHP_VIRTUAL_FOLDER = doc 239 | QHP_CUST_FILTER_NAME = 240 | QHP_CUST_FILTER_ATTRS = 241 | QHP_SECT_FILTER_ATTRS = 242 | QHG_LOCATION = 243 | GENERATE_ECLIPSEHELP = NO 244 | ECLIPSE_DOC_ID = org.doxygen.Project 245 | DISABLE_INDEX = NO 246 | GENERATE_TREEVIEW = NO 247 | FULL_SIDEBAR = NO 248 | ENUM_VALUES_PER_LINE = 4 249 | TREEVIEW_WIDTH = 250 250 | EXT_LINKS_IN_WINDOW = NO 251 | HTML_FORMULA_FORMAT = png 252 | FORMULA_FONTSIZE = 10 253 | FORMULA_TRANSPARENT = YES 254 | FORMULA_MACROFILE = 255 | USE_MATHJAX = NO 256 | MATHJAX_VERSION = MathJax_2 257 | MATHJAX_FORMAT = HTML-CSS 258 | MATHJAX_RELPATH = 259 | MATHJAX_EXTENSIONS = 260 | MATHJAX_CODEFILE = 261 | SEARCHENGINE = YES 262 | SERVER_BASED_SEARCH = NO 263 | EXTERNAL_SEARCH = NO 264 | SEARCHENGINE_URL = 265 | SEARCHDATA_FILE = searchdata.xml 266 | EXTERNAL_SEARCH_ID = 267 | EXTRA_SEARCH_MAPPINGS = 268 | #--------------------------------------------------------------------------- 269 | # Configuration options related to the LaTeX output 270 | #--------------------------------------------------------------------------- 271 | GENERATE_LATEX = NO 272 | LATEX_OUTPUT = latex 273 | LATEX_CMD_NAME = 274 | MAKEINDEX_CMD_NAME = makeindex 275 | LATEX_MAKEINDEX_CMD = makeindex 276 | COMPACT_LATEX = NO 277 | PAPER_TYPE = a4 278 | EXTRA_PACKAGES = 279 | LATEX_HEADER = 280 | LATEX_FOOTER = 281 | LATEX_EXTRA_STYLESHEET = 282 | LATEX_EXTRA_FILES = 283 | PDF_HYPERLINKS = YES 284 | USE_PDFLATEX = YES 285 | LATEX_BATCHMODE = NO 286 | LATEX_HIDE_INDICES = NO 287 | LATEX_BIB_STYLE = plain 288 | LATEX_TIMESTAMP = NO 289 | LATEX_EMOJI_DIRECTORY = 290 | #--------------------------------------------------------------------------- 291 | # Configuration options related to the RTF output 292 | #--------------------------------------------------------------------------- 293 | GENERATE_RTF = NO 294 | RTF_OUTPUT = rtf 295 | COMPACT_RTF = NO 296 | RTF_HYPERLINKS = NO 297 | RTF_STYLESHEET_FILE = 298 | RTF_EXTENSIONS_FILE = 299 | #--------------------------------------------------------------------------- 300 | # Configuration options related to the man page output 301 | #--------------------------------------------------------------------------- 302 | GENERATE_MAN = NO 303 | MAN_OUTPUT = man 304 | MAN_EXTENSION = .3 305 | MAN_SUBDIR = 306 | MAN_LINKS = NO 307 | #--------------------------------------------------------------------------- 308 | # Configuration options related to the XML output 309 | #--------------------------------------------------------------------------- 310 | GENERATE_XML = NO 311 | XML_OUTPUT = xml 312 | XML_PROGRAMLISTING = YES 313 | XML_NS_MEMB_FILE_SCOPE = NO 314 | #--------------------------------------------------------------------------- 315 | # Configuration options related to the DOCBOOK output 316 | #--------------------------------------------------------------------------- 317 | GENERATE_DOCBOOK = NO 318 | DOCBOOK_OUTPUT = docbook 319 | #--------------------------------------------------------------------------- 320 | # Configuration options for the AutoGen Definitions output 321 | #--------------------------------------------------------------------------- 322 | GENERATE_AUTOGEN_DEF = NO 323 | #--------------------------------------------------------------------------- 324 | # Configuration options related to Sqlite3 output 325 | #--------------------------------------------------------------------------- 326 | #--------------------------------------------------------------------------- 327 | # Configuration options related to the Perl module output 328 | #--------------------------------------------------------------------------- 329 | GENERATE_PERLMOD = NO 330 | PERLMOD_LATEX = NO 331 | PERLMOD_PRETTY = YES 332 | PERLMOD_MAKEVAR_PREFIX = 333 | #--------------------------------------------------------------------------- 334 | # Configuration options related to the preprocessor 335 | #--------------------------------------------------------------------------- 336 | ENABLE_PREPROCESSING = YES 337 | MACRO_EXPANSION = NO 338 | EXPAND_ONLY_PREDEF = NO 339 | SEARCH_INCLUDES = YES 340 | INCLUDE_PATH = 341 | INCLUDE_FILE_PATTERNS = 342 | PREDEFINED = 343 | EXPAND_AS_DEFINED = 344 | SKIP_FUNCTION_MACROS = YES 345 | #--------------------------------------------------------------------------- 346 | # Configuration options related to external references 347 | #--------------------------------------------------------------------------- 348 | TAGFILES = 349 | GENERATE_TAGFILE = 350 | ALLEXTERNALS = NO 351 | EXTERNAL_GROUPS = YES 352 | EXTERNAL_PAGES = YES 353 | #--------------------------------------------------------------------------- 354 | # Configuration options related to the dot tool 355 | #--------------------------------------------------------------------------- 356 | CLASS_DIAGRAMS = YES 357 | DIA_PATH = 358 | HIDE_UNDOC_RELATIONS = YES 359 | HAVE_DOT = NO 360 | DOT_NUM_THREADS = 0 361 | DOT_FONTNAME = Helvetica 362 | DOT_FONTSIZE = 10 363 | DOT_FONTPATH = 364 | CLASS_GRAPH = YES 365 | COLLABORATION_GRAPH = YES 366 | GROUP_GRAPHS = YES 367 | UML_LOOK = NO 368 | UML_LIMIT_NUM_FIELDS = 10 369 | DOT_UML_DETAILS = NO 370 | DOT_WRAP_THRESHOLD = 17 371 | TEMPLATE_RELATIONS = NO 372 | INCLUDE_GRAPH = YES 373 | INCLUDED_BY_GRAPH = YES 374 | CALL_GRAPH = NO 375 | CALLER_GRAPH = NO 376 | GRAPHICAL_HIERARCHY = YES 377 | DIRECTORY_GRAPH = YES 378 | DOT_IMAGE_FORMAT = png 379 | INTERACTIVE_SVG = NO 380 | DOT_PATH = 381 | DOTFILE_DIRS = 382 | MSCFILE_DIRS = 383 | DIAFILE_DIRS = 384 | PLANTUML_JAR_PATH = 385 | PLANTUML_CFG_FILE = 386 | PLANTUML_INCLUDE_PATH = 387 | DOT_GRAPH_MAX_NODES = 50 388 | MAX_DOT_GRAPH_DEPTH = 0 389 | DOT_TRANSPARENT = NO 390 | DOT_MULTI_TARGETS = NO 391 | GENERATE_LEGEND = YES 392 | DOT_CLEANUP = YES 393 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(origin CC),default) 2 | CC = gcc 3 | endif 4 | CCFLAGS ?= -O -lz -Wall -Wno-unused-result -Wno-deprecated-declarations -g -D_LARGEFILE64_SOURCE -D_GNU_SOURCE -fPIC -Wno-format-truncation -I./src/ldb 5 | LDFLAGS+= -lm -lpthread -lz -ldl -lgcrypt 6 | SOURCES=$(wildcard src/*.c) 7 | OBJECTS=$(SOURCES:.c=.o) 8 | TARGET=ldb 9 | LIB=libldb.so 10 | LOGDIR:=/var/log/scanoss/ldb/ 11 | $(TARGET): $(OBJECTS) 12 | $(CC) -g -o $(TARGET) $^ $(LDFLAGS) 13 | 14 | VERSION=$(shell ./version.sh) 15 | 16 | all: clean $(TARGET) lib 17 | 18 | lib: $(OBJECTS) 19 | $(CC) -g -o $(LIB) $^ $(LDFLAGS) -shared -Wl,-soname,$(LIB) 20 | .PHONY: ldb 21 | 22 | %.o: %.c 23 | $(CC) $(CCFLAGS) -o $@ -c $< 24 | 25 | clean_build: 26 | rm -rf src/*.o src/**/*.o external/src/*.o external/src/**/*.o 27 | 28 | clean: clean_build 29 | rm -rf $(TARGET) 30 | rm -rf $(LIB) 31 | distclean: clean 32 | 33 | install: $(TARGET) lib 34 | @cp $(TARGET) /usr/bin 35 | @cp $(LIB) /usr/lib 36 | @cp -r src/ldb /usr/include 37 | @cp src/ldb.h /usr/include 38 | @mkdir -p $(LOGDIR) && chown -R $(SUDO_USER) $(LOGDIR) && chmod -R u+rw $(LOGDIR) 39 | uninstall: 40 | @rm -r /usr/include/ldb 41 | @rm /usr/include/ldb.h 42 | @rm /usr/lib/libldb.so 43 | prepare_deb_package: all ## Prepares the deb Package 44 | @./package.sh deb $(VERSION) 45 | @echo deb package built 46 | 47 | prepare_rpm_package: all ## Prepares the rpm Package 48 | @./package.sh rpm $(VERSION) 49 | @echo rpm package built -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LDB Database 2 | 3 | The LDB (Linked-list database) is a headless database management system focused in single-key, read-only application on vast amounts of data, while maintaining a minimal footprint and keeping system calls to a bare minimum. Information is structured using linked lists. 4 | 5 | # Build and Install 6 | 7 | ## Prerequisites 8 | Building LDB requires libgcrypt and zlib. Make sure packages `zlib1g-dev` and `libgcrypt-dev` are installed. 9 | 10 | ## Build 11 | Run `make all` to build the shell binary and the shared library. 12 | 13 | ## Install 14 | Run `make install` to copy the binary to `/usr/bin`, the shared library to `/usr/lib` and the header to `/usr/include`. 15 | 16 | ## Run test 17 | Run `./run_test.sh` to test the ldb binary. 18 | 19 | # Features 20 | 21 | Some of the features of the LDB are: 22 | 23 | * Single, fixed size, numeric key (32-bit) 24 | * Single-field records 25 | * Larger keys also are supported by storing exceeded data keys in the data record. 26 | * No indexing: Mapping 27 | * Data tables define either fixed or variable-length data records 28 | * Read-only 29 | * Database updates are performed in a single-threaded, non-disruptive batch operation 30 | * Zlib data compression 31 | * Data records are organized in a linked list 32 | * C library for native development 33 | * LDB shell allows interaction with external languages 34 | 35 | ## LDB Shell Commands 36 | 37 | ``` 38 | create database DBNAME 39 | Creates an empty database 40 | 41 | create table DBNAME/TABLENAME keylen N reclen N 42 | Creates an empty table in the given database with 43 | the specified key length (>= 4) and record length (0=variable) 44 | 45 | show databases 46 | Lists databases 47 | 48 | show tables from DBNAME 49 | Lists tables from given database 50 | 51 | bulk insert DBNAME/TABLENAME from PATH with (CONFIG) 52 | Imports data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported. 53 | TABLENAME is optional and will be derived from the directory name's file if not specified. 54 | 55 | (CONFIG) is a configuration string with the following format: 56 | (FILE_DEL=1/0,KEYS=N,MZ=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp) 57 | 58 | Where 1/0 represents "true" / "false", and N is an integer. 59 | FILE_DEL: Delete file after importation is completed. 60 | KEYS: Number of binary keys in the CSV file. 61 | MZ: MZ file indicator. 62 | WFP: WFP file indicator. 63 | OVERWRITE: Overwrite the destination table. (Default: 0). 64 | SORT: Sort the tuples during the import process. (Default: 1). 65 | FIELDS: Number of CSV fields. 66 | VALIDATE_FIELDS: Check field quantity during importation. (Default: 1). 67 | VALIDATE_VERSION: Validate version.json.(Default: 1). 68 | VERBOSE: Enable verbose mode. (Default: 0). 69 | THREADS: Define the number of threads to be used during the importation process. Defaul value: half of system available. 70 | COLLATE: Perform collation after import, removing data larger than MAX_RECORD bytes. (Default: 0). 71 | MAX_RECORD: Maximum record size in bytes (Default value: 2048). 72 | MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50. 73 | TMP_PATH: Path to the folder used for temporary files (default: /tmp). 74 | 75 | bulk insert DBNAME/TABLENAME from PATH 76 | Imports data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported. 77 | The configuration will be retrieved from the "db.conf" file located at "/etc/local/scanoss/ldb/". A default configuration file will be created if it doesn't exist. 78 | 79 | insert into DBNAME/TABLENAME key KEY hex DATA 80 | Inserts data (hex) into given db/table for the given hex key 81 | 82 | insert into DBNAME/TABLENAME key KEY ascii DATA 83 | Inserts data (ASCII) into db/table for the given hex key 84 | 85 | select from DBNAME/TABLENAME key KEY 86 | Retrieves all records from db/table for the given hex key (hexdump output) 87 | 88 | select from DBNAME/TABLENAME key KEY ascii 89 | Retrieves all records from db/table for the given hex key (ascii output) 90 | 91 | select from DBNAME/TABLENAME key KEY csv hex N 92 | Retrieves all records from db/table for the given hex key (csv output, with first N bytes in hex) 93 | 94 | delete from DBNAME/TABLENAME max LENGTH keys KEY_LIST 95 | Deletes all records for the given comma separated hex key list from the db/table. Max record length expected 96 | 97 | delete from DBNAME/TABLENAME record CSV_RECORD\n" 98 | Deletes the specific CSV record from the specified table. Some field of the CSV may be skippet from the comparation using '*' 99 | Example: delete from db/url record key,madler,*,2.4,20171227,zlib,pkg:github/madler/pigz,https://github.com/madler/pigz/archive/v2.4.zip 100 | All the records matching the all the csv's field with exception of the second thirdone will be removed. 101 | 102 | delete from DBNAME/TABLENAME records from PATH\ 103 | Similar to the previous command, but the records (may be more than one) will be loaded from a csv file in PATH. 104 | 105 | collate DBNAME/TABLENAME max LENGTH 106 | Collates all lists in a table, removing duplicates and records greater than LENGTH bytes 107 | 108 | merge DBNAME/TABLENAME1 into DBNAME/TABLENAME2 max LENGTH 109 | Merges tables erasing tablename1 when done. Tables must have the same configuration 110 | 111 | unlink list from DBNAME/TABLENAME key KEY 112 | Unlinks the given list (32-bit KEY) from the sector map 113 | 114 | dump DBNAME/TABLENAME hex N 115 | Dumps table contents with first N bytes in hex 116 | 117 | dump keys from DBNAME/TABLENAME [sector N] 118 | Dumps a unique list of existing keys 119 | ``` 120 | ## Other Uses 121 | 122 | ### Update Database 123 | ```bash 124 | ldb -u [--update] path -n[--name] db_name -c[--collate] 125 | ``` 126 | Create or update an existing database from "path." If "db_name" is not specified, "oss" will be used by default. If the "--collate" option is present, each table will be collated during the importation process. 127 | 128 | ### Process Commands from File 129 | ```bash 130 | ldb -f [filename] 131 | ``` 132 | Process a list of commands from a file named "filename."q 133 | 134 | # Using the shell 135 | 136 | The following example explains how to create a database, a table, a record, and querying that record. 137 | 138 | ``` 139 | $ echo "create database test" | ldb 140 | OK 141 | $ echo "create table test/table1 keylen 16 reclen variable" | ldb 142 | OK 143 | $ echo "insert into test/table1 key 26e3a3bd01585cde84408f01fe981f6a ascii THIS_IS_A_TEST" | ldb 144 | $ echo "select from test/table1 key 26e3a3bd01585cde84408f01fe981f6a" | ldb 145 | 0000 544849535f49535f415f54455354 THIS_IS_A_TEST 146 | $ echo "select from test/table1 key 26e3a3bd01585cde84408f01fe981f6a ascii" | ldb 147 | THIS_IS_A_TEST 148 | ``` 149 | 150 | # License 151 | 152 | The LDB is released under the GPL 2.0 license. See the LICENSE file for more infomation. 153 | 154 | Copyright (C) 2018-2020 SCANOSS.COM 155 | http://scanoss.com 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /SBOM.json: -------------------------------------------------------------------------------- 1 | { 2 | "bomFormat": "CycloneDX", 3 | "specVersion": "1.2", 4 | "version": 1, 5 | "serialNumber": "scanner:1594067462", 6 | "components": [ 7 | { 8 | "type": "application", 9 | "publisher": "scanoss", 10 | "group": "scanoss", 11 | "name": "ldb", 12 | "url": "https://github.com/scanoss/ldb", 13 | "licenses": [ 14 | { 15 | "id": "GPL-2.0-or-later" 16 | } 17 | ] 18 | }, 19 | { 20 | "type": "testing", 21 | "publisher": "pgrange", 22 | "group": "pgrange", 23 | "name": "bash_unit", 24 | "url": "https://github.com/pgrange/bash_unit", 25 | "purl":"pkg:github/pgrange/bash_unit", 26 | "licenses": [ 27 | { 28 | "id": "GPL-3.0" 29 | } 30 | ] 31 | } 32 | ] 33 | } 34 | 35 | -------------------------------------------------------------------------------- /doc/errors.txt: -------------------------------------------------------------------------------- 1 | E050 Cannot create root LDB directory 2 | E051 Concurrent ldb writing not supported (/dev/shm/ldb.lock exists) 3 | E052 Concurrent ldb writing is not supported. (check /dev/shm/ldb.lock) 4 | E053 Data record size exceeded 5 | E054 Data corruption 6 | E055 Data corruption 7 | E056 Data sector corrupted, with %lu below map_size 8 | E057 Map location 9 | E059 LDB root directory %s is not accessible 10 | E060 Table name format should be dbname/tablename 11 | E061 db/table name is too long 12 | E062 Database does not exist 13 | E063 Table does not exist 14 | E064 Invalid characters or name is too long 15 | E065 Cannot create 16 | E066 Syntax error 17 | E067 Command not implemented 18 | E068 Database already exists 19 | E069 Table already exists 20 | E070 Cannot open LDB root directory 21 | E071 Key length cannot be less than 32 bits 22 | E072 Cannot access table 23 | E073 Provided key is longer than table key 24 | E074 Corrupted node 25 | -------------------------------------------------------------------------------- /doc/scanoss-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scanoss/ldb/b2b2f350f24bcd27aa0b3a5dc93f17c8189b85e6/doc/scanoss-logo.png -------------------------------------------------------------------------------- /doc/style.css: -------------------------------------------------------------------------------- 1 | #projectlogo { 2 | vertical-align: middle; 3 | } 4 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ### 3 | # SPDX-License-Identifier: GPL-2.0-or-later 4 | # 5 | # Copyright (C) 2018-2023 SCANOSS.COM 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | ### 18 | # 19 | # Package up the local binaries into deb or rpm packages for deployment on a server 20 | # 21 | if [ "$1" = "-h" ] || [ "$1" = "-help" ] ; then 22 | echo "$0 [-help] " 23 | echo " Create a package of the ldb" 24 | echo " the package type (deb, rpm)" 25 | echo " version number of the package" 26 | exit 1 27 | fi 28 | if [[ -z "$1" || ( "$1" != "deb" && "$1" != "rpm" ) ]] ; then 29 | echo "ERROR: Please provide a package type: deb or rpm" 30 | exit 1 31 | fi 32 | if [ -z "$2" ] ; then 33 | echo "ERROR: Please provide a package version" 34 | exit 1 35 | fi 36 | 37 | if [ "$1" = "deb" ] ; then 38 | # Add binaries 39 | mkdir -p dist/.debpkg/bin 40 | mkdir -p dist/.debpkg/lib 41 | cp ldb dist/.debpkg/bin/ldb 42 | chmod +x dist/.debpkg/bin/ldb 43 | cp libldb.so dist/.debpkg/lib/libldb.so 44 | # Add control file 45 | mkdir -p dist/.debpkg/DEBIAN 46 | cp scripts/debpkg/DEBIAN/control dist/.debpkg/DEBIAN/control 47 | sed -i 's/\LDB_VERSION/'"$2"'/g' dist/.debpkg/DEBIAN/control 48 | # Build package 49 | dpkg-deb -Zxz --root-owner-group --build ./dist/.debpkg/ . 50 | fi 51 | 52 | if [ "$1" = "rpm" ] ; then 53 | mkdir -p dist/.rpmpkg/ 54 | cp ldb dist/.rpmpkg/ldb 55 | chmod +x ./dist/.rpmpkg/ldb 56 | cp libldb.so ./dist/.rpmpkg/libldb.so 57 | cp scripts/rpmpkg/ldb.spec dist/.rpmpkg/ldb.spec 58 | sed -i 's/\LDB_VERSION/'"$2"'/g' dist/.rpmpkg/ldb.spec 59 | rpmbuild -ba --build-in-place --define "_topdir $(pwd)/dist/rpm" dist/.rpmpkg/ldb.spec 60 | fi -------------------------------------------------------------------------------- /run_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ./test/bash_unit test/test_kb -------------------------------------------------------------------------------- /scripts/debpkg/DEBIAN/control: -------------------------------------------------------------------------------- 1 | Package: ldb 2 | Version: LDB_VERSION 3 | Maintainer: SCANOSS 4 | Architecture: amd64 5 | Depends: libssl1.1 (>= 1.1.0), libc6 (>= 2.4), zlib1g (>= 1:1.1.4) 6 | Description: SCANOSS LDB 7 | The LDB (Linked-list database) is a headless database management system focused in single-key, read-only application on vast amounts of data, while maintaining a minimal footprint and keeping system calls to a bare minimum. Information is structured using linked lists. 8 | License: GPL-2+ 9 | -------------------------------------------------------------------------------- /scripts/rpmpkg/ldb.spec: -------------------------------------------------------------------------------- 1 | Name: ldb 2 | Version: LDB_VERSION 3 | Release: 1%{?dist} 4 | Summary: SCANOSS LDB 5 | License: GPLv2 6 | BuildArch: x86_64 7 | 8 | %description 9 | The LDB (Linked-list database) is a headless database management system focused in single-key, read-only application on vast amounts of data, while maintaining a minimal footprint and keeping system calls to a bare minimum. Information is structured using linked lists. 10 | 11 | %prep 12 | 13 | %build 14 | 15 | %install 16 | mkdir -p %{buildroot}/%{_bindir} 17 | mkdir -p %{buildroot}/%{_libdir} 18 | install -m 0755 %{name} %{buildroot}/%{_bindir}/%{name} 19 | install -m 0755 libldb.so %{buildroot}/%{_libdir}/libldb.so 20 | 21 | %files 22 | %{_bindir}/%{name} 23 | %{_libdir}/libldb.so 24 | 25 | %changelog -------------------------------------------------------------------------------- /src/bsort.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "bsort.h" 13 | 14 | #define SWITCH_TO_SHELL 20 15 | 16 | struct sort 17 | { 18 | int fd; 19 | off_t size; 20 | void *buffer; 21 | }; 22 | 23 | static inline void shellsort(unsigned char *a, const int n, const int record_size, const int key_size) 24 | { 25 | int i, j; 26 | char temp[record_size]; 27 | 28 | for (i = 3; i < n; i++) 29 | { 30 | memcpy(&temp, &a[i * record_size], record_size); 31 | for (j = i; j >= 3 && memcmp(a + (j - 3) * record_size, &temp, key_size) > 0; j -= 3) 32 | { 33 | memcpy(a + j * record_size, a + (j - 3) * record_size, record_size); 34 | } 35 | memcpy(a + j * record_size, &temp, record_size); 36 | } 37 | 38 | for (i = 1; i < n; i++) 39 | { 40 | memcpy(&temp, &a[i*record_size], record_size); 41 | for(j = i; j >= 1 && memcmp(a + (j - 1) * record_size, &temp, key_size) > 0; j -= 1) 42 | { 43 | memcpy(a + j * record_size, a + (j - 1) * record_size, record_size); 44 | } 45 | memcpy(a + j * record_size, &temp, record_size); 46 | } 47 | } 48 | 49 | void radixify(unsigned char *buffer, 50 | const long count, 51 | const long digit, 52 | const long char_start, 53 | const long char_stop, 54 | const long record_size, 55 | const long key_size, 56 | const long stack_size, 57 | const long cut_off) 58 | { 59 | long counts[char_stop+1]; 60 | long offsets[char_stop+1]; 61 | long starts[char_stop+1]; 62 | long ends[char_stop+1]; 63 | long offset=0; 64 | unsigned char temp[record_size]; 65 | long target, x; 66 | long stack[stack_size]; 67 | long stack_pointer; 68 | 69 | for (x = char_start; x <= char_stop; x++) 70 | { 71 | counts[x] = 0; 72 | offsets[x] = 0; 73 | } 74 | 75 | // Compute starting positions 76 | for (x=0; x SWITCH_TO_SHELL) 132 | { 133 | radixify(&buffer[starts[x] * record_size], 134 | ends[x] - starts[x], 135 | digit+1, 136 | char_start, 137 | char_stop, 138 | record_size, 139 | key_size, 140 | stack_size, 141 | cut_off); 142 | } 143 | else 144 | { 145 | if (ends[x] - starts[x] <= 1) continue; 146 | shellsort(&buffer[starts[x] * record_size], ends[x] - starts[x], record_size, key_size); 147 | } 148 | } 149 | } 150 | else 151 | { 152 | for (x=char_start; x<=char_stop; x++) 153 | { 154 | if (ends[x] - starts[x] > 1) 155 | shellsort(&buffer[starts[x] * record_size], ends[x] - starts[x], record_size, key_size); 156 | } 157 | } 158 | } 159 | 160 | bool open_sort(char *path, struct sort *sort) 161 | { 162 | void *buffer = NULL; 163 | 164 | int fd = open(path, O_RDWR); 165 | if (fd == -1) return false; 166 | 167 | struct stat stats; 168 | if (fstat(fd, &stats) == -1) 169 | { 170 | if (fd != -1) close(fd); 171 | return false; 172 | } 173 | 174 | if (!(buffer = mmap(NULL, stats.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0))) 175 | { 176 | perror(path); 177 | if (buffer) munmap(buffer, stats.st_size); 178 | if (fd != -1) close(fd); 179 | sort->buffer = 0; 180 | sort->fd = 0; 181 | return false; 182 | } 183 | 184 | madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL); 185 | 186 | sort->buffer = buffer; 187 | sort->size = stats.st_size; 188 | sort->fd = fd; 189 | return true; 190 | } 191 | 192 | void close_sort(struct sort *sort) 193 | { 194 | if (sort->buffer) 195 | { 196 | munmap(sort->buffer, sort->size); 197 | sort->buffer = 0; 198 | sort->size = 0; 199 | } 200 | if (sort->fd) 201 | { 202 | close(sort->fd); 203 | sort->fd = 0; 204 | } 205 | } 206 | 207 | int bsort(char *file_path) 208 | { 209 | int char_start = 0; 210 | int char_stop = 255; 211 | int record_size=21; 212 | int key_size=21; 213 | int stack_size=5; 214 | int cut_off = 4; 215 | 216 | struct sort sort; 217 | if (!open_sort(file_path, &sort)) return false; 218 | 219 | radixify(sort.buffer, 220 | sort.size / record_size, 221 | 0, 222 | char_start, 223 | char_stop, 224 | record_size, 225 | key_size, 226 | stack_size, 227 | cut_off); 228 | close_sort(&sort); 229 | optind++; 230 | 231 | return true; 232 | } 233 | -------------------------------------------------------------------------------- /src/bsort.h: -------------------------------------------------------------------------------- 1 | #ifndef __BSORT_H 2 | #define __BSORT_H 3 | 4 | int bsort(char *file_path); 5 | 6 | #endif -------------------------------------------------------------------------------- /src/command.h: -------------------------------------------------------------------------------- 1 | #ifndef __COMMAND_H 2 | #define _COMMAND_H 3 | #include "ldb.h" 4 | 5 | typedef enum { 6 | HEX, 7 | ASCII, 8 | CSV 9 | } select_format; 10 | 11 | typedef enum { 12 | HELP, 13 | CREATE_DATABASE, 14 | CREATE_TABLE, 15 | CREATE_CONFIG, 16 | SHOW_DATABASES, 17 | SHOW_TABLES, 18 | INSERT_ASCII, 19 | INSERT_HEX, 20 | SELECT_ASCII, 21 | SELECT_CSV, 22 | SELECT, 23 | DELETE, 24 | DELETE_RECORD, 25 | DELETE_RECORDS, 26 | COLLATE, 27 | BULK_INSERT, 28 | BULK_INSERT_DEFAULT, 29 | MERGE, 30 | VERSION, 31 | UNLINK_LIST, 32 | DUMP_SECTOR, 33 | DUMP, 34 | DUMP_KEYS, 35 | } commandtype; 36 | 37 | void ldb_command_create_database(char *command); 38 | void ldb_command_create_config(char * command); 39 | char *ldb_command_normalize(char *text); 40 | void ldb_command_show_tables(char *command); 41 | void ldb_command_show_databases(); 42 | void ldb_command_select(char *command, select_format format); 43 | void ldb_command_telect(char *command); 44 | void ldb_command_insert(char *command, commandtype type); 45 | void ldb_command_create_table(char *command); 46 | void ldb_command_bulk(char *command, commandtype type); 47 | bool ldb_import_command(char * dbtable, char * path, char * config); 48 | void ldb_command_dump(char *command); 49 | void ldb_command_dump_keys(char *command); 50 | void ldb_command_merge(char *command); 51 | void ldb_command_delete(char *command); 52 | void ldb_command_collate(char *command); 53 | void ldb_command_unlink_list(char *command); 54 | void ldb_command_delete_records(char *command); 55 | 56 | #endif -------------------------------------------------------------------------------- /src/config.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/config.c 4 | * 5 | * Configuration file handling routines 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file config.c 25 | * @date 12 Jul 2020 26 | * @brief LDB configuration functions 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/collate.c 30 | */ 31 | #include "ldb.h" 32 | #include "ldb_string.h" 33 | #include "logger.h" 34 | /** 35 | * @brief Loads table configuration from .cfg file 36 | * 37 | * @param db DB string name 38 | * @param table table string name 39 | * @param rs ldb recordset pointer 40 | * @return true if succed 41 | */ 42 | bool ldb_load_cfg(char *db, char *table, struct ldb_recordset *rs) 43 | { 44 | char *path = malloc(LDB_MAX_PATH); 45 | 46 | // Open configuration file 47 | sprintf(path, "%s/%s/%s.cfg", ldb_root, db, table); 48 | FILE *cfg = fopen(path, "r"); 49 | 50 | if (!cfg) 51 | { 52 | free(path); 53 | return false; 54 | } 55 | 56 | // Read configuration file 57 | char *buffer = malloc(LDB_MAX_NAME); 58 | if (!fread(buffer, 1, LDB_MAX_NAME, cfg)) 59 | printf("Warning: cannot open file %s\n", path); 60 | fclose(cfg); 61 | char *reclen = buffer + ldb_split_string(buffer, ','); 62 | 63 | // Read values 64 | int key_ln = atoi(buffer); 65 | int rec_ln = atoi(reclen); 66 | // Validate values 67 | if (key_ln < 4 || key_ln > 255) return false; 68 | if (rec_ln < 0 || rec_ln > 255) return false; 69 | 70 | // Load values into recordset 71 | rs->key_ln = key_ln; 72 | rs->rec_ln = rec_ln; 73 | rs->subkey_ln = key_ln - 4; 74 | strcpy(rs->db, db); 75 | strcpy(rs->table, table); 76 | 77 | free(buffer); 78 | free(path); 79 | return true; 80 | } 81 | 82 | /** 83 | * @brief Read table config from a file and loads insto a ldb_table structure 84 | * 85 | * @param db_table DB table name 86 | * @return struct with table configuration 87 | */ 88 | struct ldb_table ldb_read_cfg(char *db_table) 89 | { 90 | struct ldb_table tablecfg = {.db = "\0", .table = "\0", .key_ln = 16, .rec_ln = 0, .keys = 1, .tmp = false, .ts_ln = 2, .definitions = LDB_TABLE_DEFINITION_UNDEFINED}; // default config 91 | 92 | char tmp[LDB_MAX_PATH] = "\0"; 93 | strcpy(tmp, db_table); 94 | char *tablename = tmp + ldb_split_string(tmp, '/'); 95 | 96 | strcpy(tablecfg.db, tmp); 97 | strcpy(tablecfg.table, tablename); 98 | 99 | char path[LDB_MAX_PATH] = "\0"; 100 | 101 | // Open configuration file 102 | sprintf(path, "%s/%s.cfg", ldb_root, db_table); 103 | FILE *cfg = fopen(path, "r"); 104 | 105 | if (!cfg) 106 | { 107 | log_info("Warning: config file \"%s\" does not exist. Using table's default config\n", path); 108 | return tablecfg; 109 | } 110 | 111 | // Read configuration file 112 | int key_ln, rec_ln, keys, definitions; 113 | int result = fscanf(cfg, "%d,%d,%d,%d", &key_ln, &rec_ln, &keys, &definitions); 114 | 115 | if (result < 2) 116 | { 117 | log_info("Warning: cannot read file %s\n, using default config\n", path); 118 | fclose(cfg); 119 | return tablecfg; 120 | } 121 | 122 | tablecfg.key_ln = key_ln; 123 | tablecfg.rec_ln = rec_ln; 124 | 125 | // backward compatibility with cfg files 126 | if (result < 4) 127 | { 128 | log_info("Warning: some fields are undefined in config file %s, must be updated\n", path); 129 | keys = -1; 130 | definitions = -1; 131 | } 132 | 133 | tablecfg.keys = keys; 134 | tablecfg.definitions = definitions; 135 | fclose(cfg); 136 | return tablecfg; 137 | } 138 | 139 | /** 140 | * @brief Save db config into a file 141 | * 142 | * @param db DB name 143 | * @param table Table name 144 | * @param keylen Key lenght 145 | * @param reclen register lenght 146 | */ 147 | void ldb_write_cfg(char *db, char *table, int keylen, int reclen, int keys, int definitions) 148 | { 149 | char *path = malloc(LDB_MAX_PATH); 150 | sprintf(path, "%s/%s/%s.cfg", ldb_root, db, table); 151 | 152 | FILE *cfg = fopen(path, "w+"); 153 | fprintf(cfg,"%d,%d,%d,%d\n", keylen, reclen, keys, definitions); 154 | fclose(cfg); 155 | 156 | free(path); 157 | } 158 | 159 | -------------------------------------------------------------------------------- /src/decode.c: -------------------------------------------------------------------------------- 1 | #include "decode.h" 2 | #include "logger.h" 3 | void * lib_handle = NULL; 4 | 5 | int (*decode) (int op, unsigned char *key, unsigned char *nonce, 6 | const char *buffer_in, int buffer_in_len, unsigned char *buffer_out) = NULL; 7 | 8 | bool ldb_decoder_lib_load(void) 9 | { 10 | if (lib_handle != NULL) 11 | return true; 12 | /*set decode funtion pointer to NULL*/ 13 | decode = NULL; 14 | lib_handle = dlopen("libscanoss_encoder.so", RTLD_NOW); 15 | char * err; 16 | if (lib_handle) 17 | { 18 | log_info("Lib scanoss_encoder present\n"); 19 | decode = dlsym(lib_handle, "scanoss_decode"); 20 | if ((err = dlerror())) 21 | { 22 | log_info("%s\n", err); 23 | return false; 24 | } 25 | return true; 26 | } 27 | 28 | return false; 29 | } 30 | 31 | void ldb_decoder_lib_close(void) 32 | { 33 | dlclose(lib_handle); 34 | } 35 | -------------------------------------------------------------------------------- /src/decode.h: -------------------------------------------------------------------------------- 1 | #ifndef __DECODE_H 2 | #define __DECODE_H 3 | #include 4 | #include 5 | #include 6 | #include 7 | #define DECODE_BASE64 8 8 | extern int (*decode) (int op, unsigned char *key, unsigned char *nonce, 9 | const char *buffer_in, int buffer_in_len, unsigned char *buffer_out); 10 | extern void * lib_handle; 11 | bool ldb_decoder_lib_load(void); 12 | void ldb_decoder_lib_close(void); 13 | #endif -------------------------------------------------------------------------------- /src/dump.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/dump.c 4 | * 5 | * Dump command functions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "ldb.h" 23 | #include 24 | #include "ldb_string.h" 25 | /** 26 | * @file dump.c 27 | * @date 12 Jul 2020 28 | * @brief LDB dump functions 29 | 30 | * //TODO Long description 31 | * @see https://github.com/scanoss/ldb/blob/master/src/dump.c 32 | */ 33 | 34 | /** 35 | * @brief Dump LDB into stdout 36 | * 37 | * @param table table name string 38 | * @param hex_bytes hex bytes format 39 | * @param sectorn sectur number 40 | */ 41 | void ldb_dump(struct ldb_table table, int hex_bytes, int sectorn) 42 | { 43 | /* Read each DB sector */ 44 | uint8_t k0 = 0; 45 | setlocale(LC_NUMERIC, ""); 46 | 47 | if (sectorn >= 0) k0 = (uint8_t) sectorn; 48 | 49 | do { 50 | uint8_t *sector = ldb_load_sector(table, &k0); 51 | if (sector) 52 | { 53 | /* Read each one of the (256 ^ 3) list pointers from the map */ 54 | uint8_t k[LDB_KEY_LN]; 55 | k[0] = k0; 56 | for (int k1 = 0; k1 < 256; k1++) 57 | for (int k2 = 0; k2 < 256; k2++) 58 | for (int k3 = 0; k3 < 256; k3++) 59 | { 60 | k[1] = k1; 61 | k[2] = k2; 62 | k[3] = k3; 63 | /* Process records */ 64 | ldb_fetch_recordset(sector, table, k, true, ldb_csvprint, &hex_bytes); 65 | } 66 | free(sector); 67 | } 68 | if (sectorn >= 0) break; 69 | } while (k0++ < 255); 70 | fflush(stdout); 71 | } 72 | -------------------------------------------------------------------------------- /src/file.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/file.c 4 | * 5 | * File handling functions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file file.c 25 | * @date 12 Jul 2020 26 | * @brief LDB file management functions 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/file.c 30 | */ 31 | #include "ldb.h" 32 | /** 33 | * @brief create LDB directory 34 | * 35 | * @param path string path 36 | */ 37 | void ldb_prepare_dir(char *path) 38 | { 39 | if (ldb_dir_exists (path)) 40 | return; 41 | int err = mkdir(path, 0755); 42 | if (err) 43 | { 44 | char error[1024]; 45 | sprintf(error, "E050 There was a problem creating the directory %s. Error: %d\n", path, err); 46 | ldb_error (error); 47 | } 48 | } 49 | 50 | /** 51 | * @brief Check if exist a LDB file 52 | * 53 | * @param path string path 54 | * @return true if the file exist 55 | */ 56 | bool ldb_file_exists(char *path) 57 | { 58 | struct stat pstat; 59 | if (!stat(path, &pstat)) 60 | if (S_ISREG(pstat.st_mode)) 61 | return true; 62 | return false; 63 | } 64 | 65 | /** 66 | * @brief Check if the LDB directory exist 67 | * 68 | * @param path path string 69 | * @return true if exist 70 | */ 71 | bool ldb_dir_exists(char *path) 72 | { 73 | struct stat pstat; 74 | if (!stat(path, &pstat)) 75 | if (S_ISDIR(pstat.st_mode)) 76 | return true; 77 | return false; 78 | } 79 | 80 | /** 81 | * @brief Return the file size for path 82 | * 83 | * @param path string path 84 | * @return file size 85 | */ 86 | uint64_t ldb_file_size(char *path) 87 | { 88 | FILE *fp = fopen(path, "r"); 89 | if (!fp) return 0; 90 | 91 | fseeko64(fp, 0, SEEK_END); 92 | uint64_t size = ftello64(fp); 93 | fclose(fp); 94 | 95 | return size; 96 | } 97 | 98 | /** 99 | * @brief Check if LDB root directory exist 100 | * 101 | * @return true if exist 102 | */ 103 | bool ldb_check_root() 104 | { 105 | if (!ldb_dir_exists(ldb_root)) 106 | { 107 | printf("E059 LDB root directory %s is not accessible\n", ldb_root); 108 | return false; 109 | } 110 | return true; 111 | } 112 | 113 | /** 114 | * @brief Checks if a db/table already exists 115 | * 116 | * @param db DB name string 117 | * @param table name string 118 | * @return true if exist 119 | */ 120 | bool ldb_table_exists(char *db, char*table) 121 | { 122 | char *path = malloc(LDB_MAX_PATH); 123 | sprintf(path, "%s/%s/%s", ldb_root, db, table); 124 | bool out = ldb_dir_exists(path); 125 | free(path); 126 | return out; 127 | } 128 | 129 | /** 130 | * @brief Checks if a db already exists 131 | * 132 | * @param db db string path 133 | * @return true if exist 134 | */ 135 | bool ldb_database_exists(char *db) 136 | { 137 | char *path = malloc(LDB_MAX_PATH); 138 | sprintf(path, "%s/%s", ldb_root, db); 139 | bool out = ldb_dir_exists(path); 140 | free(path); 141 | return out; 142 | } 143 | 144 | char * ldb_file_extension(char * path) 145 | { 146 | if (!path) 147 | return NULL; 148 | 149 | char * dot = strrchr(path, '.'); 150 | 151 | if (!dot) 152 | return NULL; 153 | 154 | printf("%s\n", dot); 155 | 156 | return (dot + 1); 157 | } 158 | 159 | /** 160 | * @brief Creates a directory with 0755 permissions. 161 | * If the folder already exists the folder is not replaced. 162 | * 163 | * @param path Path to the folder to create. 164 | * @return true Folder created or already exists. False in error case. 165 | */ 166 | bool ldb_create_dir(char *path) 167 | { 168 | bool result = false; 169 | char *sep = strrchr(path, '/'); 170 | if(sep != NULL) { 171 | *sep = 0; 172 | result = ldb_create_dir(path); 173 | *sep = '/'; 174 | } 175 | 176 | if (ldb_dir_exists(path)) 177 | { 178 | result = true; 179 | } 180 | else if (!mkdir(path, 0755)) 181 | { 182 | result = true; 183 | //fprintf(stderr, "Created dir: %s\n", path); 184 | } 185 | 186 | return result; 187 | } 188 | -------------------------------------------------------------------------------- /src/hex.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/hex.c 4 | * 5 | * Hexadecimal and numeric conversions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file hex.c 25 | * @date 12 Jul 2020 26 | * @brief Contains HEX print and process utilities 27 | * 28 | * //TODO Long description 29 | * 30 | * @see https://github.com/scanoss/ldb/blob/master/src/hex.c 31 | */ 32 | #include "ldb.h" 33 | #include "logger.h" 34 | /** 35 | * @brief Prints a hexdump of 'len' bytes from 'data' organized 'width' columns 36 | * 37 | * 38 | * Example: 39 | * 0000 54686973206973206120 This is a 40 | * 0010 737472696e6720746861 string tha 41 | * 0020 74207761732068657864 t was hexd 42 | * 0030 756d706564202e2e2e2e umped .... 43 | * 0040 2e2e2e2e2e2e2e2e2e2e .......... 44 | * 0050 2e2e2e2e2e2e2e2e2e2e .......... 45 | * 46 | * 47 | * @param data Buffer to print 48 | * @param len Length of buffer 49 | * @param width width of columns 50 | */ 51 | void ldb_hexprint(uint8_t *data, uint32_t len, uint8_t width) 52 | { 53 | uint8_t b16[] = "0123456789abcdef"; 54 | for (int i = 0; i <= width * (int)((len + width) / width); i++) 55 | if (i && !(i % width)) 56 | { 57 | printf("%04d ", i - width); 58 | for (int t = i - width; t < i; t++) 59 | printf("%c%c", t < len ? b16[(data[t] & 0xF0) >> 4] : 32, t < len ? b16[data[t] & 0x0F] : 32); 60 | printf(" "); 61 | for (int t = i - width; t < i; t++) 62 | printf("%c", t < len ? ((data[t] > 31 && data[t] < 127) ? data[t] : 46) : 32); 63 | printf("\n"); 64 | if (i == len) 65 | break; 66 | } 67 | } 68 | 69 | /** 70 | * @brief Fixed width recordset handler for hexdump 71 | * 72 | * For Example: See in function ldb_hexprint(); 73 | * 74 | * @param key block key 75 | * @param subkey block subkey 76 | * @param subkey_ln block subkey lenght 77 | * @param data Buffer to print 78 | * @param len Length of buffer 79 | * @param iteration Not used 80 | * @param ptr Pointer to integer. Stores the number of columns to be printed. 81 | * @return false Always return false. It 82 | */ 83 | bool ldb_hexprint16(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t len, int iteration, void *ptr) 84 | { 85 | int *width = ptr; 86 | for (int i = 0; i < LDB_KEY_LN; i++) printf("%02x", key[i]); 87 | for (int i = 0; i < subkey_ln; i++) printf("%02x", subkey[i]); 88 | printf("\n"); 89 | ldb_hexprint(data, len, *width); 90 | printf("\n"); 91 | return false; 92 | } 93 | 94 | /** 95 | * @brief Converts binary to a string of hex digits 96 | * Does the opposite of ldb_hex_to_bin(); 97 | * 98 | * @param bin binary data to convert 99 | * @param len Length in bytes of the binary data 100 | * @param out Buffer to write the hex string 101 | */ 102 | void ldb_bin_to_hex(uint8_t *bin, uint32_t len, char *out) 103 | { 104 | *out = 0; 105 | for (uint32_t i = 0; i < len; i++) 106 | sprintf(out + strlen(out), "%02x", bin[i]); 107 | } 108 | 109 | 110 | /** 111 | * @brief Converts a string of hex digits to binary. 112 | * Does the opposite of ldb_hexprint() and ldb_bin_to_hex(); 113 | * 114 | * Example: ldb_hex_to_bin("48656c6c6f20576f726c6421", 24, out); 115 | * 48656c6c6f20576f726c6421 -> Hello World! 116 | * 117 | * @param hex String representing hex data 118 | * @param len Lenght of the string in bytes 119 | * @param out Buffer to write the binary 120 | */ 121 | void ldb_hex_to_bin(char *hex, int len, uint8_t *out) 122 | { 123 | 124 | for (int i = 0; i < len; i += 2) { 125 | sscanf(&hex[i], "%2hhx", &out[i/2]); 126 | } 127 | } 128 | 129 | 130 | /** 131 | * @brief Verify if a string only contains hexadecimal characters 132 | * 133 | * @param str String to verify 134 | * @return true If the string only contains hexadecimal characters. False otherwise. 135 | */ 136 | bool ldb_valid_hex(char *str) 137 | { 138 | if (strlen(str) % 2) return false; 139 | if (strlen(str) < 2) return false; 140 | for (int i = 0; i < strlen(str); i++) 141 | { 142 | char h = str[i]; 143 | if (h < '0' || (h > '9' && h < 'a') || h > 'f') return false; 144 | } 145 | return true; 146 | } 147 | 148 | /** 149 | * @brief Write an unsigned long integer (40-bit) in the provided ldb_sector at the current location 150 | * 151 | * @param ldb_sector LDB sector to write to 152 | * @param value Value to write 153 | */ 154 | void ldb_uint40_write(FILE *ldb_sector, uint64_t value) 155 | { 156 | fwrite((uint8_t*)&value, 1, 5, ldb_sector); 157 | } 158 | 159 | /** 160 | * @brief Write an unsigned long integer (32-bit) in the provided ldb_sector at the current location 161 | * 162 | * @param ldb_sector LDB sector to write to 163 | * @param value Value to write 164 | */ 165 | void ldb_uint32_write(FILE *ldb_sector, uint32_t value) 166 | { 167 | fwrite((uint8_t*)&value, 1, 4, ldb_sector); 168 | } 169 | 170 | /** 171 | * @brief Read an unsigned long integer (32-bit) from the provided ldb_sector at the current location 172 | * 173 | * @param ldb_sector LDB sector to read from 174 | * @return uint32_t Value readed 175 | */ 176 | uint32_t ldb_uint32_read(FILE *ldb_sector) 177 | { 178 | uint32_t out = 0; 179 | if (!fread((uint8_t*)&out, 1, 4, ldb_sector)) 180 | { 181 | log_debug("Warning: cannot read LDB sector\n"); 182 | ldb_read_failure = true; 183 | 184 | } 185 | return out; 186 | } 187 | 188 | /** 189 | * @brief Read an unsigned long integer (40-bit) from the provided ldb_sector at the current location 190 | * 191 | * @param ldb_sector LDB sector to read from 192 | * @return uint64_t Value readed 193 | */ 194 | uint64_t ldb_uint40_read(FILE *ldb_sector) 195 | { 196 | uint64_t out = 0; 197 | if (!fread((uint8_t*)&out, 1, 5, ldb_sector)) 198 | { 199 | log_debug("Warning: cannot read LDB sector\n"); 200 | ldb_read_failure = true; 201 | } 202 | return out; 203 | } 204 | 205 | /** 206 | * @brief Read an unsigned integer (16-bit) from the provided ldb_sector at the current location 207 | * 208 | * @param ldb_sector LDB sector to read from 209 | * @return uint16_t Value readed 210 | */ 211 | uint16_t ldb_uint16_read(FILE *ldb_sector) 212 | { 213 | uint16_t out = 0; 214 | if (!fread((uint8_t*)&out, 1, 2, ldb_sector)) 215 | { 216 | log_debug("Warning: cannot read LDB sector\n"); 217 | ldb_read_failure = true; 218 | } 219 | return out; 220 | } 221 | 222 | /** 223 | * @brief Read an unsigned integer (16-bit) from the provided pointer. 224 | * Copy two bytes starting from the provided pointer into an unsigned short. 225 | * 226 | * @param pointer Pointer to read from 227 | * @return uint16_t Value readed 228 | */ 229 | uint16_t uint16_read(uint8_t *pointer) 230 | { 231 | uint16_t out; 232 | memcpy((uint8_t*)&out, pointer, 2); 233 | return out; 234 | } 235 | 236 | /** 237 | * @brief Write an unsigned integer (16-bit) in the provided location 238 | * Copy two bytes from the provided value into the provided pointer. 239 | * pointer must point to a memory allocated of at least 2 bytes long. 240 | * 241 | * @param pointer Pointer to write to 242 | * @param value Value to write 243 | */ 244 | void uint16_write(uint8_t *pointer, uint16_t value) 245 | { 246 | memcpy(pointer, (uint8_t*)&value, 2); 247 | } 248 | 249 | /** 250 | * @brief Read an unsigned integer (32-bit) from the provided pointer 251 | * 252 | * @param pointer Pointer to read from 253 | * @return uint32_t Value readed 254 | */ 255 | uint32_t uint32_read(uint8_t *pointer) 256 | { 257 | uint32_t out; 258 | memcpy((uint8_t*)&out, pointer, 4); 259 | return out; 260 | } 261 | 262 | /** 263 | * @brief Write an unsigned integer (32-bit) in the provided location 264 | * Copy four bytes from the provided value into the provided pointer. 265 | * pointer must point to a memory allocated of at least 4 bytes long. 266 | * 267 | * 268 | * @param pointer Pointer to write to 269 | * @param value Value to write 270 | */ 271 | void uint32_write(uint8_t *pointer, uint32_t value) 272 | { 273 | memcpy(pointer, (uint8_t*)&value, 4); 274 | } 275 | 276 | /** 277 | * @brief Read an unsigned integer (40-bit) from the provided pointer 278 | * 279 | * @param pointer Pointer to read from 280 | * @return uint64_t Value readed 281 | */ 282 | uint64_t uint40_read(uint8_t *pointer) 283 | { 284 | uint64_t out = 0; 285 | memcpy((uint8_t*)&out, pointer, 5); 286 | return out; 287 | } 288 | 289 | /** 290 | * @brief Write an unsigned integer (40-bit) in the provided location 291 | * Copy five bytes from the provided value into the provided pointer. 292 | * pointer must point to a memory allocated of at least 5 bytes long. 293 | * 294 | * @param pointer Pointer to write to 295 | * @param value Value to write 296 | */ 297 | void uint40_write(uint8_t *pointer, uint64_t value) 298 | { 299 | memcpy(pointer, (uint8_t*)&value, 5); 300 | } 301 | 302 | /** 303 | * @brief Verify if a memory block of 4 bytes is all zeros 304 | * 305 | * @param n Buffer to verify 306 | * @return true If all values are 0. False otherwise. 307 | */ 308 | bool uint32_is_zero(uint8_t *n) 309 | { 310 | if (*n == 0) 311 | if (*(n+1) == 0) 312 | if (*(n+2) == 0) 313 | if (*(n+3) == 0) return true; 314 | return false; 315 | } 316 | 317 | -------------------------------------------------------------------------------- /src/import.h: -------------------------------------------------------------------------------- 1 | #ifndef __IMPORT_H 2 | #define __IMPORT_H 3 | #include 4 | #include "ldb.h" 5 | 6 | #define IMPORT_PARAMS_NUMBER 15 7 | typedef union import_params { 8 | struct __attribute__((__packed__)) params 9 | { 10 | int delete_after_import; 11 | int keys_number; 12 | int overwrite; 13 | int sort; 14 | int version_validation; 15 | int verbose; 16 | int is_mz_table; 17 | int binary_mode; 18 | int is_wfp_table; 19 | int csv_fields; 20 | int validate_fields; 21 | int threads; 22 | int collate; 23 | int collate_max_rec; 24 | int collate_max_ram_percent; 25 | char tmp_path[LDB_MAX_PATH]; 26 | } params; 27 | int params_arr[IMPORT_PARAMS_NUMBER]; 28 | } import_params_t; 29 | 30 | typedef struct ldb_importation_config_t 31 | { 32 | char path[LDB_MAX_PATH]; 33 | char dbname[LDB_MAX_NAME]; 34 | char table[LDB_MAX_NAME]; 35 | char csv_path[LDB_MAX_PATH]; 36 | import_params_t opt; 37 | } ldb_importation_config_t; 38 | 39 | 40 | bool ldb_importation_config_parse(import_params_t * opt, char * line); 41 | bool ldb_create_db_config_default(char * dbname); 42 | int ldb_import(ldb_importation_config_t * job); 43 | uint64_t ldb_file_size(char *path); 44 | 45 | #endif -------------------------------------------------------------------------------- /src/join.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | /** 4 | * @file join.c 5 | * @date 14 Sep 2021 6 | * @brief Contains functions used for implentent minr join funtionality 7 | */ 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "ldb.h" 16 | #include "import.h" 17 | #include "logger.h" 18 | #include "ldb_error.h" 19 | 20 | /** 21 | * @brief Creates a directory with 0755 permissions. 22 | * 23 | * @param destination destination path 24 | */ 25 | void mkdir_if_not_exist(char *destination) 26 | { 27 | char *dst_dir = strdup(destination); 28 | char *dir = dirname(dst_dir); 29 | if (ldb_dir_exists(dir)) 30 | { 31 | free(dst_dir); 32 | return; 33 | } 34 | 35 | ldb_create_dir(dir); 36 | if (!ldb_dir_exists(dir)) 37 | { 38 | printf("Cannot create directory %s\n", dst_dir); 39 | free(dst_dir); 40 | exit(EXIT_FAILURE); 41 | } 42 | 43 | free(dst_dir); 44 | } 45 | 46 | /** 47 | * @brief Move a file to a new location by copying byte by byte. 48 | * If the file already exist it is overwritten. 49 | * 50 | * @param src src path 51 | * @param dst dst path 52 | * @param skip_delete if true the src file is not deleted after the copy is done. 53 | * @return true success. False otherwise. 54 | */ 55 | static bool write_file(char *src, char *dst, char * mode, bool mkdir, bool delete) { 56 | 57 | if (mkdir) 58 | { 59 | mkdir_if_not_exist(dst); 60 | } 61 | 62 | FILE *srcf = fopen(src, "rb"); 63 | if (!srcf) 64 | { 65 | printf("Cannot open source file %s\n", src); 66 | exit(EXIT_FAILURE); 67 | } 68 | 69 | FILE *dstf = fopen(dst, mode); 70 | if (!dstf) 71 | { 72 | printf("Cannot open destinstion file %s\n", dst); 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | /* Copy byte by byte */ 77 | int byte = 0; 78 | while (!feof(srcf)) 79 | { 80 | byte = fgetc(srcf); 81 | if (feof(srcf)) break; 82 | fputc(byte, dstf); 83 | } 84 | 85 | fclose(srcf); 86 | fclose(dstf); 87 | if (delete) unlink(src); 88 | return true; 89 | } 90 | 91 | bool move_file(char *src, char *dst, bool delete) 92 | { 93 | return write_file(src,dst, "wb", true, delete); 94 | } 95 | /** 96 | * @brief Append the contents of a file to the end of another file. 97 | * 98 | * @param file Origin of the data to be appended. 99 | * @param destination path to out file 100 | */ 101 | 102 | bool file_append(char *file, char *destination, bool delete) 103 | { 104 | return write_file(file,destination, "ab", false, delete); 105 | } 106 | 107 | 108 | 109 | /** 110 | * @brief join two binary files 111 | * 112 | * @param source path to the source file 113 | * @param destination path to destination file 114 | * @param snippets true if it is a snippet file 115 | * @param skip_delete true to avoid deletion 116 | */ 117 | int ldb_bin_join(char *source, char *destination, bool overwrite, bool snippets, bool delete) 118 | { 119 | /* If source does not exist, no need to join */ 120 | if (!ldb_file_exists(source)) 121 | { 122 | fprintf(stderr,"Error: File %s does not exist\n", source); 123 | return -1; 124 | } 125 | 126 | if (ldb_file_exists(destination) && !overwrite) 127 | { 128 | /* Snippet records should divide by 21 */ 129 | if (snippets) if (ldb_file_size(destination) % 21) 130 | { 131 | printf("File %s does not contain 21-byte records\n", destination); 132 | return -1; 133 | } 134 | } 135 | 136 | /* If destination does not exist. Source is moved */ 137 | else 138 | { 139 | log_info("Moving %s into %s\n", source, destination); 140 | if (!move_file(source, destination, delete)) 141 | { 142 | printf("Cannot move file\n"); 143 | return -1; 144 | } 145 | return LDB_ERROR_NOERROR; 146 | } 147 | 148 | log_info("Joining into %s\n", destination); 149 | file_append(source, destination, delete); 150 | if (delete) unlink(source); 151 | 152 | return LDB_ERROR_NOERROR; 153 | } 154 | 155 | 156 | -------------------------------------------------------------------------------- /src/join.h: -------------------------------------------------------------------------------- 1 | #ifndef __JOIN_H 2 | #define __JOIN_H 3 | void ldb_join_mz(char * table, char *source, char *destination, bool skip_delete, bool encrypted); 4 | void ldb_join_snippets(char * table, char *source, char *destination, bool skip_delete); 5 | int64_t ldb_bin_join(char *source, char *destination, bool overwrite, bool snippets, bool delete); 6 | #endif 7 | 8 | -------------------------------------------------------------------------------- /src/keys.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/keys.c 4 | * 5 | * "Dump keys" command functions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file keys.c 25 | * @date 24 Dec 2020 26 | * @brief Handle the dump keys record: Write in stdout unique keys 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/keys.c 30 | */ 31 | #include "ldb.h" 32 | /** 33 | * @brief Handle the dump keys record: Write in stdout unique keys 34 | * 35 | * @param key block key 36 | * @param subkey block subkey 37 | * @param subkey_ln block subkey lenght 38 | * @param data uint8_t pointer to data to be added 39 | * @param size data size 40 | * @param iteration number of iterations 41 | * @param ptr[out] output pointer 42 | * @return true to finish the fetch 43 | * @return false to continue the fetch 44 | */ 45 | bool ldb_dump_keys_handler(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr) 46 | { 47 | struct ldb_table *table = ptr; 48 | 49 | /* Assemble full key */ 50 | memcpy(table->current_key, key, LDB_KEY_LN); 51 | memcpy(table->current_key + LDB_KEY_LN, subkey, subkey_ln); 52 | 53 | /* Skip if same as last key */ 54 | if (ldb_reverse_memcmp(table->current_key, table->last_key, table->key_ln)) return false; 55 | 56 | /* Save into last key */ 57 | memcpy(table->last_key, table->current_key, table->key_ln); 58 | 59 | char hex[MD5_LEN * 2 + 1]; 60 | ldb_bin_to_hex(table->current_key, MD5_LEN, hex); 61 | printf("%s\n", hex); 62 | return false; 63 | } 64 | 65 | /** 66 | * @brief LDB dump keys trought stdout 67 | * 68 | * @param table input table 69 | */ 70 | void ldb_dump_keys(struct ldb_table table, int s) 71 | { 72 | /* Read each DB sector */ 73 | uint8_t k0 = s >= 0 ? s : 0; 74 | setlocale(LC_NUMERIC, ""); 75 | 76 | table.current_key = calloc(table.key_ln, 1); 77 | table.last_key = calloc(table.key_ln, 1); 78 | 79 | do { 80 | uint8_t *sector = ldb_load_sector(table, &k0); 81 | if (sector) 82 | { 83 | /* Read each one of the (256 ^ 3) list pointers from the map */ 84 | uint8_t k[LDB_KEY_LN]; 85 | k[0] = k0; 86 | for (int k1 = 0; k1 < 256; k1++) 87 | for (int k2 = 0; k2 < 256; k2++) 88 | for (int k3 = 0; k3 < 256; k3++) 89 | { 90 | k[1] = k1; 91 | k[2] = k2; 92 | k[3] = k3; 93 | 94 | /* Process records */ 95 | ldb_fetch_recordset(sector, table, k, true, ldb_dump_keys_handler, &table); 96 | 97 | } 98 | free(sector); 99 | if (s >=0) 100 | break; 101 | } 102 | } while (k0++ < 255); 103 | 104 | fflush(stdout); 105 | 106 | free(table.current_key); 107 | free(table.last_key); 108 | } 109 | -------------------------------------------------------------------------------- /src/ldb.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/ldb.c 4 | * 5 | * LDB Database - A mapped linked-list database 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file ldb.c 25 | * @date 12 Jul 2020 26 | * @brief LDB commands definition 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/keys.c 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | 43 | #include "ldb.h" 44 | 45 | 46 | /* Global */ 47 | char ldb_root[] = "/var/lib/ldb"; 48 | char ldb_lock_path[] = "/dev/shm/ldb.lock"; 49 | int ldb_cmp_width = 0; 50 | 51 | bool ldb_read_failure = false; 52 | /** 53 | * @brief Display LDB error and exit program 54 | * 55 | * @param txt error string 56 | */ 57 | void ldb_error (char *txt) 58 | { 59 | fprintf(stdout, "%s\n", txt); 60 | exit(EXIT_FAILURE); 61 | } 62 | 63 | /** 64 | * @brief Print ldb version 65 | * 66 | */ 67 | void ldb_version(char **version) 68 | { 69 | if (!version) 70 | printf("ldb-%s\n", LDB_VERSION); 71 | else 72 | *version = strdup(LDB_VERSION); 73 | } 74 | 75 | -------------------------------------------------------------------------------- /src/ldb.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/ldb.h 4 | * 5 | * LDB Database - A mapped linked-list database 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | #ifndef _LDB_GLOBAL_ 24 | #define _LDB_GLOBAL_ 25 | 26 | #include "./ldb/definitions.h" 27 | #include "./ldb/types.h" 28 | #include "./ldb/mz.h" 29 | 30 | #define LDB_VERSION "4.1.4" 31 | 32 | #define LDB_TABLE_DEFINITION_UNDEFINED -1 33 | #define LDB_TABLE_DEFINITION_STANDARD 0 34 | #define LDB_TABLE_DEFINITION_ENCRYPTED 1 35 | #define LDB_TABLE_DEFINITION_COMPRESSED 4 36 | #define LDB_TABLE_DEFINITION_MZ 2 37 | 38 | extern bool ldb_read_failure; 39 | 40 | bool ldb_file_exists(char *path); 41 | bool ldb_dir_exists(char *path); 42 | bool ldb_locked(); 43 | void ldb_error (char *txt); 44 | void ldb_prepare_dir(char *path); 45 | void ldb_lock(char * db_table); 46 | void ldb_unlock(char * db_table); 47 | void ldb_create_sector(char *sector_path); 48 | void ldb_uint40_write(FILE *ldb_sector, uint64_t value); 49 | void ldb_uint32_write(FILE *ldb_sector, uint32_t value); 50 | uint32_t ldb_uint32_read(FILE *ldb_sector); 51 | uint64_t ldb_uint40_read(FILE *ldb_sector); 52 | uint16_t ldb_uint16_read(FILE *ldb_sector); 53 | uint16_t uint16_read(uint8_t *pointer); 54 | void uint16_write(uint8_t *pointer, uint16_t value); 55 | uint32_t uint32_read(uint8_t *pointer); 56 | void uint32_write(uint8_t *pointer, uint32_t value); 57 | uint64_t uint40_read(uint8_t *pointer); 58 | void uint40_write(uint8_t *pointer, uint64_t value); 59 | uint64_t ldb_map_pointer_pos(uint8_t *key); 60 | uint64_t ldb_list_pointer(FILE *ldb_sector, uint8_t *key); 61 | uint64_t ldb_last_node_pointer(FILE *ldb_sector, uint64_t list_pointer); 62 | void ldb_update_list_pointers(FILE *ldb_sector, uint8_t *key, uint64_t list, uint64_t new_node); 63 | int ldb_node_write (struct ldb_table table, FILE *ldb_sector, uint8_t *key, uint8_t *data, uint32_t dataln, uint16_t records); 64 | uint64_t ldb_node_read (uint8_t *sector, struct ldb_table table, FILE *ldb_sector, uint64_t ptr, uint8_t *key, uint32_t *bytes_read, uint8_t **out, int max_node_size); 65 | char *ldb_sector_path (struct ldb_table table, uint8_t *key, char *mode); 66 | FILE *ldb_open (struct ldb_table table, uint8_t *key, char *mode); 67 | bool ldb_close(FILE * sector); 68 | int ldb_close_unlock(FILE *fp); 69 | void ldb_node_unlink (struct ldb_table table, uint8_t *key); 70 | void ldb_hexprint(uint8_t *data, uint32_t len, uint8_t width); 71 | void ldb_hex_to_bin(char *hex, int hex_ln, uint8_t *out); 72 | void ldb_bin_to_hex(uint8_t *bin, uint32_t len, char *out); 73 | bool ldb_check_root(); 74 | struct ldb_table ldb_read_cfg(char *db_table); 75 | void ldb_write_cfg(char *db, char *table, int keylen, int reclen, int keys, int definitions); 76 | bool ldb_valid_table(char *table); 77 | bool ldb_syntax_check(char *command, int *command_nr, int *word_nr); 78 | void ldb_version(char **version); 79 | bool ldb_database_exists(char *db); 80 | bool ldb_table_exists(char *db, char*table); 81 | bool ldb_create_table_new(char *db, char *table, int keylen, int reclen, int keys, int definitions); 82 | bool ldb_create_table(char *db, char *table, int keylen, int reclen); 83 | bool ldb_create_database(char *database); 84 | struct ldb_recordset ldb_recordset_init(char *db, char *table, uint8_t *key); 85 | void ldb_list_unlink(FILE *ldb_sector, uint8_t *key); 86 | uint8_t *ldb_load_sector (struct ldb_table table, uint8_t *key); 87 | bool ldb_validate_node(uint8_t *node, uint32_t node_size, int subkey_ln); 88 | //bool uint32_is_zero(uint8_t *n); 89 | bool ldb_key_exists(struct ldb_table table, uint8_t *key); 90 | bool ldb_key_in_recordset(uint8_t *rs, uint32_t rs_len, uint8_t *subkey, uint8_t subkey_ln); 91 | uint32_t ldb_fetch_recordset(uint8_t *sector, struct ldb_table table, uint8_t* key, bool skip_subkey, bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *), void *void_ptr); 92 | bool ldb_hexprint_width(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr); 93 | void ldb_sector_update(struct ldb_table table, uint8_t *key); 94 | void ldb_sector_erase(struct ldb_table table, uint8_t *key); 95 | void ldb_dump(struct ldb_table table, int hex_bytes, int sector); 96 | void ldb_dump_keys(struct ldb_table table, int s); 97 | 98 | char * ldb_file_extension(char * path); 99 | bool ldb_create_dir(char *path); 100 | bool ldb_reverse_memcmp(uint8_t *a, uint8_t *b, int bytes); 101 | 102 | void md5_string(const unsigned char *input, int len, unsigned char output[16]); 103 | uint8_t * md5_file(char *path); 104 | 105 | #define MD5(a, b, c) md5_string(a, b, c) 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /src/ldb/collate.h: -------------------------------------------------------------------------------- 1 | #ifndef __COLLATE_H 2 | #define __COLLATE_H 3 | #include "../ldb.h" 4 | struct ldb_collate_data; 5 | typedef bool (*collate_handler)(struct ldb_collate_data *collate, uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size); 6 | 7 | 8 | typedef struct tuple_t 9 | { 10 | uint8_t key[MD5_LEN]; 11 | int keys; 12 | char * data; 13 | } tuple_t; 14 | 15 | typedef struct job_delete_tuples_t 16 | { 17 | tuple_t ** tuples; 18 | int tuples_number; 19 | int key_ln; 20 | int keys_number; 21 | collate_handler handler; 22 | } job_delete_tuples_t; 23 | 24 | struct ldb_collate_data 25 | { 26 | void *data; 27 | void *tmp_data; 28 | long data_ptr; 29 | int table_key_ln; 30 | int table_rec_ln; 31 | int max_rec_ln; 32 | long rec_width; 33 | long rec_count; 34 | FILE *out_sector; 35 | struct ldb_table in_table; 36 | struct ldb_table out_table; 37 | uint8_t last_key[LDB_KEY_LN]; 38 | time_t last_report; 39 | bool merge; 40 | long del_count; 41 | long key_rec_count; 42 | job_delete_tuples_t * del_tuples; 43 | collate_handler handler; 44 | }; 45 | bool ldb_collate_init(struct ldb_collate_data * collate, struct ldb_table table, struct ldb_table out_table, int max_rec_ln, bool merge, uint8_t sector); 46 | void ldb_collate_sector(struct ldb_collate_data *collate, uint8_t sector, uint8_t *sector_mem); 47 | int ldb_collate_load_tuples_to_delete(job_delete_tuples_t* job, char * buffer, char * d, struct ldb_table table); 48 | void ldb_collate(struct ldb_table table, struct ldb_table out_table, int max_rec_ln, bool merge, int p_sector, collate_handler handler); 49 | void ldb_collate_delete(struct ldb_table table, struct ldb_table out_table, job_delete_tuples_t * delete, collate_handler handler); 50 | 51 | #endif -------------------------------------------------------------------------------- /src/ldb/definitions.h: -------------------------------------------------------------------------------- 1 | #ifndef __DEFINITIONS_H 2 | #define __DEFINITIONS_H 3 | 4 | #define LDB_CFG_PATH "/usr/local/etc/scanoss/ldb/" 5 | #define LDB_MAX_PATH 1024 6 | #define LDB_MAX_NAME 64 7 | #define LDB_MAX_RECORDS 500000 // Max number of records per list 8 | #define LDB_MAX_REC_LN 65535 9 | #define LDB_KEY_LN 4 // Main LDB key: 32-bit 10 | #define LDB_PTR_LN 5 // Node pointers: 40-bit 11 | #define LDB_MAP_SIZE (256 * 256 * 256 * 5) // Size of sector map 12 | #define LDB_MAX_NODE_DATA_LN (4 * 1048576) // Maximum length for a data record in a node (4Mb) 13 | #define LDB_MAX_NODE_LN ((256 * 256 * 18) - 1) 14 | #define LDB_MAX_COMMAND_SIZE (64 * 1024) // Maximum length for an LDB command statement 15 | #define COLLATE_REPORT_SEC 5 // Report interval for collate status 16 | #define MD5_LEN 16 17 | #define MD5_LEN_HEX 32 18 | #define BUFFER_SIZE 1048576 19 | #define MAX_CSV_LINE_LEN 1024 20 | 21 | extern char ldb_root[]; 22 | extern char ldb_lock_path[]; 23 | extern char *ldb_commands[]; 24 | extern int ldb_commands_count; 25 | extern int ldb_cmp_width; 26 | 27 | #endif -------------------------------------------------------------------------------- /src/ldb/mz.h: -------------------------------------------------------------------------------- 1 | #ifndef __MZ_LDB_H 2 | #define __MZ_LDB_H 3 | 4 | #include "../ldb.h" 5 | #include "collate.h" 6 | 7 | /* MZ */ 8 | #define MZ_CACHE_SIZE 16384 9 | #define MZ_FILES 65536 10 | #define MZ_HEAD 18 // Head contains 14 bytes of the MD5 + 4 bytes for compressed SIZE 11 | #define MZ_MD5 14 12 | #define MZ_SIZE 4 13 | #define MZ_MAX_FILE (4 * 1048576) 14 | 15 | struct mz_cache_item 16 | { 17 | uint16_t length; 18 | uint8_t data[MZ_CACHE_SIZE]; 19 | }; 20 | 21 | struct mz_job 22 | { 23 | char path[LDB_MAX_PATH]; // Path to mz file 24 | uint8_t *mz; // Pointer to entire mz file contents 25 | uint64_t mz_ln; // MZ file length 26 | uint8_t mz_id[2]; // MZ file ID (first two bytes of MD5s) 27 | uint8_t *id; // MZ record ID 28 | uint64_t ln; // MZ record length 29 | char md5[33]; // MZ record hex ID (MD5) 30 | char *data; // Pointer to uncompressed data 31 | uint64_t data_ln; // Uncompressed data length 32 | uint8_t *zdata; // Pointer to compressed data 33 | uint64_t zdata_ln; // Compressed data length 34 | void *ptr; // Pointer to temporary data 35 | uint64_t ptr_ln; // Temporary data length 36 | uint32_t dup_c; // Duplicated counter 37 | uint32_t igl_c; // Ignored counter 38 | uint32_t orp_c; // Orphan file counter 39 | uint32_t exc_c; // Excluded file counter 40 | uint32_t min_c; // Under MIN_FILE_SIZE file counter 41 | bool check_only; // Perform only an mz validation (without list output) 42 | bool dump_keys; // Dump unique keys to STDOUT 43 | bool orphan_rm; // Remove orphans 44 | uint8_t *key; // File key to be printed via STDOUT (-k) 45 | uint8_t *xkeys; // List of keys to be excluded in (-o/-O)ptimisation 46 | uint64_t xkeys_ln; // Length of xkeys 47 | void *licenses; // Array of known license identifiers 48 | int license_count; // Number of known license identifiers 49 | bool key_found; // Used with mz_key_exists 50 | void (*decrypt) (uint8_t *data, uint32_t len); 51 | }; 52 | 53 | 54 | bool mz_key_exists(struct mz_job *job, uint8_t *key); 55 | bool mz_id_exists(uint8_t *mz, uint64_t size, uint8_t *id); 56 | uint8_t *file_read(char *filename, uint64_t *size); 57 | 58 | void mz_deflate(struct mz_job *job); 59 | #define MZ_DEFLATE(job) mz_deflate(job) 60 | 61 | void mz_id_fill(char *md5, uint8_t *mz_id); 62 | void mz_parse(struct mz_job *job, bool (*mz_parse_handler) ()); 63 | void file_write(char *filename, uint8_t *src, uint64_t src_ln); 64 | void mz_id_fill(char *md5, uint8_t *mz_id); 65 | void mz_corrupted(); 66 | void mz_add(char *mined_path, uint8_t *md5, char *src, int src_ln, bool check, uint8_t *zsrc, struct mz_cache_item *mz_cache); 67 | bool mz_check(char *path); 68 | void mz_flush(char *mined_path, struct mz_cache_item *mz_cache); 69 | void mz_list_check(struct mz_job *job); 70 | void mz_list_keys(struct ldb_table table, int sector); 71 | void mz_extract(struct mz_job *job); 72 | void mz_cat(struct mz_job *job, char *key); 73 | void ldb_mz_collate(struct ldb_table table, int p_sector); 74 | void ldb_mz_collate_delete(struct ldb_table table, job_delete_tuples_t * delete); 75 | 76 | #endif -------------------------------------------------------------------------------- /src/ldb/types.h: -------------------------------------------------------------------------------- 1 | #ifndef __TYPES_H 2 | #define __TYPES_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "definitions.h" 20 | struct ldb_table 21 | { 22 | char db[LDB_MAX_NAME]; 23 | char table[LDB_MAX_NAME]; 24 | int key_ln; 25 | int rec_ln; // data record length, otherwise 0 for variable-length data 26 | int ts_ln; // 2 or 4 (16-bit or 32-bit reserved for total sector size) 27 | bool tmp; // is this a .tmp sector instead of a .ldb? 28 | int keys; 29 | uint8_t *current_key; 30 | uint8_t *last_key; 31 | int definitions; // Table definitions: is MZ? is encrypted? 32 | }; 33 | 34 | struct ldb_recordset 35 | { 36 | char db[LDB_MAX_NAME]; 37 | char table[LDB_MAX_NAME]; 38 | FILE *sector; // Data sector file pointer 39 | uint8_t key[255]; // Data key 40 | uint8_t key_ln; // Key length: 4-255 41 | uint8_t subkey_ln; // remaining part of the key that goes into the data: key_ln - 4 42 | uint8_t rec_ln; // Fixed length of data records: 0-255, where 0 means variable-length data 43 | uint8_t *node; // Pointer to current node. This will point to mallocated memory. 44 | uint32_t node_ln; // Length of the current node 45 | uint8_t *record; // Pointer to current record within node 46 | uint64_t next_node; // Location of next node inside the 47 | uint64_t last_node; // Location of last node of the list 48 | uint8_t ts_ln; // 2 or 4 (16-bit or 32-bit reserved for total sector size) 49 | }; 50 | 51 | typedef bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *); 52 | 53 | 54 | 55 | #endif -------------------------------------------------------------------------------- /src/ldb_error.h: -------------------------------------------------------------------------------- 1 | #ifndef LDB_ERROR_H 2 | #define LDB_ERROR_H 3 | 4 | #define LDB_ERROR_NOERROR 0 5 | #define LDB_ERROR_DATA_RECORD_SIZE_EXCEED -53 //E053 Data record size exceeded 6 | #define LDB_ERROR_DATA_SECTOR_CORRUPTED -56 // "E056 Data sector corrupted 7 | #define LDB_ERROR_MAP_CORRUPTED -57//E057 Map location 8 | #define LDB_ERROR_NODE_WRITE_FAILS -58//E058 Error writing node 9 | #define LDB_ERROR_NODE_SIZE_INVALID -60 //E060 Unsupported node_length size (must be 2 or 4 bytes) 10 | #define LDB_ERROR_RECORD_LENGHT_INVAID -76 //E076 Max record length should equal fixed record length 11 | #define LDB_ERROR_CSV_WRONG_ENCODING -80 // E080 the csv file has an incorrect encoding 12 | 13 | #define LDB_ERROR_MEM_NOMEM -200 //no memory available 14 | #endif -------------------------------------------------------------------------------- /src/ldb_string.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/string.c 4 | * 5 | * String handling routines 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file string.c 25 | * @date 12 Jul 2020 26 | * @brief String utils 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/string.c 30 | */ 31 | 32 | #include "ldb.h" 33 | #include "ldb_string.h" 34 | /** 35 | * @brief Verifies if a buffer contains printable characters 36 | * 37 | * @param str String to verify 38 | * @return true String contains printable characters, false otherwise. 39 | */ 40 | 41 | bool ldb_valid_ascii(char *str) 42 | { 43 | if (strlen(str) < 1) return false; 44 | for (int i = 0; i < strlen(str); i++) 45 | if (str[i] < 33 || str[i] > 126) return false; 46 | return true; 47 | } 48 | 49 | /** 50 | * @brief remove whitespace characters from the start and end of a string 51 | * 52 | * @param str String to work on. 53 | */ 54 | void ldb_trim(char *str) 55 | { 56 | int i = 0; 57 | 58 | /* Left trim */ 59 | int len = strlen(str); 60 | for (i = 0; i < len; i++) if (!isspace(str[i])) break; 61 | if (i) memmove(str, str + i, strlen(str + i) + 1); 62 | 63 | /* Right trim */ 64 | len = strlen(str); 65 | for (i = len - 1; i >= 0 ; i--) if (!isspace(str[i])) break; 66 | str[i + 1] = 0; 67 | } 68 | 69 | /** 70 | * @brief Divides a string into two when the first occurrence of a delimiter is found. 71 | * In the position where the delimiter was found, a null character is added. This means that the first part of the string is in the input buffer. 72 | * To obtain the position of the second part use the return value. 73 | * 74 | * @param string The string to be split. 75 | * @param separator The delimiter to split the string on. 76 | * @return int Pointer to the second part of the string. 77 | */ 78 | int ldb_split_string(char *string, char separator) 79 | { 80 | int pos; 81 | for (pos = 0; pos < strlen(string); pos++) if (string[pos] == separator) break; 82 | string[pos] = 0; 83 | return pos + 1; 84 | } 85 | 86 | /** 87 | * @brief Verifies if a string is a valid dbname/tablename 88 | * To identificate a table, a string must have the following format: "dbname/tablename" 89 | * 90 | * @param str String to verify 91 | * @return true valid dbname/tablename, false otherwise. 92 | */ 93 | bool ldb_valid_name(char *str) 94 | { 95 | if (strlen(str) >= LDB_MAX_NAME) return false; 96 | if (strstr(str, "/")) return false; 97 | if (strstr(str, ".")) return false; 98 | return true; 99 | } 100 | 101 | /** 102 | * @brief Gives the length of the first word in a string. 103 | * It counts how many characters are before the first space. 104 | * 105 | * @param text String to be searched. 106 | * @return int Characters found before the first space (Space not included) 107 | */ 108 | int ldb_word_len(char *text) 109 | { 110 | for (int i=0; i (strlen(table) - 1) || c != 1) 135 | { 136 | printf("E060 Table name format should be dbname/tablename\n"); 137 | return false; 138 | } 139 | 140 | // Verify that db/table path is not too long 141 | if (strlen(table) + strlen(ldb_root) + 1 >= LDB_MAX_PATH) 142 | { 143 | printf("E061 db/table name is too long\n"); 144 | return false; 145 | } 146 | 147 | bool out = true; 148 | 149 | char *db_path = malloc(LDB_MAX_PATH); 150 | sprintf(db_path, "%s/%s", ldb_root, table); 151 | db_path[strlen(ldb_root) + 1 + s] = 0; 152 | 153 | char *table_path = malloc(LDB_MAX_PATH); 154 | sprintf(table_path, "%s/%s", ldb_root, table); 155 | 156 | // Verify that db exists 157 | 158 | if (!ldb_dir_exists(db_path)) 159 | { 160 | printf("E062 Database %s does not exist\n", db_path + strlen(ldb_root) + 1); 161 | out = false; 162 | } 163 | 164 | // Verify that table exists 165 | else if (!ldb_dir_exists(table_path)) 166 | { 167 | printf("E063 Table %s does not exist\n", table); 168 | out = false; 169 | } 170 | 171 | free(db_path); 172 | free(table_path); 173 | 174 | return out; 175 | } 176 | 177 | 178 | /** 179 | * @brief Counts number of words in normalized text 180 | * A word is considered to be a sequence of characters separated by spaces. 181 | * 182 | * @param text String to be searched. 183 | * @return int Return the number of words in the text. 184 | */ 185 | int ldb_word_count(char *text) 186 | { 187 | int words = 1; 188 | for (int i = 0; i < strlen(text); i++) if (text[i] == ' ') words++; 189 | return words; 190 | } 191 | 192 | /** 193 | * @brief Returns a pointer to a string containing the n word of the (normalized) list 194 | * 195 | * Example: ldb_extract_word(3, "This is a test"); returns "a" 196 | * 197 | * @param n The word number to extract. Starts at 1. 198 | * @param wordlist A list of words separated by spaces. 199 | * @return char* Pointer to the nth word in the list. 200 | */ 201 | char *ldb_extract_word(int n, char *wordlist) 202 | { 203 | int word_start = 0; 204 | char *out = calloc(LDB_MAX_COMMAND_SIZE + 1, 1); 205 | int limit = strlen(wordlist); 206 | if (limit > LDB_MAX_COMMAND_SIZE) limit = LDB_MAX_COMMAND_SIZE; 207 | 208 | // Look for word start 209 | if (n>1) 210 | { 211 | int c = 2; 212 | for (int i = 1; i < limit; i++) 213 | { 214 | if (wordlist[i] == ' ') 215 | { 216 | if (c++ == n) 217 | { 218 | word_start = i + 1; 219 | break; 220 | } 221 | } 222 | } 223 | if (word_start == 0) return out; 224 | } 225 | 226 | // Copy desired word to out 227 | int bytes = ldb_word_len(wordlist + word_start); 228 | if (bytes > LDB_MAX_COMMAND_SIZE) bytes = LDB_MAX_COMMAND_SIZE; 229 | 230 | memcpy(out, wordlist + word_start, bytes); 231 | return out; 232 | 233 | } 234 | -------------------------------------------------------------------------------- /src/ldb_string.h: -------------------------------------------------------------------------------- 1 | #ifndef _LDB_STRING_H 2 | #define __LDB_STRING_H 3 | #include "ldb.h" 4 | bool ldb_asciiprint(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr); 5 | bool ldb_csvprint(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr); 6 | void ldb_trim(char *str); 7 | int ldb_word_len(char *text); 8 | int ldb_word_count(char *text); 9 | bool ldb_valid_hex(char *str); 10 | bool ldb_valid_ascii(char *str); 11 | int ldb_split_string(char *string, char separator); 12 | bool ldb_valid_name(char *str); 13 | char *ldb_extract_word(int n, char *wordlist); 14 | #endif -------------------------------------------------------------------------------- /src/ldb_wrapper.c: -------------------------------------------------------------------------------- 1 | #include "ldb_wrapper.h" 2 | /** 3 | * @brief Queries a LDB table given a key 4 | * 5 | * @param dbtable / to be queried 6 | * @param key key string to search 7 | */ 8 | T_RawRes * ldb_query_raw(char *dbtable, char *key) 9 | { 10 | /* Extract values from command */ 11 | uint8_t *keybin = malloc(LDB_MAX_NODE_LN); 12 | char *rs = malloc(LDB_MAX_NODE_DATA_LN); 13 | if (ldb_valid_table(dbtable)) 14 | { 15 | if (strlen(key) < 8) printf("E071 Key length cannot be less than 32 bits\n"); 16 | else 17 | { 18 | // Convert key to binary 19 | ldb_hex_to_bin(key, strlen(key), keybin); 20 | int key_ln = (int) strlen(key) / 2; 21 | 22 | // Assembly ldb table structure 23 | struct ldb_table ldbtable = ldb_read_cfg(dbtable); 24 | 25 | // Verify that provided key matches table key_ln (or main LDB_KEY_LEN) 26 | if ((key_ln != ldbtable.key_ln) && (key_ln != LDB_KEY_LN)) 27 | printf("E073 Provided key length is invalid\n"); 28 | 29 | else 30 | { 31 | T_RawRes *results =malloc(sizeof( T_RawRes)); 32 | results->data=malloc(LDB_MAX_NODE_DATA_LN); 33 | results->size=0; 34 | results->capacity=LDB_MAX_NODE_DATA_LN; 35 | ldb_fetch_recordset(NULL, ldbtable, keybin, false, ldb_dump_row, results); 36 | 37 | free(dbtable); 38 | free(keybin); 39 | free(rs); 40 | return results; 41 | } 42 | } 43 | } 44 | free(dbtable); 45 | free(keybin); 46 | free(rs); 47 | return NULL; 48 | } 49 | /** 50 | * @brief Function handle to retrieve a recordset 51 | * @details Appends to T_RawRes output a record for a that key. As result size is unknown, 52 | * this function realocate memory in chunks. 53 | * @param key key of the record to be fetched 54 | * @param subkey subkey of the record to be fetched 55 | * @param subkey_ln number of byes considering a subkey 56 | * @param data data fetched 57 | * @param size size of the fetched record 58 | * @param iteration record index 59 | * @param ptr result pointer 60 | */ 61 | 62 | 63 | bool ldb_dump_row(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr) { 64 | 65 | T_RawRes *r=ptr; 66 | if (r->size + size > r->capacity) { 67 | size_t new_capacity = r->capacity + 2*LDB_MAX_NODE_DATA_LN; 68 | while (r->size + size > new_capacity) { 69 | new_capacity += 2*LDB_MAX_NODE_DATA_LN; 70 | } 71 | uint8_t *new_data = realloc(r->data, new_capacity); 72 | if (!new_data) { 73 | perror("Failed to reallocate memory"); 74 | exit(EXIT_FAILURE); 75 | } 76 | r->data = new_data; 77 | r->capacity = new_capacity; 78 | } 79 | 80 | memcpy(&r->data[r->size],&size,4); 81 | memcpy(&r->data[r->size+4],data,size); 82 | r->size+=size+4; 83 | 84 | return false; 85 | } 86 | -------------------------------------------------------------------------------- /src/ldb_wrapper.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/ldb_wrapper.h 4 | * 5 | * This file defines an entry point to query database from a shared library 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | #ifndef _LDB_WRAPPER_ 24 | #define _LDB_WRAPPER_ 25 | 26 | #include "./ldb/definitions.h" 27 | 28 | #include "./ldb/definitions.h" 29 | #include "./ldb/types.h" 30 | #include "./ldb/mz.h" 31 | 32 | typedef struct { 33 | uint32_t size; 34 | uint32_t capacity; 35 | uint8_t *data; 36 | } T_RawRes; 37 | 38 | 39 | T_RawRes * ldb_query_raw(char *dbtable, char *key); 40 | bool ldb_dump_row(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr) ; 41 | 42 | #endif -------------------------------------------------------------------------------- /src/lock.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/lock.c 4 | * 5 | * DB Locking mechanisms 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file lock.c 25 | * @date 12 Jul 2020 26 | * @brief Contains LDB lock functions 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/lock.c 30 | */ 31 | 32 | #include 33 | #include "ldb.h" 34 | /** 35 | * @brief Verifies if the db is locked. 36 | * Reads the file ldb.lock and if exists the db is locked. Otherwise is free to use. 37 | * 38 | * @return true if the db is locked, false otherwise. 39 | */ 40 | bool ldb_locked(char * db_name) 41 | { 42 | char file_lock[LDB_MAX_PATH]; 43 | 44 | char * db = strdup(db_name); 45 | char * sub = strrchr(db,'/'); 46 | 47 | // Replace the '/' by '.' in the lock file 48 | if (sub) 49 | *sub = '.'; 50 | 51 | sprintf(file_lock,"%s.%s", ldb_lock_path, db); 52 | free(db); 53 | 54 | return ldb_file_exists (file_lock); 55 | } 56 | 57 | /** 58 | * @brief Lock LDB for writing 59 | * 60 | */ 61 | void ldb_lock(char * db_table) 62 | { 63 | pid_t pid = getpid(); 64 | char file_lock[LDB_MAX_PATH]; 65 | 66 | char * db_name = strdup(db_table); 67 | char * sub = strrchr(db_name,'/'); 68 | 69 | // Replace the '/' by '.' in the lock file 70 | if (sub) 71 | *sub = '.'; 72 | 73 | sprintf(file_lock,"%s.%s", ldb_lock_path, db_name); 74 | free (db_name); 75 | 76 | if (ldb_locked(db_table)) 77 | { 78 | fprintf(stderr, "Lock file: %s exists\n", file_lock); 79 | ldb_error("E051 Concurrent ldb writing not supported"); 80 | } 81 | 82 | /* Write lock file */ 83 | FILE *lock = fopen (file_lock, "wb"); 84 | 85 | if (!lock) 86 | { 87 | printf("Failed to create lock file %s\n", file_lock); 88 | exit(1); 89 | } 90 | 91 | if (!fwrite (&pid, 4, 1, lock)) printf("Warning: cannot write lock file\n"); 92 | fclose (lock); 93 | 94 | /* Validate lock file */ 95 | lock = fopen (file_lock, "rb"); 96 | if (!fread (&pid, 4, 1, lock)) printf("Warning: cannot read lock file\n"); 97 | fclose(lock); 98 | 99 | if (pid != getpid()) ldb_error ("E052 Concurrent ldb writing is not supported. (check /dev/shm/ldb.lock)"); 100 | } 101 | 102 | /** 103 | * @brief Unlock LDB 104 | * 105 | */ 106 | void ldb_unlock(char * db_table) 107 | { 108 | char file_lock[LDB_MAX_PATH]; 109 | char * db_name = strdup(db_table); 110 | char * sub = strrchr(db_name,'/'); 111 | 112 | // Replace the '/' by '.' in the lock file 113 | if (sub) 114 | *sub = '.'; 115 | 116 | sprintf(file_lock,"%s.%s", ldb_lock_path, db_name); 117 | free(db_name); 118 | unlink(file_lock); 119 | 120 | } 121 | 122 | -------------------------------------------------------------------------------- /src/logger.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "ldb.h" 4 | #include "logger.h" 5 | #include 6 | #include 7 | 8 | int logger_offset = 0; 9 | struct winsize logger_window; 10 | #define gotoxy(x,y) fprintf(stderr,"\033[%d;%dH", (y), (x)) 11 | 12 | #define LOGGER_DIR "/var/log/scanoss/ldb/" 13 | char import_logger_path[LDB_MAX_PATH] = "\0"; 14 | static pthread_mutex_t logger_lock; 15 | static pthread_t * threads_list = NULL; 16 | static int threads_number = 0; 17 | static log_level_t level = LOG_BASIC; 18 | static double progress_timer = 0; 19 | 20 | char animation[] = {'|', '/', '-','\\'}; 21 | int animation_index = 0; 22 | 23 | int yi = 0; 24 | static bool first_cls = false; 25 | static bool quiet = false; 26 | 27 | void logger_basic(const char * fmt, ...) 28 | { 29 | if (level != LOG_BASIC || quiet) 30 | return; 31 | pthread_mutex_lock(&logger_lock); 32 | if (animation_index >= sizeof(animation)) 33 | animation_index = 0; 34 | 35 | struct timeval t; 36 | gettimeofday(&t, NULL); 37 | double tmp = (double)(t.tv_usec) / 1000000 + (double)(t.tv_sec); 38 | 39 | if (!fmt && tmp - progress_timer < 2) 40 | { 41 | pthread_mutex_unlock(&logger_lock); 42 | return; 43 | } 44 | 45 | progress_timer = tmp; 46 | 47 | if (fmt) 48 | { 49 | va_list args; 50 | va_start(args, fmt); 51 | char * string; 52 | vasprintf(&string, fmt, args); 53 | fprintf(stderr, "\33[2K\r%c Import in progress: %s", animation[animation_index], string); 54 | fflush(stderr); 55 | free(string); 56 | va_end(args); 57 | } 58 | else 59 | { 60 | fprintf(stderr, "\r%c", animation[animation_index]); 61 | } 62 | animation_index++; 63 | pthread_mutex_unlock(&logger_lock); 64 | } 65 | 66 | void log_info(const char * fmt, ...) 67 | { 68 | //logger_basic(NULL); 69 | 70 | pthread_mutex_lock(&logger_lock); 71 | 72 | char * string = NULL; 73 | //save to log file 74 | if (fmt) 75 | { 76 | va_list args; 77 | va_start(args, fmt); 78 | vasprintf(&string, fmt, args); 79 | va_end(args); 80 | FILE * f = fopen(import_logger_path, "a"); 81 | if (f) 82 | { 83 | fprintf(f, "%s", string); 84 | if (!strrchr(string, '\n')) 85 | fprintf(f, "\n"); 86 | fclose(f); 87 | } 88 | } 89 | //print on stderr 90 | if (level > LOG_BASIC && string && !quiet) 91 | { 92 | if (!first_cls && !quiet) 93 | { 94 | system("clear"); 95 | first_cls = true; 96 | } 97 | pthread_t t = pthread_self(); 98 | int i = 0; 99 | bool found = false; 100 | if (threads_list) 101 | { 102 | for (; i < threads_number; i++) 103 | { 104 | if (t == threads_list[i]) 105 | { 106 | found = true; 107 | break; 108 | } 109 | } 110 | } 111 | if (!found) 112 | i = 0; 113 | 114 | if (threads_number > 1) 115 | { 116 | if (i+logger_offset+threads_number/2 > logger_window.ws_row) 117 | { 118 | logger_offset = 0; 119 | system("clear"); 120 | } 121 | gotoxy(0, i + 1 + logger_offset); 122 | fprintf(stderr, "\33[2K\r"); 123 | gotoxy(1, i + 1 + logger_offset); 124 | fprintf(stderr, "Thread %d: ", i); 125 | } 126 | fprintf(stderr, "%s\r", string); 127 | } 128 | pthread_mutex_unlock(&logger_lock); 129 | } 130 | 131 | void logger_dbname_set(char * db) 132 | { 133 | if (*import_logger_path) 134 | return; 135 | 136 | ldb_prepare_dir(LOGGER_DIR); 137 | sprintf(import_logger_path, "%s/%s.log", LOGGER_DIR, db); 138 | time_t currentTime = time(NULL); 139 | struct tm *localTime = localtime(¤tTime); 140 | char timeString[64]; 141 | strftime(timeString, sizeof(timeString), "Exec Time: %Y-%m-%d %H:%M:%S", localTime); 142 | FILE * f = fopen(import_logger_path, "a"); 143 | if (f) 144 | { 145 | fprintf(f, "%s\n", timeString); 146 | fclose(f); 147 | } 148 | } 149 | 150 | void logger_init(char * db, int tnumber, pthread_t * tlist) 151 | { 152 | pthread_mutex_init(&logger_lock, NULL); 153 | 154 | if (tlist) 155 | { 156 | threads_list = tlist; 157 | threads_number = tnumber; 158 | } 159 | 160 | int stdout_fd = fileno(stderr); 161 | ioctl(stdout_fd, TIOCGWINSZ, &logger_window); 162 | 163 | logger_dbname_set(db); 164 | } 165 | 166 | void logger_offset_increase(int off) 167 | { 168 | if (level > LOG_BASIC) 169 | logger_offset += off; 170 | fflush(stderr); 171 | } 172 | 173 | void logger_set_level(log_level_t l) 174 | { 175 | level = l; 176 | } 177 | 178 | void log_debug(const char * fmt, ...) 179 | { 180 | if (level > LOG_INFO) 181 | { 182 | va_list args; 183 | va_start(args, fmt); 184 | char * string; 185 | vasprintf(&string, fmt, args); 186 | log_info("%s", string); 187 | free(string); 188 | va_end(args); 189 | } 190 | } 191 | 192 | void log_set_quiet(bool mode) 193 | { 194 | quiet = mode; 195 | } 196 | 197 | 198 | -------------------------------------------------------------------------------- /src/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef __LOGGER_H 2 | #define __LOGGER_H 3 | 4 | #include 5 | 6 | typedef enum { 7 | LOG_BASIC = 0, 8 | LOG_INFO, 9 | LOG_DEBUG 10 | } log_level_t; 11 | 12 | void logger_init(char * db, int tnumber, pthread_t * tlist); 13 | void log_set_quiet(bool mode); 14 | void logger_offset_increase(int off); 15 | void import_logger(const char * fmt, ...); 16 | void log_info(const char * fmt, ...); 17 | void logger_set_level(log_level_t l); 18 | void log_debug(const char * fmt, ...); 19 | void logger_dbname_set(char * db); 20 | void logger_basic(const char * fmt, ...); 21 | #define LOG_INF(fmt,args...) import_logger(NULL, fmt, args) 22 | #endif -------------------------------------------------------------------------------- /src/md5.c: -------------------------------------------------------------------------------- 1 | #include "ldb.h" 2 | #include 3 | 4 | /* Adapter function for compatibility with openssl*/ 5 | void md5_string(const unsigned char *input, int len, unsigned char output[16]) 6 | { 7 | gcry_md_hd_t h; 8 | gcry_md_open(&h, GCRY_MD_MD5, GCRY_MD_FLAG_SECURE); // initialize the hash context 9 | gcry_md_write(h, input, len); // hash the input data 10 | const unsigned char *hash_result = gcry_md_read(h, GCRY_MD_MD5); // get the result 11 | 12 | // Copy the hash result to the output array 13 | memcpy(output, hash_result, 16); 14 | 15 | // Free memory for the hash result 16 | gcry_md_close(h); 17 | } 18 | 19 | /** 20 | * @brief Returns the hexadecimal md5 sum for "path" 21 | * 22 | * @param path string path 23 | * @return pointer to file md5 array 24 | */ 25 | uint8_t * md5_file(char *path) 26 | { 27 | uint8_t *c = calloc(1, gcry_md_get_algo_dlen(GCRY_MD_MD5)); // Allocate memory for MD5 hash 28 | FILE *fp = fopen(path, "rb"); 29 | 30 | if (!fp) { 31 | fprintf(stderr, "Unable to open file for reading.\n"); 32 | return NULL; 33 | } 34 | 35 | gcry_md_hd_t md5_hd; 36 | gcry_md_open(&md5_hd, GCRY_MD_MD5, GCRY_MD_FLAG_SECURE); 37 | 38 | uint8_t *buffer = malloc(BUFFER_SIZE); 39 | size_t bytes; 40 | 41 | while ((bytes = fread(buffer, 1, BUFFER_SIZE, fp)) > 0) { 42 | gcry_md_write(md5_hd, buffer, bytes); 43 | } 44 | 45 | fclose(fp); 46 | free(buffer); 47 | 48 | const uint8_t *digest = gcry_md_read(md5_hd, GCRY_MD_MD5); 49 | memcpy(c, digest, gcry_md_get_algo_dlen(GCRY_MD_MD5)); 50 | 51 | gcry_md_close(md5_hd); 52 | return c; 53 | } -------------------------------------------------------------------------------- /src/mz_optimise.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/mz_optimise.c 4 | * 5 | * SCANOSS .mz optimisation functions 6 | * 7 | * Copyright (C) 2018-2021 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | 23 | /** 24 | * @file mz_optimize.c 25 | * @date 26 Oct 2021 26 | * @brief Implement MZ optimization functions 27 | */ 28 | 29 | #include 30 | #include 31 | #include "ldb.h" 32 | #include 33 | #include "mz.h" 34 | #include "logger.h" 35 | #include "collate.h" 36 | #define MAX_FILE_SIZE (4 * 1048576) 37 | 38 | /** 39 | * @brief Check if job->id is found in job->xkeys (see -X) 40 | * 41 | * @param job pointer to mz job 42 | * @return true if it was found 43 | */ 44 | bool mz_id_excluded(struct mz_job *job) 45 | { 46 | if (!job->xkeys_ln) return false; 47 | 48 | for (uint64_t i = 0; i < job->xkeys_ln; i += MD5_LEN) 49 | { 50 | /* Compare job id (bytes 3-16) */ 51 | if (!memcmp(job->xkeys + i + 2, job->id, MD5_LEN - 2)) 52 | { 53 | /* Compare mz id (bytes 1-2) */ 54 | if (!memcmp(job->xkeys + i, job->mz_id, 2)) return true; 55 | } 56 | } 57 | 58 | return false; 59 | } 60 | 61 | bool mz_optimise_dup_handler(struct mz_job *job) 62 | { 63 | /* Check if file is not duplicated */ 64 | if (mz_id_excluded(job)) 65 | { 66 | job->exc_c++; 67 | } 68 | else if (mz_id_exists(job->ptr, job->ptr_ln, job->id)) 69 | { 70 | job->dup_c++; 71 | } 72 | else if ((long) job->mz_ln - (long) job->ptr_ln - (long) job->ln < 0) 73 | { 74 | mz_id_fill(job->md5, job->id); 75 | log_info("Incorrect size of source file %s on pos %u from sector: %s\n", job->md5, job->ptr_ln, job->path); 76 | } 77 | else 78 | { 79 | memcpy(job->ptr + job->ptr_ln, job->id, job->ln); 80 | job->ptr_ln += job->ln; 81 | } 82 | 83 | return true; 84 | } 85 | 86 | 87 | /** 88 | * @brief Optimise an mz file removing duplicated data 89 | * 90 | * @param job pointer to mz job 91 | */ 92 | void mz_collate(struct mz_job *job) 93 | { 94 | /* Extract first two MD5 bytes from the file name */ 95 | memcpy(job->md5, basename(job->path), 4); 96 | ldb_hex_to_bin(job->md5, 4, job->mz_id); 97 | 98 | /* Read source mz file into memory */ 99 | job->mz = file_read(job->path, &job->mz_ln); 100 | 101 | if (!job->mz) 102 | return; 103 | 104 | /* Reserve memory for destination mz */ 105 | job->ptr = calloc(job->mz_ln, 1); 106 | if (!job->ptr) 107 | { 108 | free(job->mz); 109 | return; 110 | } 111 | 112 | /* Launch optimisation */ 113 | 114 | mz_parse(job, mz_optimise_dup_handler); 115 | 116 | /* Write updated mz file */ 117 | file_write(job->path, job->ptr, job->ptr_ln); 118 | 119 | if (job->dup_c) 120 | log_info("%s: %u duplicated files eliminated\n", job->path, job->dup_c); 121 | 122 | if (job->exc_c) 123 | log_info("%s: %u keys excluded\n", job->path, job->exc_c); 124 | 125 | free(job->mz); 126 | free(job->ptr); 127 | } 128 | 129 | void ldb_mz_collate(struct ldb_table table, int p_sector) 130 | { 131 | /* Start with sector 0*/ 132 | uint16_t k0 = 0; 133 | logger_dbname_set(table.db); 134 | 135 | if (p_sector >= 0) 136 | { 137 | k0 = p_sector; 138 | } 139 | 140 | do { 141 | char sector_path[LDB_MAX_PATH] = "\0"; 142 | sprintf(sector_path, "%s/%s/%s/%.4x.mz", ldb_root, table.db, table.table, k0); 143 | bool file_exist = ldb_file_exists(sector_path); 144 | if (!file_exist) //check for encoded files 145 | { 146 | strcat(sector_path, ".enc"); 147 | file_exist = ldb_file_exists(sector_path); 148 | } 149 | if (file_exist) 150 | { 151 | log_info("Processing %s (remove duplicates)\n", sector_path); 152 | 153 | struct mz_job job; 154 | *job.path = 0; 155 | memset(job.mz_id, 0, 2); 156 | job.mz = NULL; 157 | job.mz_ln = 0; 158 | job.id = NULL; 159 | job.ln = 0; 160 | job.data = NULL; // Uncompressed data 161 | job.data_ln = 0; 162 | job.zdata = NULL; // Compressed data 163 | job.zdata_ln = 0; 164 | job.ptr = NULL; // Temporary data 165 | job.ptr_ln = 0; 166 | job.dup_c = 0; 167 | job.igl_c = 0; 168 | job.orp_c = 0; 169 | job.min_c = 0; 170 | job.md5[32] = 0; 171 | job.check_only = false; 172 | job.dump_keys = false; 173 | job.orphan_rm = false; 174 | job.key = NULL; 175 | job.xkeys = NULL; 176 | job.xkeys_ln = 0; 177 | job.licenses = NULL; 178 | job.license_count = 0; 179 | job.exc_c = 0; 180 | job.dup_c = 0; 181 | strcpy(job.path, sector_path); 182 | mz_collate(&job); 183 | free(job.xkeys); 184 | } 185 | 186 | if (p_sector >=0) 187 | break; 188 | 189 | 190 | } while (k0++ < 0xffff); 191 | 192 | } 193 | 194 | void ldb_mz_collate_delete(struct ldb_table table, job_delete_tuples_t * delete) 195 | { 196 | 197 | /* Otherwise use the first byte of the first key */ 198 | if (!delete) 199 | return; 200 | 201 | long total_records = 0; 202 | setlocale(LC_NUMERIC, ""); 203 | 204 | logger_dbname_set(table.db); 205 | /* Read each DB sector */ 206 | for (int i = 0; i < delete->tuples_number; i++) 207 | { 208 | /* Start with sector 0, unless it is a delete command */ 209 | uint16_t k = 0; 210 | uint8_t * k0 = (uint8_t*) &k; 211 | k0[1] = delete->tuples[i]->key[0]; 212 | k0[0] = delete->tuples[i]->key[1]; 213 | log_info("Removing keys from Table %s - Reading sector %04x\n", table.table, k); 214 | char sector_path[LDB_MAX_PATH] = "\0"; 215 | sprintf(sector_path, "%s/%s/%s/%.4x.mz", ldb_root, table.db, table.table, k); 216 | bool file_exist = ldb_file_exists(sector_path); 217 | if (!file_exist) //check for encoded files 218 | { 219 | strcat(sector_path, ".enc"); 220 | file_exist = ldb_file_exists(sector_path); 221 | log_info("Sector does not exist %s", sector_path); 222 | } 223 | if (file_exist) 224 | { 225 | struct mz_job job; 226 | *job.path = 0; 227 | memset(job.mz_id, 0, 2); 228 | job.mz = NULL; 229 | job.mz_ln = 0; 230 | job.id = NULL; 231 | job.ln = 0; 232 | job.data = NULL; // Uncompressed data 233 | job.data_ln = 0; 234 | job.zdata = NULL; // Compressed data 235 | job.zdata_ln = 0; 236 | job.ptr = NULL; // Temporary data 237 | job.ptr_ln = 0; 238 | job.dup_c = 0; 239 | job.igl_c = 0; 240 | job.orp_c = 0; 241 | job.min_c = 0; 242 | job.md5[32] = 0; 243 | job.check_only = false; 244 | job.dump_keys = false; 245 | job.orphan_rm = false; 246 | job.key = NULL; 247 | job.xkeys = NULL; 248 | job.xkeys_ln = 0; 249 | job.licenses = NULL; 250 | job.license_count = 0; 251 | job.exc_c = 0; 252 | job.dup_c = 0; 253 | strcpy(job.path, sector_path); 254 | 255 | job.xkeys = calloc(delete->tuples_number, MD5_LEN); 256 | job.xkeys_ln = delete->tuples_number * MD5_LEN; 257 | for (; i < delete->tuples_number; i++) 258 | { 259 | memcpy(job.xkeys + i * MD5_LEN, delete->tuples[i]->key, MD5_LEN); 260 | if (i + 1 < delete->tuples_number) 261 | if (memcmp(delete->tuples[i + 1]->key, delete->tuples[i]->key, 2)) 262 | break; 263 | } 264 | 265 | mz_collate(&job); 266 | free(job.xkeys); 267 | } 268 | /* Exit here if it is a delete command, otherwise move to the next sector */ 269 | } 270 | 271 | /* Show processed totals */ 272 | log_info("Table %s: cleanup completed with %'ld records\n", table.table, total_records); 273 | fflush(stdout); 274 | } -------------------------------------------------------------------------------- /src/node.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/node.c 4 | * 5 | * LDB node handling routines 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "ldb.h" 23 | #include "logger.h" 24 | #include "ldb_error.h" 25 | /** 26 | * @file node.c 27 | * @date 12 Jul 2020 28 | * @brief Contains functions to handle LDB nodes 29 | 30 | * NODE STRUCTURE 31 | * Every data list starts with a pointer to the last node in the list, followed by the first node: 32 | 33 | * List header: 34 | * LN = is the 40-bit pointer to the last node 35 | 36 | * Node header: 37 | * Each node starts with a pointer to the next node, followed by the node size, followed by the node data 38 | * NN = is the 40-bit pointer to the next node 39 | * TS = is the 16-bit (or 32-bit) total size of the node data (in bytes, if variable-sized records, in number of records if fixed-sized records) 40 | * K = is the remaining part of the key for the group of records that follow (in case the key is bigger than 32-bit, otherwise K is omitted) 41 | * GS = is the total size of the group records that follow (those sharing K, omitted when key is 32-bit) 42 | 43 | * Node data is a series of records (size+data): 44 | * R: Data record 45 | * s = is a 16-bit record size (omitted when record size is fixed) 46 | * d = is the data record 47 | * @see https://github.com/scanoss/ldb/blob/master/src/node.c 48 | */ 49 | 50 | /** 51 | * @brief Gets a next node addr from the header provided and loads it into the rs list. 52 | * 53 | * @param rs Struct that wraps the list 54 | * @param header Buffer with information about: addr of a node and lenght of itself 55 | */ 56 | void ldb_load_node_header(struct ldb_recordset *rs, uint8_t *header) 57 | { 58 | 59 | /* Load next node and node length */ 60 | rs->next_node = uint40_read(header); 61 | rs->node_ln = uint16_read(header + 5); 62 | 63 | /* When records are fixed in length, node size is expressed in number of records */ 64 | if (rs->rec_ln) rs->node_ln = rs->rec_ln * rs->node_ln; 65 | } 66 | 67 | /** 68 | * @brief Reads and loads the node that the list is pointing to. 69 | * Free the node if it is already loaded and then loads the new node. 70 | * 71 | * @param rs Struct that wraps the list 72 | */ 73 | void ldb_load_node(struct ldb_recordset *rs) 74 | { 75 | if (rs->node) free(rs->node); 76 | 77 | /* Read node */ 78 | rs->node = malloc(rs->node_ln + 1); 79 | if (!fread(rs->node, 1, rs->node_ln, rs->sector)) log_info("Warning: cannot load node\n"); 80 | 81 | /* Terminate with a chr(0) */ 82 | rs->node[rs->node_ln] = 0; 83 | 84 | } 85 | /** 86 | * @brief Writes data into a node and updates pointers 87 | * 88 | * @param table The table the data is going to be written into 89 | * @param ldb_sector The sector of the LDB where the data is going to be written. The file descriptor must be opened before. 90 | * @param key table key 91 | * @param data Buffer with the data to be written 92 | * @param dataln Length of the data to be written 93 | * @param records The number of records 94 | */ 95 | int ldb_node_write (struct ldb_table table, FILE *ldb_sector, uint8_t *key, uint8_t *data, uint32_t dataln, uint16_t records) 96 | { 97 | uint8_t subkey_ln = table.key_ln - LDB_KEY_LN; 98 | 99 | /* Check that record length is within bounds */ 100 | if (dataln > LDB_MAX_NODE_LN) ldb_error ("E053 Data record size exceeded"); 101 | 102 | if ((!records) && (dataln + LDB_PTR_LN + LDB_PTR_LN + table.ts_ln >= LDB_MAX_NODE_LN)) 103 | { 104 | log_info("E053 Data record size exceeded"); 105 | return LDB_ERROR_DATA_RECORD_SIZE_EXCEED; 106 | //ldb_error ("E053 Data record size exceeded"); 107 | } 108 | 109 | /* Obtain the pointer to the last node of the list */ 110 | uint64_t list = ldb_list_pointer(ldb_sector, key); 111 | 112 | if ((list > 0 && list < LDB_MAP_SIZE) || ldb_read_failure) { 113 | log_info("\nFatal data corruption on list %lu for key %02x%02x%02x%02x from table %s/%s\n", list, key[0], key[1], key[2], key[3], table.db, table.table); 114 | log_info("E057 Map location %08lx\n", ldb_map_pointer_pos(key)); 115 | ldb_read_failure = false; 116 | return LDB_ERROR_MAP_CORRUPTED; 117 | } 118 | 119 | /* Seek end of file, and save the pointer to the new node */ 120 | fseeko64(ldb_sector, 0, SEEK_END); 121 | uint64_t new_node = ftello64(ldb_sector); 122 | 123 | if (new_node < LDB_MAP_SIZE) { 124 | log_info("E056 Data sector %02x corrupted, with %lu below map_size\n", *key, new_node); 125 | return LDB_ERROR_DATA_SECTOR_CORRUPTED;//exit(EXIT_FAILURE); 126 | } 127 | 128 | /* Allocate memory for new node, plus LN(5), NN(5) and TS(4 max)*/ 129 | uint8_t *node = malloc(LDB_MAX_NODE_LN + LDB_PTR_LN + LDB_PTR_LN + table.ts_ln); 130 | uint64_t node_ptr = 0; 131 | 132 | /* LN: A new list starts with a pointer to the last node (which is itself after LN(5)) */ 133 | if (list == 0) 134 | { 135 | uint40_write(node, new_node + LDB_PTR_LN); 136 | node_ptr = LDB_PTR_LN; 137 | } 138 | 139 | /* NN: Node starts with a zeroed "next" pointer, since it will be the last in the list */ 140 | uint40_write(node + node_ptr, 0); 141 | node_ptr += LDB_PTR_LN; 142 | 143 | /* TS: Write the node length (either number of records (fixed-recln) or number of bytes (variable_recln)) */ 144 | if (table.ts_ln == 2) 145 | { 146 | if (records) uint16_write(node + node_ptr, records); 147 | else uint16_write(node + node_ptr, dataln + subkey_ln); 148 | node_ptr += 2; 149 | } 150 | else if (table.ts_ln == 4) 151 | { 152 | if (records) uint32_write(node + node_ptr, records); 153 | else uint32_write(node + node_ptr, dataln + subkey_ln); 154 | node_ptr += 4; 155 | } 156 | else 157 | { 158 | log_info("E060 Unsupported node_length size (must be 2 or 4 bytes)\n"); 159 | return LDB_ERROR_NODE_SIZE_INVALID; 160 | } 161 | 162 | /* K: Write the key after the 4th byte (if needed) */ 163 | if (table.key_ln > LDB_KEY_LN) 164 | { 165 | memcpy(node + node_ptr, key + LDB_KEY_LN, table.key_ln - LDB_KEY_LN); 166 | node_ptr += (table.key_ln - LDB_KEY_LN); 167 | } 168 | 169 | /* R: Write the data */ 170 | memcpy(node + node_ptr, data, dataln); 171 | node_ptr += dataln; 172 | 173 | /* Write actual node */ 174 | if (node_ptr != fwrite(node, 1, node_ptr, ldb_sector)) 175 | { 176 | //ldb_error("E058 Error writing node"); 177 | log_info("E058 Error writing node\n"); 178 | return LDB_ERROR_NODE_WRITE_FAILS; 179 | } 180 | 181 | /* Update list pointers */ 182 | ldb_update_list_pointers(ldb_sector, key, list, new_node); 183 | 184 | free(node); 185 | return LDB_ERROR_NOERROR; 186 | } 187 | 188 | /** 189 | * @brief Reads a node from the given location (ptr) for a 32-bit key. If ptr is set to zero, the location is 190 | * obtained from the sector map. The function returns a pointer to the next node, which is zero if it 191 | * is the last node in the list. 192 | * 193 | * @param sector Optional: Pointer to a LDB sector allocated in memory. If NULL the function will use the table struct and key to open the ldb 194 | * @param table table struct config 195 | * @param ldb_sector A file descriptor to the LDB sector. If sector is not NULL, this parameter is ignored. 196 | * @param ptr If ptr is set to zero, the location is obtained from the sector map 197 | * @param key key of the associated table 198 | * @param bytes_read Number of bytes readed from the node (output) 199 | * @param out Buffer with the data readed from the node 200 | * @param max_node_size Indicates the maximum size of the node. If the node is bigger than this value, the function will return an error. 201 | * @return uint64_t The addr of the next node 202 | * 203 | */ 204 | uint64_t ldb_node_read(uint8_t *sector, struct ldb_table table, FILE *ldb_sector, uint64_t ptr, uint8_t *key, uint32_t *bytes_read, uint8_t **out, int max_node_size) 205 | { 206 | *bytes_read = 0; 207 | 208 | /* If pointer is zero, get the list location from the map */ 209 | if (ptr == 0) 210 | { 211 | /* Read sector pointer either from disk (ldb_sector) or memory (sector) */ 212 | if (sector) 213 | ptr = uint40_read(sector + ldb_map_pointer_pos(key)); 214 | else 215 | ptr = ldb_list_pointer(ldb_sector, key); 216 | 217 | /* If pointer is zero, then there are no records for the key */ 218 | if (ptr == 0) 219 | { 220 | return 0; 221 | } 222 | 223 | /* If there is a list, we skip the first bytes (LN: last node pointer) to move into the first node */ 224 | ptr += LDB_PTR_LN; 225 | } 226 | 227 | uint8_t *buffer; 228 | 229 | /* Read node information into buffer: NN(5) and TS(2/4) */ 230 | if (sector) 231 | buffer = sector + ptr; 232 | else 233 | { 234 | fseeko64(ldb_sector, ptr, SEEK_SET); 235 | buffer = calloc(LDB_PTR_LN + table.ts_ln + LDB_KEY_LN, 1); 236 | if (!fread(buffer, 1, LDB_PTR_LN + table.ts_ln, ldb_sector)) 237 | { 238 | log_debug("Warning: cannot read LDB node\n"); 239 | ldb_read_failure = true; 240 | } 241 | } 242 | 243 | /* NN: Obtain the next node */ 244 | uint64_t next_node = uint40_read(buffer); 245 | 246 | /* TS: Obtain the size of the node */ 247 | uint32_t node_size = 0; 248 | if (table.ts_ln == 2) node_size = uint16_read(buffer + LDB_PTR_LN); 249 | else node_size = uint32_read(buffer + LDB_PTR_LN); 250 | 251 | uint32_t actual_size = node_size; 252 | 253 | /* When records are fixed in length, node size is expressed in number of records */ 254 | if (table.rec_ln) actual_size = node_size * table.rec_ln; 255 | 256 | /* If the node size exceeds the wanted limit, then ignore it entirely */ 257 | if (max_node_size) if (actual_size > max_node_size) actual_size = 0; 258 | 259 | /* A deleted node will have a size set to zero. */ 260 | if (actual_size) 261 | { 262 | 263 | if (table.rec_ln) if (actual_size > 64800) actual_size = 64800; //TODO: EXPAND? 264 | 265 | /* Return the entire node */ 266 | if (sector) 267 | { 268 | *out = buffer + LDB_PTR_LN + table.ts_ln; 269 | } 270 | else 271 | { 272 | if (!fread(*out, 1, actual_size, ldb_sector)) 273 | { 274 | log_debug("Warning: cannot read entire LDB node\n"); 275 | ldb_read_failure = true; 276 | } 277 | } 278 | *bytes_read = actual_size; 279 | 280 | /* Gracefully terminate non-fixed records (strings) with a chr(0) */ 281 | if (!sector) if (table.rec_ln == 0) *(*out+actual_size) = 0; 282 | } 283 | 284 | if (!sector) free(buffer); 285 | return next_node; 286 | } 287 | 288 | /** 289 | * @brief Unlinks a first node found for the given table and key 290 | * 291 | * @param table Configuration of a table 292 | * @param key The key of the table 293 | */ 294 | void ldb_node_unlink(struct ldb_table table, uint8_t *key) 295 | { 296 | 297 | uint16_t subkeyln = table.key_ln - LDB_KEY_LN; 298 | 299 | /* Open sector */ 300 | FILE *ldb_sector = ldb_open(table, key, "r+"); 301 | 302 | if (ldb_sector) 303 | { 304 | 305 | /* For a 32-bit key, we simply wipe out the map pointer, killing the entire list */ 306 | if (table.key_ln == LDB_KEY_LN) 307 | { 308 | /* Move pointer to the map pointer */ 309 | fseeko64(ldb_sector, ldb_map_pointer_pos(key), SEEK_SET); 310 | 311 | /* Set pointer to zero */ 312 | ldb_uint40_write(ldb_sector, 0); 313 | } 314 | 315 | /* A key greater than 32-bit will require reading the entire list and searching every node for matching subkeys */ 316 | else 317 | { 318 | 319 | /* If pointer is zero, then there are no records for the key */ 320 | uint64_t next = ldb_list_pointer(ldb_sector, key); 321 | 322 | if (next) 323 | { 324 | /* Skip the first bytes (LN: last node pointer) to move into the first node */ 325 | next += LDB_PTR_LN; 326 | 327 | do 328 | { 329 | uint64_t last = next; 330 | 331 | /* Move the file pointer */ 332 | fseeko64(ldb_sector, next, SEEK_SET); 333 | 334 | /* Read node information into buffer: NN(5) and TS(2/4) */ 335 | uint8_t *buffer = malloc(LDB_PTR_LN + table.ts_ln + table.key_ln); 336 | if (!fread(buffer, 1, LDB_PTR_LN + table.ts_ln, ldb_sector)) 337 | { 338 | log_info("Warning: cannot read LDB node info\n"); 339 | break; 340 | } 341 | 342 | /* NN: Obtain the next node */ 343 | next = uint40_read(buffer); 344 | 345 | /* TS: Obtain the size of the node */ 346 | uint16_t node_size = uint16_read(buffer + LDB_PTR_LN); 347 | 348 | /* When records are fixed in length, node size is expressed in number of records */ 349 | if (table.rec_ln) node_size = node_size * table.rec_ln; 350 | 351 | /* A deleted node will have a size set to zero. */ 352 | 353 | if (node_size != 0) 354 | { 355 | 356 | uint64_t node_ptr = 0; 357 | 358 | do 359 | { 360 | /* K: Compare the remaining part of the key */ 361 | bool key_ok = true; 362 | uint16_t gs = node_size; 363 | uint64_t last_key = node_ptr; 364 | uint32_t get_bytes = subkeyln + (table.rec_ln ? 0 : 2); 365 | 366 | /* Read K and GS (2) if needed */ 367 | if (!fread(buffer, 1, get_bytes, ldb_sector)) 368 | { 369 | log_info("Warning: cannot read LDB node info (K/GS)\n"); 370 | break; 371 | } 372 | 373 | if (memcmp(buffer, key + LDB_KEY_LN, subkeyln) != 0) key_ok = false; 374 | 375 | if (key_ok) 376 | { 377 | if (!table.rec_ln) 378 | { 379 | /* Read and write GS if needed */ 380 | gs = uint16_read(buffer + subkeyln); 381 | node_ptr += 2; 382 | } 383 | 384 | /* Move pointer back to the subkey and wipe it */ 385 | fseeko64(ldb_sector, last + last_key + LDB_PTR_LN + table.ts_ln, SEEK_SET); 386 | uint8_t *empty_key = calloc(subkeyln , 1); 387 | fwrite(empty_key, 1, subkeyln, ldb_sector); 388 | free(empty_key); 389 | 390 | /* We leave after deleting */ 391 | node_ptr = node_size; 392 | 393 | } 394 | else fseeko64(ldb_sector, last + LDB_PTR_LN + table.ts_ln + get_bytes + gs + (table.rec_ln ? 0 : 2), SEEK_SET); 395 | node_ptr += gs; 396 | 397 | } while (node_ptr < node_size); 398 | } 399 | 400 | free(buffer); 401 | } while (next); 402 | } 403 | } 404 | } 405 | 406 | if (ldb_sector) 407 | ldb_close_unlock(ldb_sector); 408 | } 409 | 410 | /** 411 | * @brief Validates a node checking for the dataset size. 412 | * 413 | * @param node Buffer containing the node data 414 | * @param node_size Size of the node 415 | * @param subkey_ln block subkey lenght 416 | * @return true if the node is valid. False otherwise. 417 | */ 418 | bool ldb_validate_node(uint8_t *node, uint32_t node_size, int subkey_ln) 419 | { 420 | 421 | /* Make sure we have enough bytes in the node */ 422 | if (node_size < (subkey_ln + 2)) return false; 423 | 424 | /* Extract datasets from node */ 425 | uint32_t node_ptr = 0; 426 | while (node_ptr < node_size) 427 | { 428 | 429 | /* Skip subkey */ 430 | node_ptr += subkey_ln; 431 | 432 | /* Get dataset size */ 433 | int dataset_size = uint16_read(node + node_ptr); 434 | node_ptr += 2; 435 | 436 | /* Is the reported dataset_size greater than the remaining node? Then fail */ 437 | if (node_ptr + dataset_size > node_size) return false; 438 | 439 | /* Extract records from dataset */ 440 | if (subkey_ln) 441 | { 442 | uint32_t dataset_ptr = 0; 443 | while (dataset_ptr < dataset_size) 444 | { 445 | 446 | /* Get record size */ 447 | int record_size = uint16_read(node + node_ptr + dataset_ptr); 448 | dataset_ptr += 2; 449 | 450 | /* Is the reported record_size greater than the remaining dataset? Then fail */ 451 | if (node_ptr + dataset_ptr + record_size > node_size) return false; 452 | 453 | /* Move pointer to end of record */ 454 | dataset_ptr += record_size; 455 | 456 | /* If we passed the dataset_size, the node is bad */ 457 | if (dataset_ptr > dataset_size) return false; 458 | } 459 | } 460 | /* Move pointer to end of dataset */ 461 | node_ptr += dataset_size; 462 | } 463 | return true; 464 | } 465 | 466 | -------------------------------------------------------------------------------- /src/pointer.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/pointer.c 4 | * 5 | * Pointer handling functions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "ldb.h" 23 | /** 24 | * @file pointer.c 25 | * @date 12 Jul 2020 26 | * @brief List helpers function 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/pointer.c 30 | */ 31 | 32 | /** 33 | * @brief Returns the map position for the given record ID, using only the last 3 bytes 34 | * of the key, since the sector name contains the first byte 35 | * 36 | * @param key The key to get the map position for 37 | * @return uint64_t The map position 38 | */ 39 | uint64_t ldb_map_pointer_pos(uint8_t *key) 40 | { 41 | 42 | uint64_t out = 0; 43 | 44 | /* Obtain a nuclear "char" pointer to the uint32 */ 45 | uint8_t *k = (uint8_t *) &out; 46 | 47 | /* Assign less significant bytes (inverting for easy debugging, so that 00 00 01 is the second position in the map)*/ 48 | k[0]=key[3]; 49 | k[1]=key[2]; 50 | k[2]=key[1]; 51 | 52 | return out * LDB_PTR_LN; 53 | } 54 | 55 | /** 56 | * @brief Return pointer to the beginning of the given list (The last node) 57 | * 58 | * @param ldb_sector Sector of ldb 59 | * @param key Key of the ldb 60 | * @return uint64_t Obtain the pointer to the last node of the list 61 | */ 62 | uint64_t ldb_list_pointer(FILE *ldb_sector, uint8_t *key) 63 | { 64 | fseeko64(ldb_sector, ldb_map_pointer_pos(key), SEEK_SET); 65 | return ldb_uint40_read(ldb_sector); 66 | } 67 | 68 | /** 69 | * @brief Return pointer to the last node of the list 70 | * 71 | * @param ldb_sector Stream of the opened ldb 72 | * @param list_pointer Pointer to the list 73 | * @return uint64_t The 40 bits with the ldb sector 74 | */ 75 | uint64_t ldb_last_node_pointer(FILE *ldb_sector, uint64_t list_pointer) 76 | { 77 | if (list_pointer == 0) return 0; 78 | fseeko64(ldb_sector, list_pointer, SEEK_SET); 79 | return ldb_uint40_read(ldb_sector); 80 | } 81 | 82 | /** 83 | * @brief Update list pointers 84 | * 85 | * @param ldb_sector Stream of the opened ldb 86 | * @param key Associated key 87 | * @param list List pointer where the new node will be added 88 | * @param new_node The new node to add 89 | */ 90 | void ldb_update_list_pointers(FILE *ldb_sector, uint8_t *key, uint64_t list, uint64_t new_node) 91 | { 92 | /* If this is the first node of the list, we update the map and leave */ 93 | if (list == 0) 94 | { 95 | fseeko64(ldb_sector, ldb_map_pointer_pos(key), SEEK_SET); 96 | ldb_uint40_write(ldb_sector, new_node); 97 | if (new_node < LDB_MAP_SIZE) ldb_error("E054 Data corruption"); 98 | } 99 | 100 | /* Otherwise we update the list */ 101 | else 102 | { 103 | 104 | /* Get the current last node pointer */ 105 | fseeko64(ldb_sector, list, SEEK_SET); 106 | uint64_t last_node = ldb_uint40_read(ldb_sector); 107 | 108 | if (last_node < LDB_MAP_SIZE) { 109 | printf("\nMap size is %u\n", LDB_MAP_SIZE); 110 | printf ("\nData corruption on list %lu for key %02x%02x%02x%02x with last node %lu < %u\n", list, key[0], key[1], key[2], key[3], last_node, LDB_MAP_SIZE); 111 | ldb_error("E055 Data corruption"); 112 | } 113 | 114 | /* Update the list pointer to the new last node */ 115 | fseeko64(ldb_sector, list, SEEK_SET); 116 | ldb_uint40_write(ldb_sector, new_node); 117 | 118 | 119 | /* Update the last node pointer to next (new) node */ 120 | fseeko64(ldb_sector, last_node, SEEK_SET); 121 | ldb_uint40_write(ldb_sector, new_node); 122 | 123 | } 124 | } 125 | 126 | /** 127 | * @brief // TODO 128 | * 129 | * @param ldb_sector // TODO 130 | * @param key // TODO 131 | */ 132 | void ldb_list_unlink(FILE *ldb_sector, uint8_t *key) 133 | { 134 | fseeko64(ldb_sector, ldb_map_pointer_pos(key), SEEK_SET); 135 | ldb_uint40_write(ldb_sector, 0); 136 | } 137 | 138 | -------------------------------------------------------------------------------- /src/recordset.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/recordset.c 4 | * 5 | * LDB recordset reading functions 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "ldb.h" 23 | /** 24 | * @file recordset.c 25 | * @date 12 Jul 2020 26 | * @brief Contains functions to read records from a LDB recordset 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/recordset.c 30 | */ 31 | #include "ldb.h" 32 | #include "logger.h" 33 | 34 | #ifndef LDB_EXTRACT_DEFINED_EXTERN 35 | int ldb_extract_record(char **msg, const uint8_t *data, uint32_t dataset_ptr, int record_size, struct ldb_table table, const uint8_t *key, const uint8_t *subkey) 36 | { 37 | *msg = strdup((char *) data + dataset_ptr); 38 | msg[record_size-1] = 0; 39 | return strlen(*msg); 40 | } 41 | #else 42 | extern int ldb_extract_record(char **msg, const uint8_t *data, uint32_t dataset_ptr, int record_size, struct ldb_table table, const uint8_t *key, const uint8_t *subkey); 43 | 44 | #endif 45 | 46 | /** 47 | * @brief Recurses all records in *table* for *key* and calls the provided handler funcion in each iteration, passing 48 | * subkey, subkey length, fetched data, length and iteration number. This function acts on the .ldb for the 49 | * provided *key*, but can also work from memory, if a pointer to a *sector* is provided (not NULL) 50 | * 51 | * @param sector Optional: Pointer to a LDB sector allocated in memory. If NULL the function will use tha table struct and key to open the ldb 52 | * @param table table struct config 53 | * @param key key of the associated table 54 | * @param skip_subkey true for skip the subkey 55 | * @param ldb_record_handler Handler to print the data 56 | * @param void_ptr This pointer is passed to the handler function 57 | * @return uint32_t The number of records found 58 | */ 59 | uint32_t ldb_fetch_recordset(uint8_t *sector, struct ldb_table table, uint8_t* key, bool skip_subkey, bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *), void *void_ptr) 60 | { 61 | FILE *ldb_sector = NULL; 62 | uint8_t *node; 63 | 64 | /* Open sector from disk (if *sector is not provided) */ 65 | if (sector) node = sector; 66 | else 67 | { 68 | ldb_sector = ldb_open(table, key, "r"); 69 | if (!ldb_sector) return 0; 70 | node = calloc(LDB_MAX_REC_LN + 1, 1); 71 | } 72 | 73 | uint64_t next = 0; 74 | uint32_t node_size = 0; 75 | uint32_t node_ptr; 76 | uint8_t subkey_ln = table.key_ln - LDB_KEY_LN; 77 | 78 | uint32_t records = 0; 79 | bool done = false; 80 | logger_dbname_set(table.db); 81 | do 82 | { 83 | /* Read node */ 84 | next = ldb_node_read(sector, table, ldb_sector, next, key, &node_size, &node, 0); 85 | if (ldb_read_failure) 86 | { 87 | log_info("Error reading table %s/%s - sector %02x: the file is not available or the node doesn't exist\n", table.db, table.table, key[0]); 88 | ldb_read_failure = false; 89 | next = 0; 90 | } 91 | if (!node_size && !next) 92 | break; // reached end of list 93 | 94 | /* Pass entire node (fixed record length) to handler */ 95 | if (table.rec_ln) 96 | done = ldb_record_handler(key, NULL, 0 , node, node_size, records++, void_ptr); 97 | 98 | /* Extract and pass variable-size records to handler */ 99 | else 100 | { 101 | if (!ldb_validate_node(node, node_size, subkey_ln)) 102 | continue; 103 | 104 | /* Extract datasets from node */ 105 | node_ptr = 0; 106 | 107 | while (node_ptr < node_size && !done) 108 | { 109 | /* Get subkey */ 110 | uint8_t *subkey = node + node_ptr; 111 | node_ptr += subkey_ln; 112 | 113 | /* Get recordset length */ 114 | int dataset_size = uint16_read(node + node_ptr); 115 | node_ptr += 2; 116 | 117 | /* Compare subkey */ 118 | bool key_matched = true; 119 | if (!skip_subkey && subkey_ln) 120 | key_matched = (memcmp(subkey, key + 4, subkey_ln) == 0); 121 | 122 | if (key_matched) 123 | { 124 | /* Extract records from dataset */ 125 | uint32_t dataset_ptr = 0; 126 | while (dataset_ptr < dataset_size) 127 | { 128 | uint8_t *dataset = node + node_ptr; 129 | 130 | /* Get record length */ 131 | int record_size = uint16_read(dataset + dataset_ptr); 132 | dataset_ptr += 2; 133 | 134 | /* We drop records longer than the desired limit */ 135 | if (record_size + 32 < LDB_MAX_REC_LN) 136 | done = ldb_record_handler(key, subkey, subkey_ln, dataset + dataset_ptr, record_size, records++, void_ptr); 137 | /* Move pointer to end of record */ 138 | dataset_ptr += record_size; 139 | } 140 | } 141 | /* Move pointer to end of dataset */ 142 | node_ptr += dataset_size; 143 | } 144 | } 145 | } while (next && !done); 146 | 147 | if (!sector) 148 | { 149 | free(node); 150 | fclose(ldb_sector); 151 | } 152 | 153 | return records; 154 | } 155 | 156 | /** 157 | * @brief Handler function for ldb_get_first_record 158 | * 159 | * @param key Not used 160 | * @param subkey Not used 161 | * @param subkey_ln Not used 162 | * @param data Source for the record 163 | * @param datalen Length of the record 164 | * @param iteration Not used 165 | * @param ptr Buffer where the record is written 166 | * @return true if datalen is > 0. False otherwise 167 | */ 168 | bool ldb_get_first_record_handler(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t datalen, int iteration, void *ptr) 169 | { 170 | uint8_t *record = (uint8_t *) ptr; 171 | if (datalen) 172 | { 173 | uint32_write(record, datalen); 174 | memcpy(record + 4, data, datalen); 175 | return true; 176 | } 177 | return false; 178 | } 179 | 180 | /** 181 | * @brief Return the first record for the given table/key 182 | * 183 | * @param table Struct with the config of the table 184 | * @param key Key of the table 185 | * @param void_ptr Pointer passed to the handler function 186 | */ 187 | void ldb_get_first_record(struct ldb_table table, uint8_t* key, void *void_ptr) 188 | { 189 | ldb_fetch_recordset(NULL, table, key, false, ldb_get_first_record_handler, void_ptr); 190 | } 191 | 192 | /** 193 | * @brief Handler function for ldb_key_exists 194 | * 195 | * @param key Not used 196 | * @param subkey Not used 197 | * @param subkey_ln Not used 198 | * @param data Not used 199 | * @param datalen Not used 200 | * @param iteration Not used 201 | * @param ptr Not used 202 | * @return true always (the key exists) 203 | */ 204 | bool ldb_key_exists_handler(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t datalen, int iteration, void *ptr) 205 | { 206 | return true; 207 | } 208 | 209 | /** 210 | * @brief Returns true if there is at least a record for the "key" in the "table" 211 | * 212 | * @param table Struct with the config of the table 213 | * @param key Key of the table 214 | * @return true if there is at least a record for the "key" in the "table" 215 | */ 216 | bool ldb_key_exists(struct ldb_table table, uint8_t *key) 217 | { 218 | return (ldb_fetch_recordset(NULL, table, key, false, ldb_key_exists_handler, NULL) > 0); 219 | } 220 | 221 | /** 222 | * @brief Fixed width recordset handler for hexdump 223 | * 224 | * For Example: See in function ldb_hexprint(); 225 | * 226 | * @param key block key 227 | * @param subkey block subkey 228 | * @param subkey_ln block subkey lenght 229 | * @param data Buffer to print 230 | * @param len Length of buffer 231 | * @param iteration Not used 232 | * @param ptr Pointer to integer. Stores the number of columns to be printed. 233 | * @return false Always return false. It 234 | */ 235 | bool ldb_hexprint_width(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t len, int iteration, void *ptr) 236 | { 237 | int *width = ptr; 238 | for (int i = 0; i < LDB_KEY_LN; i++) printf("%02x", key[i]); 239 | for (int i = 0; i < subkey_ln; i++) printf("%02x", subkey[i]); 240 | printf("\n"); 241 | ldb_hexprint(data, len, *width); 242 | printf("\n"); 243 | return false; 244 | } 245 | 246 | /** 247 | * @brief Prints to stdout all the parameters in pretty CSV format. 248 | * 249 | * Prints the key and subkey in hex format and then prints the data in ascii format. 250 | * If the data contains non printable characters a dot will be printed instead. 251 | * 252 | * @param key key to print 253 | * @param subkey subkey to print 254 | * @param subkey_ln length of the subkey 255 | * @param data data to print 256 | * @param size size of the data 257 | * @param iteration not used 258 | * @param ptr not used 259 | * @return false always. not used 260 | */ 261 | bool ldb_csvprint(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr) 262 | { 263 | /* Print key in hex (first CSV field) */ 264 | for (int i = 0; i < LDB_KEY_LN; i++) printf("%02x", key[i]); 265 | for (int i = 0; i < subkey_ln; i++) printf("%02x", subkey[i]); 266 | 267 | /* Print remaining hex bytes (if any, as a second CSV field) */ 268 | int *hex_bytes = ptr; 269 | int remaining_hex = 0; 270 | if (*hex_bytes < 0) 271 | remaining_hex = size; 272 | else 273 | remaining_hex = *hex_bytes - LDB_KEY_LN - subkey_ln; 274 | 275 | if (remaining_hex < 0) remaining_hex = 0; 276 | 277 | if (remaining_hex) 278 | { 279 | printf(","); 280 | for (int i = 0; i < remaining_hex; i++) printf("%02x", data[i]); 281 | } 282 | 283 | /* Print remaining CSV data */ 284 | printf(","); 285 | for (int i = remaining_hex; i < size; i++) 286 | fwrite(data + i, 1, 1, stdout); 287 | 288 | fwrite("\n", 1, 1, stdout); 289 | return false; 290 | } 291 | 292 | /** 293 | * @brief Prints to stdout all the parameters in pretty format. 294 | * 295 | * Prints the key and subkey in hex format and then prints the data in ascii format. 296 | * If the data contains non printable characters a dot will be printed instead. 297 | * 298 | * @param key key to print 299 | * @param subkey subkey to print 300 | * @param subkey_ln length of the subkey 301 | * @param data data to print 302 | * @param size size of the data 303 | * @param iteration not used 304 | * @param ptr not used 305 | * @return false always. not used 306 | */ 307 | bool ldb_asciiprint(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr) 308 | { 309 | for (int i = 0; i < LDB_KEY_LN; i++) printf("%02x", key[i]); 310 | for (int i = 0; i < subkey_ln; i++) printf("%02x", subkey[i]); 311 | 312 | printf(": "); 313 | 314 | for (int i = 0; i < size; i++) 315 | if (data[i] >= 32 && data[i] <= 126) 316 | fwrite(data + i, 1, 1, stdout); 317 | else 318 | fwrite(".", 1, 1, stdout); 319 | 320 | fwrite("\n", 1, 1, stdout); 321 | return false; 322 | } 323 | -------------------------------------------------------------------------------- /src/sector.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/sector.c 4 | * 5 | * LDB sector handling routines 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "./ldb.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "ldb_string.h" 30 | /** 31 | * @file sector.c 32 | * @date 12 Jul 2020 33 | * @brief Contains functions related to table and database creations. Also has helper functions for sector handling. 34 | 35 | * //TODO Long description 36 | * @see https://github.com/scanoss/ldb/blob/master/src/sector.c 37 | */ 38 | 39 | FILE *lock_file(const char *filename, int wait_s, const char *mode) 40 | { 41 | 42 | int fd = open(filename, O_RDWR); 43 | 44 | if (fd == -1) 45 | { 46 | perror("open"); 47 | return NULL; 48 | } 49 | 50 | struct flock fl; 51 | fl.l_type = F_WRLCK; 52 | fl.l_whence = SEEK_SET; 53 | fl.l_start = 0; 54 | fl.l_len = 0; 55 | 56 | int ret; 57 | time_t start_time = time(NULL); 58 | while ((ret = fcntl(fd, F_SETLK, &fl)) == -1) 59 | { 60 | if (errno != EACCES && errno != EAGAIN) 61 | { 62 | perror("fcntl"); 63 | close(fd); 64 | return NULL; 65 | } 66 | 67 | if (difftime(time(NULL), start_time) >= wait_s) 68 | { 69 | fprintf(stderr, "Timeout waiting for lock\n"); 70 | close(fd); 71 | return NULL; 72 | } 73 | 74 | usleep(250); // Wait 1 millisecond before trying again 75 | } 76 | 77 | FILE *fp = fdopen(fd, mode); 78 | 79 | if (fp == NULL) 80 | { 81 | perror("fdopen"); 82 | close(fd); 83 | //return NULL; 84 | exit(1); 85 | } 86 | return fp; 87 | } 88 | 89 | int ldb_close_unlock(FILE *fp) 90 | { 91 | int fd = fileno(fp); 92 | struct flock fl; 93 | fl.l_type = F_UNLCK; 94 | fl.l_whence = SEEK_SET; 95 | fl.l_start = 0; 96 | fl.l_len = 0; 97 | 98 | int ret = fcntl(fd, F_SETLK, &fl); 99 | if (ret == -1) 100 | { 101 | perror("fcntl"); 102 | return -1; 103 | } 104 | 105 | if (fclose(fp) == EOF) 106 | { 107 | perror("fclose"); 108 | return -1; 109 | } 110 | return 0; 111 | } 112 | /** 113 | * @brief Opens an LDB sector and returns the file descriptor. If read mode, returns NULL 114 | * in case it does not exist. Otherwise an empty sector file is created in case it 115 | * does not exist 116 | * 117 | * @param table Struct with the table configuration. 118 | * @param key Key of the table 119 | * @param mode Opens the db in read or write mode 120 | * @return FILE* File descriptor of the ldb_sector (sector_path is the filepath associated with a pair tablename and a key) 121 | */ 122 | FILE *ldb_open(struct ldb_table table, uint8_t *key, char *mode) { 123 | 124 | /* Create sector (file) if it doesn't already exist */ 125 | char *sector_path = ldb_sector_path(table, key, mode); 126 | if (!sector_path) 127 | return NULL; 128 | 129 | FILE *out = NULL; 130 | 131 | /* Open data sector */ 132 | if (strcmp(mode, "r")) 133 | out = lock_file(sector_path, 5, mode); 134 | else 135 | out = fopen(sector_path, mode); 136 | 137 | if (!out) 138 | fprintf(stderr, "Cannot open LDB with mode %s: %s\n", mode, strerror(errno)); 139 | 140 | free(sector_path); 141 | return out; 142 | } 143 | 144 | bool ldb_close(FILE * sector) 145 | { 146 | if (fclose(sector) != 0) 147 | { 148 | printf("error closing sector\n"); 149 | return false; 150 | } 151 | 152 | return true; 153 | } 154 | 155 | 156 | /** 157 | * @brief Creates a table in the ldb directory 158 | * 159 | * @param db database name 160 | * @param table table name 161 | * @param keylen length of the key 162 | * @param reclen length of the record 163 | * @return true success. false failure 164 | */ 165 | bool ldb_create_table_new(char *db, char *table, int keylen, int reclen, int keys, int definitions) 166 | { 167 | bool out = false; 168 | 169 | char *dbpath = malloc(LDB_MAX_PATH); 170 | sprintf(dbpath, "%s/%s", ldb_root, db); 171 | 172 | char *tablepath = malloc(LDB_MAX_PATH); 173 | sprintf(tablepath, "%s/%s/%s", ldb_root, db, table); 174 | 175 | if (keys < 1) 176 | keys = 1; 177 | 178 | if (!ldb_valid_name(db) || !ldb_valid_name(table)) 179 | { 180 | printf("E064 Invalid characters or name is too long\n"); 181 | } 182 | else if (!ldb_dir_exists(dbpath)) 183 | { 184 | printf("E062 Database does not exist\n"); 185 | } 186 | else if (ldb_dir_exists(tablepath)) 187 | { 188 | printf("E069 Table already exists\n"); 189 | } 190 | else { 191 | mkdir(tablepath, 0755); 192 | if (ldb_dir_exists(tablepath)) 193 | { 194 | ldb_write_cfg(db, table, keylen, reclen, keys, definitions); 195 | out = true; 196 | } 197 | else printf("E065 Cannot create %s\n", tablepath); 198 | } 199 | 200 | free(dbpath); 201 | free(tablepath); 202 | return out; 203 | } 204 | 205 | //For backward compatibility 206 | bool ldb_create_table(char *db, char *table, int keylen, int reclen) 207 | { 208 | return ldb_create_table_new(db, table, keylen, reclen, 1, 0); 209 | } 210 | 211 | /** 212 | * @brief Creates the databases folders from a database name 213 | * The path for the folder is a concatenation of ldb_root + database name. ldb_root is defined in ldb.c 214 | * 215 | * @param database String with the database name 216 | * @return true Success 217 | */ 218 | bool ldb_create_database(char *database) 219 | { 220 | bool out = false; 221 | 222 | char *path = malloc(LDB_MAX_PATH); 223 | sprintf(path, "%s/%s", ldb_root, database); 224 | if (ldb_dir_exists(path)) 225 | { 226 | printf("E068 Database already exists\n"); 227 | } 228 | else { 229 | mkdir(path, 0755); 230 | if (ldb_dir_exists(path)) 231 | out = true; 232 | else 233 | printf("E065 Cannot create %s\n", path); 234 | } 235 | 236 | free(path); 237 | return out; 238 | } 239 | 240 | 241 | /** 242 | * @brief Loads an entire LDB sector into memory and returns a pointer 243 | (NULL if the sector does not exist) 244 | * 245 | * @param table Instance of the table struct. 246 | * @param key Key of the sector to load. 247 | * @return uint8_t* Pointer to the block of memory with the sector loaded. 248 | */ 249 | uint8_t *ldb_load_sector(struct ldb_table table, uint8_t *key) { 250 | 251 | FILE *ldb_sector = ldb_open(table, key, "r"); 252 | if (!ldb_sector) return NULL; 253 | 254 | fseeko64(ldb_sector, 0, SEEK_END); 255 | uint64_t size = ftello64(ldb_sector); 256 | 257 | uint8_t *out = malloc(size); 258 | if (!out) 259 | return NULL; 260 | 261 | fseeko64(ldb_sector, 0, SEEK_SET); 262 | if (!fread(out, 1, size, ldb_sector)) printf("Warning: ldb_load_sector failed\n"); 263 | fclose(ldb_sector); 264 | 265 | return out; 266 | } 267 | 268 | /** 269 | * @brief Reserves memory for storing a copy of an entire LDB sector 270 | * (returns NULL if the source sector does not exist) 271 | * 272 | * @param table Instance of the table struct. 273 | * @param key Key of the sector 274 | * @return uint8_t* Pointer to the block of memory. 275 | */ 276 | uint8_t *ldb_load_new_sector(struct ldb_table table, uint8_t *key) { 277 | 278 | FILE *ldb_sector = ldb_open(table, key, "r"); // Opens a ldb in read mode. Will return NULL if it does not exist. 279 | if (!ldb_sector) return NULL; 280 | 281 | fseeko64(ldb_sector, 0, SEEK_END); 282 | uint64_t size = ftello64(ldb_sector); 283 | fclose(ldb_sector); 284 | 285 | if (!size) return NULL; 286 | 287 | uint8_t *out = malloc(size); 288 | return out; 289 | } 290 | 291 | /** 292 | * @brief Create an empty data sector (empty map) 293 | * 294 | * @param sector_path Path to the sector 295 | */ 296 | void ldb_create_sector(char *sector_path) 297 | { 298 | uint8_t *ldb_empty_map = calloc(LDB_MAP_SIZE, 1); 299 | 300 | FILE *ldb_map = fopen(sector_path, "w"); 301 | if (!ldb_map) 302 | { 303 | ldb_error("E065 Cannot access ldb table. Check permissions."); 304 | exit(EXIT_FAILURE); 305 | } 306 | fwrite(ldb_empty_map, LDB_MAP_SIZE, 1, ldb_map); 307 | fclose(ldb_map); 308 | 309 | free(ldb_empty_map); 310 | } 311 | 312 | /** 313 | * @brief Moves sector.tmp into sector.ldb 314 | * Copy a temporary sector into a permanent sector. 315 | * 316 | * @param table Instance of the table struct. 317 | * @param key Key of the sector. 318 | */ 319 | void ldb_sector_update(struct ldb_table table, uint8_t *key) 320 | { 321 | char sector_ldb[LDB_MAX_PATH] = "\0"; 322 | char sector_tmp[LDB_MAX_PATH] = "\0"; 323 | sprintf(sector_ldb, "%s/%s/%s/%02x.ldb", ldb_root, table.db, table.table, key[0]); 324 | sprintf(sector_tmp, "%s/%s/%s/%02x.tmp", ldb_root, table.db, table.table, key[0]); 325 | 326 | if (!ldb_file_exists(sector_tmp)) 327 | { 328 | printf("%s\n", sector_tmp); 329 | ldb_error("E074 Cannot update sector with .tmp "); 330 | } 331 | 332 | if (ldb_file_exists(sector_ldb) && unlink(sector_ldb)) 333 | ldb_error("E074 Cannot update sector with .tmp, cannot remove old .ldb file."); 334 | 335 | if (!rename(sector_tmp, sector_ldb)) 336 | return; 337 | 338 | ldb_error("E074 Error replacing sector with .tmp"); 339 | } 340 | 341 | /** 342 | * @brief Erases sector.ldb 343 | * 344 | * Does not erase sector.tmp 345 | * 346 | * @param table Table struct that will be erased 347 | * @param key Key of the sector to be erased 348 | */ 349 | void ldb_sector_erase(struct ldb_table table, uint8_t *key) 350 | { 351 | char sector_ldb[LDB_MAX_PATH] = "\0"; 352 | sprintf(sector_ldb, "%s/%s/%s/%02x.ldb", ldb_root, table.db, table.table, key[0]); 353 | 354 | if (!ldb_file_exists(sector_ldb)) 355 | { 356 | ldb_error("E074 Cannot erase sector"); 357 | } 358 | 359 | if (!unlink(sector_ldb)) return; 360 | 361 | ldb_error("E074 Error erasing sector"); 362 | } 363 | 364 | /** 365 | * @brief Returns the sector path for a given table_path and key 366 | * 367 | * Table_path is a concatenation of ldb_root + database_name + table_name 368 | * - ldb_root is defined on ldb.c 369 | * - database_name is obtained from the struct table (table.db) 370 | * - table_name is obtained from the struct table (table.table) 371 | * 372 | * @param table Instance of the table struct in order to get the sector_path 373 | * @param key Key of the sector to be accessed 374 | * @param mode Mode of the sector to be accessed mode = "r" for only read. mode = "r+" for reads and write 375 | * @param tmp Boolean to indicate if the sector is temporary or not. (Will return the path with extention .tmp or .ldb) 376 | * @return char* Sector path according to the table and key provided. 377 | */ 378 | char *ldb_sector_path(struct ldb_table table, uint8_t *key, char *mode) 379 | { 380 | /* Create table (directory) if it doesn't already exist */ 381 | char table_path[LDB_MAX_PATH] = "\0"; 382 | sprintf (table_path, "%s/%s/%s", ldb_root, table.db, table.table); 383 | 384 | if (!ldb_dir_exists(table_path)) 385 | { 386 | printf("E063 Table %s does not exist\n", table_path); 387 | exit(EXIT_FAILURE); 388 | } 389 | 390 | char *sector_path = malloc(strlen(table_path) + LDB_MAX_PATH + 1); 391 | if (table.tmp) 392 | sprintf(sector_path, "%s/%02x.tmp", table_path, key[0]); 393 | else 394 | sprintf(sector_path, "%s/%02x.ldb", table_path, key[0]); 395 | 396 | /* If opening a tmp table, we remove the file if it exists */ 397 | //if (ldb_file_exists(sector_path) && tmp) remove(sector_path); 398 | 399 | if (!ldb_file_exists(sector_path)) 400 | { 401 | if (!strcmp(mode,"r")) 402 | { 403 | free(sector_path); 404 | return NULL; 405 | } 406 | ldb_create_sector(sector_path); 407 | } 408 | 409 | return sector_path; 410 | } -------------------------------------------------------------------------------- /src/shell.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-or-later 2 | /* 3 | * src/shell.c 4 | * 5 | * LDB Database simple shell 6 | * 7 | * Copyright (C) 2018-2020 SCANOSS.COM 8 | * 9 | * This program is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 2 of the License, or 12 | * (at your option) any later version. 13 | 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | */ 22 | #include "ldb.h" 23 | /** 24 | * @file shell.c 25 | * @date 12 Jul 2020 26 | * @brief Contains shell functions and help text 27 | 28 | * //TODO Long description 29 | * @see https://github.com/scanoss/ldb/blob/master/src/shell.c 30 | */ 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include "ldb.h" 45 | #include "command.h" 46 | #include "ldb_string.h" 47 | #include "logger.h" 48 | /** 49 | * @brief Contains the shell help text 50 | * 51 | */ 52 | void help() 53 | { 54 | printf("LDB stores information using single, 32-bit keys and single data records. Data records could be fixed in size (drastically footprint for large amounts of short, fixed-sized records). The LDB console accepts the following commands:\n"); 55 | printf("\n"); 56 | printf("Shell Commands\n"); 57 | printf(" create database DBNAME\n"); 58 | printf(" Creates an empty database\n\n"); 59 | 60 | printf(" create table DBNAME/TABLENAME keylen N reclen N\n"); 61 | printf(" Creates an empty table in the given database with\n"); 62 | printf(" the specified key length (>= 4) and record length (0=variable)\n\n"); 63 | 64 | printf(" show databases\n"); 65 | printf(" Lists databases\n\n"); 66 | 67 | printf(" show tables from DBNAME\n"); 68 | printf(" Lists tables from given database\n\n"); 69 | 70 | printf("bulk insert DBNAME/TABLENAME from PATH with (CONFIG)\n"); 71 | printf("Import data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported.\n"); 72 | printf("TABLENAME is optional and will be defined from the directory name's file if not specified.\n"); 73 | printf("(CONFIG) is a configuration string with the following format:\n"); 74 | printf(" (FILE_DEL=1/0,KEYS=N,MZ=1/0,BIN=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp)\n"); 75 | printf(" Where 1/0 represents true/false, and N is an integer.\n"); 76 | printf(" FILE_DEL: Delete file after importation is complete.\n"); 77 | printf(" KEYS: Number of binary keys in the CSV file.\n"); 78 | printf(" MZ: MZ file indicator.\n"); 79 | printf(" WFP: WFP file indicator.\n"); 80 | printf(" OVERWRITE: Overwrite the destination table.\n"); 81 | printf(" SORT: Sort during the importation. Default: 1\n"); 82 | printf(" FIELDS: Number of CSV fields.\n"); 83 | printf(" VALIDATE_FIELDS: Check field quantity during importation. Default: 1\n"); 84 | printf(" VALIDATE_VERSION: Validate version.json. Default: 1\n"); 85 | printf(" VERBOSE: Enable verbose mode. Default: 0\n"); 86 | printf(" THREADS: Define the number of threads to be used during the importation process. Defaul value: half of system available.\n"); 87 | printf(" COLLATE: Perform collation after import, removing data larger than MAX_RECORD bytes. Default: 0\n"); 88 | printf(" MAX_RECORD: define the max record size, if a sector is bigger than \"MAX_RECORD\" bytes will be removed.\n"); 89 | printf(" MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50.\n"); 90 | printf(" TMP_PATH: Define the temporary directory. Default value \"/tmp\".\n"); 91 | printf(" It is not mandatory to specify all parameters; default values will be assumed for missing parameters.\n\n"); 92 | 93 | printf(" bulk insert DBNAME/TABLENAME from PATH\n"); 94 | printf(" Import data from PATH into given db/table. If PATH is a directory, the files inside will be recursively imported.\n"); 95 | printf(" The configuration will be taken from the file \"db.conf\" at %s. A default configuration file will be created if it does not exist\n", LDB_CFG_PATH); 96 | 97 | printf(" insert into DBNAME/TABLENAME key KEY hex DATA\n"); 98 | printf(" Inserts data (hex) into given db/table for the given hex key\n\n"); 99 | 100 | printf(" insert into DBNAME/TABLENAME key KEY ascii DATA\n"); 101 | printf(" Inserts data (ASCII) into db/table for the given hex key\n\n"); 102 | 103 | printf(" select from DBNAME/TABLENAME key KEY\n"); 104 | printf(" Retrieves all records from db/table for the given hex key (hexdump output)\n\n"); 105 | 106 | printf(" select from DBNAME/TABLENAME key KEY ascii\n"); 107 | printf(" Retrieves all records from db/table for the given hex key (ascii output)\n\n"); 108 | 109 | printf(" select from DBNAME/TABLENAME key KEY csv hex N\n"); 110 | printf(" Retrieves all records from db/table for the given hex key (csv output, with first N bytes in hex)\n\n"); 111 | 112 | printf(" delete from DBNAME/TABLENAME max LENGTH keys KEY_LIST\n"); 113 | printf(" Deletes all records for the given comma separated hex key list from the db/table. Max record length expected\n\n"); 114 | 115 | printf(" delete from DBNAME/TABLENAME record CSV_RECORD\n"); 116 | printf(" Deletes the specific CSV record from the specified table. Some field of the CSV may be skippet from the comparation using '*'\n"); 117 | printf(" Example 1: delete from db/url record key,madler,*,2.4,20171227,zlib,pkg:github/madler/pigz,https://github.com/madler/pigz/archive/v2.4.zip\n"); 118 | printf(" All the records matching the all the csv's field with exception of the second thirdone will be removed\n\n"); 119 | 120 | printf(" delete from DBNAME/TABLENAME records from PATH\n"); 121 | printf(" Similar to the previous command, but the records (may be more than one) will be loaded from a csv file in PATH\n\n"); 122 | 123 | printf(" collate DBNAME/TABLENAME max LENGTH\n"); 124 | printf(" Collates all lists in a table, removing duplicates and records greater than LENGTH bytes\n\n"); 125 | 126 | printf(" merge DBNAME/TABLENAME1 into DBNAME/TABLENAME2 max LENGTH\n"); 127 | printf(" Merges tables erasing tablename1 when done. Tables must have the same configuration\n\n"); 128 | 129 | printf(" unlink list from DBNAME/TABLENAME key KEY\n"); 130 | printf(" Unlinks the given list (32-bit KEY) from the sector map\n\n"); 131 | 132 | printf(" dump DBNAME/TABLENAME hex N [sector N], use 'hex -1' to print the complete register as hex\n"); 133 | printf(" Dumps table contents with first N bytes in hex\n\n"); 134 | 135 | printf(" dump keys from DBNAME/TABLENAME [sector N]\n"); 136 | printf(" Dumps a unique list of existing keys\n\n"); 137 | 138 | printf(" cat KEY from DBNAME/MZTABLE\n"); 139 | printf(" Shows the contents for KEY in MZ archive\n"); 140 | 141 | printf("Other uses\n"); 142 | printf(" ldb -u [--update] path -n[--name] db_name -c[--collate]\n"); 143 | printf(" create \"db_name\" or update a existent one from \"path\". If \"db_name\" is not specified \"oss\" will be used by default.\n"); 144 | printf(" If \"--collate\" option is present, each table will be collated during the importation process.\n"); 145 | printf(" This command is an alias of \"bulk insert\" using the default parameters of an standar ldb\n"); 146 | printf(" ldb -f [filename] Process a list of commands from a file named filename\n"); 147 | } 148 | 149 | /** 150 | * @brief Process and run the user input command 151 | * 152 | * @param raw_command string with the user input 153 | * @return true if the program must keept running, false otherwise 154 | */ 155 | bool execute(char *raw_command) 156 | { 157 | 158 | char *command = ldb_command_normalize(raw_command); 159 | 160 | // Empty command does nothing 161 | if (!strlen(command)) return true; 162 | 163 | // QUIT quits 164 | if (!strcmp(command,"quit")) return false; 165 | 166 | // Parse other commands 167 | int command_nr = 0; 168 | int word_nr = 0; 169 | if (!ldb_syntax_check(command, &command_nr, &word_nr)) 170 | { 171 | printf("E066 Syntax error\n"); 172 | free(command); 173 | return true; 174 | } 175 | switch (command_nr) 176 | { 177 | case HELP: 178 | help(); 179 | break; 180 | 181 | case SHOW_TABLES: 182 | ldb_command_show_tables(command); 183 | break; 184 | 185 | case SHOW_DATABASES: 186 | ldb_command_show_databases(); 187 | break; 188 | 189 | case INSERT_ASCII: 190 | ldb_command_insert(command, command_nr); 191 | break; 192 | 193 | case BULK_INSERT: 194 | case BULK_INSERT_DEFAULT: 195 | ldb_command_bulk(command, command_nr); 196 | break; 197 | 198 | case INSERT_HEX: 199 | ldb_command_insert(command, command_nr); 200 | break; 201 | 202 | case SELECT: 203 | ldb_command_select(command, HEX); 204 | break; 205 | 206 | case SELECT_ASCII: 207 | ldb_command_select(command, ASCII); 208 | break; 209 | 210 | case SELECT_CSV: 211 | ldb_command_select(command, CSV); 212 | break; 213 | 214 | case CREATE_DATABASE: 215 | ldb_command_create_database(command); 216 | break; 217 | 218 | case CREATE_TABLE: 219 | ldb_command_create_table(command); 220 | break; 221 | case CREATE_CONFIG: 222 | ldb_command_create_config(command); 223 | break; 224 | 225 | case UNLINK_LIST: 226 | ldb_command_unlink_list(command); 227 | break; 228 | 229 | case COLLATE: 230 | ldb_command_collate(command); 231 | break; 232 | 233 | case DELETE: 234 | ldb_command_delete(command); 235 | break; 236 | case DELETE_RECORD: 237 | case DELETE_RECORDS: 238 | ldb_command_delete_records(command); 239 | break; 240 | 241 | case MERGE: 242 | ldb_command_merge(command); 243 | break; 244 | 245 | case DUMP_KEYS: 246 | ldb_command_dump_keys(command); 247 | break; 248 | 249 | case VERSION: 250 | ldb_version(NULL); 251 | break; 252 | 253 | case DUMP: 254 | ldb_command_dump(command); 255 | break; 256 | 257 | case DUMP_SECTOR: 258 | ldb_command_dump(command); 259 | break; 260 | 261 | default: 262 | printf("E067 Command not implemented\n"); 263 | break; 264 | } 265 | 266 | free(command); 267 | return true; 268 | } 269 | 270 | /** 271 | * @brief Handle the command line interface 272 | * To close the program, the variable stay should be set to false 273 | * 274 | * @return true if the program must keept running, false otherwise 275 | */ 276 | bool stdin_handle() 277 | { 278 | 279 | char *command = NULL; 280 | size_t size = 0; 281 | 282 | if (!getline(&command, &size, stdin)) printf("Warning: cannot read STDIN\n"); 283 | ldb_trim(command); 284 | 285 | bool stay = execute(command); 286 | 287 | free(command); 288 | return stay; 289 | } 290 | 291 | void file_handle(char *filename) 292 | { 293 | char line[LDB_MAX_PATH]; 294 | 295 | FILE * cmd_file = fopen(filename, "r"); 296 | if (cmd_file == NULL) 297 | { 298 | printf("Can not open commands file.\n"); 299 | exit(EXIT_FAILURE); 300 | } 301 | 302 | while (fgets(line, sizeof(line), cmd_file) != NULL) 303 | { 304 | ldb_trim(line); 305 | execute(line); 306 | } 307 | 308 | fclose(cmd_file); 309 | } 310 | 311 | /** 312 | * @brief Prints the welcome banner 313 | * 314 | */ 315 | void welcome() 316 | { 317 | printf("Welcome to LDB %s\n", LDB_VERSION); 318 | printf("Use help for a command list and quit for leaving this session\n\n"); 319 | } 320 | 321 | /** 322 | * @brief Prints the ldb prompt 323 | * 324 | */ 325 | void ldb_prompt() 326 | { 327 | printf("ldb> "); 328 | } 329 | 330 | bool is_stdin_off() 331 | { 332 | struct termios t; 333 | return (tcgetattr(STDIN_FILENO, &t) == 0); 334 | } 335 | 336 | typedef enum 337 | { 338 | LDB_CONSOLE, 339 | LDB_UPDATE 340 | } ldb_mode_t; 341 | 342 | ldb_mode_t mode = LDB_CONSOLE; 343 | static int collate = 0; 344 | int main(int argc, char **argv) 345 | { 346 | char * dbname = NULL;//[LDB_MAX_NAME] = "\0"; 347 | char * path = NULL; 348 | 349 | static struct option long_options[] = 350 | { 351 | {"version", no_argument, 0, 'v'}, 352 | {"help", no_argument, 0, 'h'}, 353 | {"collate", no_argument, 0 , 'c'}, 354 | {"update", required_argument, 0, 'u'}, 355 | {"name", required_argument, 0, 'n'}, 356 | {"file", required_argument, 0, 'f'}, 357 | {"verbose", no_argument, 0, 'V'}, 358 | {"quiet", no_argument, 0, 'q'}, 359 | {0, 0, 0, 0} 360 | }; 361 | 362 | /* getopt_long stores the option index here. */ 363 | int option_index = 0; 364 | int opt; 365 | bool verbose = false; 366 | while ( (opt = getopt_long (argc, argv, "u:n:f:qchvV", long_options, &option_index)) >= 0) 367 | { 368 | /* Check valid alpha is entered */ 369 | switch (opt) 370 | { 371 | case 'v': 372 | ldb_version(NULL); 373 | return EXIT_SUCCESS; 374 | case 'h': 375 | help(); 376 | return EXIT_SUCCESS; 377 | case 'u': 378 | { 379 | mode = LDB_UPDATE; 380 | path = strdup(optarg); 381 | break; 382 | } 383 | case 'n': 384 | { 385 | dbname = strdup(optarg); 386 | break; 387 | } 388 | case 'f': 389 | { 390 | char * filename = strdup(optarg); 391 | file_handle(filename); 392 | free(filename); 393 | exit(EXIT_SUCCESS); 394 | } 395 | case 'c': 396 | collate = true; 397 | break; 398 | case 'V': 399 | { 400 | verbose = true; 401 | break; 402 | } 403 | case 'q': 404 | { 405 | log_set_quiet(true); 406 | break; 407 | } 408 | default: 409 | break; 410 | } 411 | } 412 | 413 | switch (mode) 414 | { 415 | case LDB_UPDATE: 416 | { 417 | char cmd [LDB_MAX_PATH] = "(VALIDATE_VERSION=1"; 418 | if (*path) 419 | { 420 | if (collate) 421 | strcat(cmd, ",COLLATE=1"); 422 | 423 | if (verbose) 424 | strcat(cmd, ",VERBOSE=1"); 425 | 426 | strcat(cmd, ")"); 427 | if (!dbname || !*dbname) 428 | ldb_import_command("oss", path, cmd); 429 | else 430 | ldb_import_command(dbname, path, cmd); 431 | fprintf(stderr, "\r\nImport process end\n\n"); 432 | return EXIT_SUCCESS; 433 | } 434 | break; 435 | } 436 | default: 437 | break; 438 | } 439 | 440 | free(dbname); 441 | 442 | bool stdin_off = is_stdin_off(); 443 | 444 | if (!ldb_check_root()) return EXIT_FAILURE; 445 | 446 | if (stdin_off) welcome(); 447 | 448 | do if (stdin_off) ldb_prompt(); 449 | while (stdin_handle() && stdin_off); 450 | return EXIT_SUCCESS; 451 | 452 | } 453 | 454 | 455 | -------------------------------------------------------------------------------- /test/bash_unit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # bash unit testing enterprise edition framework for professionals 4 | # Copyright (C) 2011-2016 Pascal Grange 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 3 of the License, or 8 | # (at your option) any later version. 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program; if not, write to the Free Software Foundation, 15 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 16 | # 17 | # https://github.com/pgrange/bash_unit 18 | 19 | VERSION=v2.1.0 20 | 21 | ESCAPE=$(printf "\033") 22 | NOCOLOR="${ESCAPE}[0m" 23 | RED="${ESCAPE}[91m" 24 | GREEN="${ESCAPE}[92m" 25 | YELLOW="${ESCAPE}[93m" 26 | BLUE="${ESCAPE}[94m" 27 | 28 | # Make bash_unit immune to some basic unix commands faking 29 | CAT="$(which cat)" 30 | SED="$(which sed)" 31 | GREP="$(which grep)" 32 | RM="$(which rm)" 33 | SHUF="$(which shuf)" 34 | 35 | fail() { 36 | local message=${1:-} 37 | local stdout=${2:-} 38 | local stderr=${3:-} 39 | 40 | notify_test_failed "$__bash_unit_current_test__" "$message" 41 | [[ ! -z $stdout ]] && [ -s "$stdout" ] && notify_stdout < "$stdout" 42 | [[ ! -z $stderr ]] && [ -s "$stderr" ] && notify_stderr < "$stderr" 43 | 44 | stacktrace | notify_stack 45 | exit 1 46 | } 47 | 48 | assert() { 49 | local assertion=$1 50 | local message=${2:-} 51 | 52 | _assert_expression \ 53 | "$assertion" \ 54 | "[ \$status == 0 ]" \ 55 | "\"$message\"" 56 | } 57 | 58 | assert_fails() { 59 | local assertion=$1 60 | local message=${2:-} 61 | 62 | _assert_expression \ 63 | "$assertion" \ 64 | "[ \$status != 0 ]" \ 65 | "\"$message\"" 66 | } 67 | 68 | assert_fail() { 69 | #deprecated, use assert_fails instead 70 | assert_fails "$@" 71 | } 72 | 73 | assert_status_code() { 74 | local expected_status=$1 75 | local assertion="$2" 76 | local message="${3:-}" 77 | 78 | _assert_expression \ 79 | "$assertion" \ 80 | "[ \$status == $expected_status ]" \ 81 | "\"$message\" expected status code $expected_status but was \$status" 82 | } 83 | 84 | _assert_expression() { 85 | local assertion=$1 86 | local condition=$2 87 | local message=$3 88 | ( 89 | local stdout=$(mktemp) 90 | local stderr=$(mktemp) 91 | trap "$RM -f \"$stdout\" \"$stderr\"" EXIT 92 | 93 | local status 94 | eval "($assertion)" >"$stdout" 2>"$stderr" && status=$? || status=$? 95 | if ! eval "$condition" 96 | then 97 | fail "$(eval echo $message)" "$stdout" "$stderr" 98 | fi 99 | ) || exit $? 100 | } 101 | 102 | assert_equals() { 103 | local expected=$1 104 | local actual=$2 105 | local message=${3:-} 106 | [[ -z $message ]] || message="$message\n" 107 | 108 | if [ "$expected" != "$actual" ] 109 | then 110 | fail "$message expected [$expected] but was [$actual]" 111 | fi 112 | } 113 | 114 | assert_not_equals() { 115 | local unexpected=$1 116 | local actual=$2 117 | local message=${3:-} 118 | [[ -z $message ]] || message="$message\n" 119 | 120 | [ "$unexpected" != "$actual" ] || \ 121 | fail "$message expected different value than [$unexpected] but was the same" 122 | } 123 | 124 | assert_matches() { 125 | local expected=$1 126 | local actual=$2 127 | local message=${3:-} 128 | [[ -z $message ]] || message="$message\n" 129 | 130 | if [[ ! "${actual}" =~ ${expected} ]]; then 131 | fail "$message expected regex [$expected] to match [$actual]" 132 | fi 133 | } 134 | 135 | assert_not_matches() { 136 | local unexpected=$1 137 | local actual=$2 138 | local message=${3:-} 139 | [[ -z $message ]] || message="$message\n" 140 | 141 | if [[ "${actual}" =~ ${unexpected} ]]; then 142 | fail "$message expected regex [$unexpected] should not match but matched [$actual]" 143 | fi 144 | } 145 | 146 | assert_within_delta() { 147 | function abs() { 148 | local value=$1 149 | local sign=$(( value < 0 ? -1 : 1 )) 150 | echo $((value * sign)) 151 | } 152 | function is_number() { 153 | local value=$1 154 | test $value -eq $value 2>/dev/null 155 | } 156 | local expected=$1 157 | local actual=$2 158 | local max_delta=$3 159 | assert "is_number $expected" "$message expected value [$expected] is not a number" 160 | assert "is_number $actual" "$message actual value [$actual] is not a number" 161 | assert "is_number $max_delta" "$message max_delta [$max_delta] is not a number" 162 | local message=${4:-} 163 | [[ -z $message ]] || message="$message\n" 164 | 165 | local actual_delta="$(abs $(($expected - $actual)))" 166 | 167 | if (( $actual_delta > $max_delta )); then 168 | fail "$message expected value [$expected] to match [$actual] with a maximum delta of [$max_delta]" 169 | fi 170 | } 171 | 172 | assert_no_diff() { 173 | local expected=$1 174 | local actual=$2 175 | local message=${3:-} 176 | [[ -z $message ]] || message="$message\n" 177 | 178 | assert 'diff '"${expected}"' '"${actual}" \ 179 | "$message expected '"${actual}"' to be identical to '"${expected}"' but was different" 180 | } 181 | 182 | fake() { 183 | local command=$1 184 | shift 185 | if [ $# -gt 0 ] 186 | then 187 | eval "function $command() { export FAKE_PARAMS=(\"\$@\") ; $@ ; }" 188 | else 189 | eval "function $command() { echo \"$($CAT)\" ; }" 190 | fi 191 | export -f $command 192 | } 193 | 194 | stacktrace() { 195 | local i=1 196 | while ! [ -z "${BASH_SOURCE[$i]:-}" ] 197 | do 198 | echo ${BASH_SOURCE[$i]}:${BASH_LINENO[$((i-1))]}:${FUNCNAME[$i]}\(\) 199 | i=$((i + 1)) 200 | done | "$GREP" -v "^$BASH_SOURCE" 201 | } 202 | 203 | run_test_suite() { 204 | local failure=0 205 | 206 | if run_setup_suite 207 | then 208 | run_tests || failure=$? 209 | else 210 | failure=$? 211 | fi 212 | run_teardown_suite 213 | 214 | return $failure 215 | } 216 | 217 | run_setup_suite() { 218 | if declare -F | "$GREP" ' setup_suite$' >/dev/null 219 | then 220 | setup_suite 221 | fi 222 | } 223 | 224 | maybe_shuffle() { 225 | ((randomise)) && $SHUF || $CAT 226 | } 227 | 228 | run_tests() { 229 | local failure=0 230 | 231 | for pending_test in $(set | "$GREP" -E '^(pending|todo).* \(\)' | "$GREP" -E "$test_pattern" | "$SED" -e 's: .*::') 232 | do 233 | notify_test_starting "$pending_test" 234 | notify_test_pending "$pending_test" 235 | done 236 | 237 | 238 | for test in $(set | "$GREP" -E '^test.* \(\)' | "$GREP" -E "$test_pattern" | "$SED" -e 's: .*::' | maybe_shuffle) 239 | do 240 | ( 241 | local status=0 242 | declare -F | "$GREP" ' setup$' >/dev/null && setup 243 | (__bash_unit_current_test__="$test" run_test) || status=$? 244 | declare -F | "$GREP" ' teardown$' >/dev/null && teardown 245 | exit $status 246 | ) 247 | failure=$(( $? || failure)) 248 | done 249 | return $failure 250 | } 251 | 252 | run_test() { 253 | set -e 254 | notify_test_starting "$__bash_unit_current_test__" 255 | "$__bash_unit_current_test__" && notify_test_succeeded "$__bash_unit_current_test__" 256 | } 257 | 258 | run_teardown_suite() { 259 | if declare -F | "$GREP" ' teardown_suite$' >/dev/null 260 | then 261 | teardown_suite 262 | fi 263 | } 264 | 265 | usage() { 266 | echo "$1" >&2 267 | echo "$0 [-f ] [-p ] [-p ] [-r] ... ..." >&2 268 | echo >&2 269 | echo "Runs tests in test files that match s" >&2 270 | echo " is optional only supported value is tap" >&2 271 | echo "-r to execute test cases in random order" >&2 272 | echo "-v to get current version information" >&2 273 | echo "See https://github.com/pgrange/bash_unit" >&2 274 | exit 1 275 | } 276 | 277 | # Formating 278 | 279 | pretty_success() { 280 | pretty_format "$GREEN" "\u2713" "${1:-}" 281 | } 282 | 283 | pretty_warning() { 284 | pretty_format "$YELLOW" "\u2717" "$1" 285 | } 286 | 287 | pretty_failure() { 288 | pretty_format "$RED" "\u2717" "${1:-}" 289 | } 290 | 291 | pretty_format() { 292 | local color="$1" 293 | local pretty_symbol="$2" 294 | local alt_symbol="${3:-}" 295 | local term_utf8=false 296 | #env 297 | if is_terminal && [[ "${LANG:-}" =~ .*UTF-8.* ]] 298 | then 299 | term_utf8=true 300 | fi 301 | ( 302 | $CAT 303 | if $term_utf8 304 | then 305 | echo -en " $pretty_symbol " 306 | else 307 | [[ ! -z "$alt_symbol" ]] && echo -en " $alt_symbol " 308 | fi 309 | ) | color "$color" 310 | } 311 | 312 | color() { 313 | _start_color() { 314 | if is_terminal ; then echo -en "$color" ; fi 315 | } 316 | _stop_color() { 317 | if is_terminal ; then echo -en "$NOCOLOR" ; fi 318 | } 319 | local color=$1 320 | shift 321 | _start_color 322 | if [ $# -gt 0 ] 323 | then 324 | echo $* 325 | else 326 | $CAT 327 | fi 328 | _stop_color 329 | } 330 | 331 | is_terminal() { 332 | [ -t 1 ] || [[ "${FORCE_COLOR:-}" == true ]] 333 | } 334 | 335 | text_format() { 336 | notify_suite_starting() { 337 | local test_file="$1" 338 | echo "Running tests in $test_file" 339 | } 340 | notify_test_starting() { 341 | local test="$1" 342 | echo -e -n "\tRunning $test ... " | color "$BLUE" 343 | } 344 | notify_test_pending() { 345 | echo -n "PENDING" | pretty_warning 346 | echo 347 | } 348 | 349 | notify_test_succeeded() { 350 | echo -n "SUCCESS" | pretty_success 351 | echo 352 | } 353 | notify_test_failed() { 354 | local message="$2" 355 | echo -n "FAILURE" | pretty_failure 356 | echo 357 | [[ -z $message ]] || printf -- "$message\n" 358 | } 359 | notify_stdout() { 360 | "$SED" 's:^:out> :' | color "$GREEN" 361 | } 362 | notify_stderr() { 363 | "$SED" 's:^:err> :' | color "$RED" 364 | } 365 | notify_stack() { 366 | color "$YELLOW" 367 | } 368 | notify_suites_succeded() { 369 | echo -n "Overall result: SUCCESS" | pretty_success 370 | echo 371 | } 372 | notify_suites_failed() { 373 | echo -n "Overall result: FAILURE" | pretty_failure 374 | echo 375 | } 376 | } 377 | 378 | tap_format() { 379 | notify_suite_starting() { 380 | local test_file="$1" 381 | echo "# Running tests in $test_file" 382 | } 383 | notify_test_starting() { 384 | : 385 | } 386 | notify_test_pending() { 387 | local test="$1" 388 | echo -n "ok" | pretty_warning - 389 | echo -n "$test" | color "$BLUE" 390 | echo " # skip test to be written" | color "$YELLOW" 391 | } 392 | notify_test_succeeded() { 393 | local test="$1" 394 | echo -n "ok" | pretty_success - 395 | echo "$test" | color "$BLUE" 396 | } 397 | notify_test_failed() { 398 | local test="$1" 399 | local message="$2" 400 | echo -n "not ok" | pretty_failure - 401 | echo "$test" | color "$BLUE" 402 | [[ -z $message ]] || printf -- "$message\n" | "$SED" -u -e 's/^/# /' 403 | } 404 | notify_stdout() { 405 | "$SED" 's:^:# out> :' | color "$GREEN" 406 | } 407 | notify_stderr() { 408 | "$SED" 's:^:# err> :' | color "$RED" 409 | } 410 | notify_stack() { 411 | "$SED" 's:^:# :' | color "$YELLOW" 412 | } 413 | notify_suites_succeded() { 414 | : 415 | } 416 | notify_suites_failed() { 417 | : 418 | } 419 | } 420 | 421 | output_format=text 422 | test_pattern="" 423 | separator="" 424 | randomise=0 425 | while getopts "vp:f:r" option 426 | do 427 | case "$option" in 428 | p) 429 | test_pattern="${test_pattern}${separator}${OPTARG}" 430 | separator="|" 431 | ;; 432 | f) 433 | output_format="${OPTARG}" 434 | ;; 435 | r) 436 | randomise=1 437 | ;; 438 | v) 439 | echo "bash_unit $VERSION" 440 | exit 441 | ;; 442 | ?|:) 443 | usage 444 | ;; 445 | esac 446 | done 447 | shift $((OPTIND-1)) 448 | 449 | for test_file in "$@" 450 | do 451 | test -e "$test_file" || usage "file does not exist: $test_file" 452 | test -r "$test_file" || usage "can not read file: $test_file" 453 | done 454 | 455 | case "$output_format" in 456 | text) 457 | text_format 458 | ;; 459 | tap) 460 | tap_format 461 | ;; 462 | *) 463 | usage "unsupported output format: $output_format" 464 | ;; 465 | esac 466 | 467 | #run tests received as parameters 468 | failure=0 469 | for test_file in "$@" 470 | do 471 | notify_suite_starting "$test_file" 472 | ( 473 | set -e # Ensure bash_unit will exit with failure 474 | # in case of syntax error. 475 | if [[ "${STICK_TO_CWD}" != true ]] 476 | then 477 | cd "$(dirname "$test_file")" 478 | source "$(basename "$test_file")" 479 | else 480 | source "$test_file" 481 | fi 482 | set +e 483 | run_test_suite 484 | ) 485 | failure=$(( $? || failure)) 486 | done 487 | 488 | if ((failure)) 489 | then 490 | notify_suites_failed 491 | else 492 | notify_suites_succeded 493 | fi 494 | 495 | exit $failure 496 | -------------------------------------------------------------------------------- /test/source/mined/attribution/10.csv: -------------------------------------------------------------------------------- 1 | 000c8aba74e0a452326ffd3591a8cbd4,10441cffac36ce3d4ca6af938f6bf5f0 2 | 0015b9cd03e8c8f70a30546b31348864,10ffcc4297a5f10a4147d518e84ea272 3 | 0056dc87a7a34955159ef77e16629dba,10e03952c97f225d4229d7524619eb7f 4 | 0063408dca20e06c0828d11829a0021a,10d60a0ddbb944de5b7040dbb419df70 5 | 0064ddefbd4ce7cc4b9b3f7775010e64,10ffcc4297a5f10a4147d518e84ea272 6 | 0072295606049db6a09d00918498db1b,10c25b9e4cf853ef114304fd45ecfab9 7 | 00779085e59795b872db276db574c6a9,10675e3a68c6f89eb0b41569c542ce91 8 | 00837c4575fc64e01a78e5d87f298440,10d1495ba46609e55e586ab71fa81f45 9 | 00838aaf468d27419653f9bb5e5748c7,10a8524df41cb07b69ae049421c6f9d5 10 | 0091692946f94de202322efebfca29db,1059dd60915ecde68f43807e66b5b4af 11 | -------------------------------------------------------------------------------- /test/source/mined/file/00.csv: -------------------------------------------------------------------------------- 1 | 000000000000000000000000000001,ca80efeae39f6de1dc102fb0fd9df8e1,test1/file/to/see/if/the/fisrt/record/is/imported0.test 2 | 00100000000000000000000000001,ba80efeae39f6de1dc102fb0fd9df8e2,test2/file/to/see/if/the/fisrt/record/is/imported2.test 3 | 100000000000000000000000000001,ba80efeae39f6de1dc102fb0fd9df8e1,key/to/be/removed/with/delete/keys 4 | befd3ab1fb69962aa00f9a23aef8f7,ba80efeae39f6de1dc102fb0fd9df8e3,test3/file/to/see/if/the/fisrt/record/is/imported3.test 5 | #This line and the second one must be skipped. (non askii char and incorrect key size) 6 | -------------------------------------------------------------------------------- /test/source/mined/file/39.csv: -------------------------------------------------------------------------------- 1 | 7754ba49e9e0cf4e7c190da78dda05,06e47378dedd2ca50e4c7257a54de9de,prettyCheckable-1.1/jquery-1.9.1.min.js 2 | -------------------------------------------------------------------------------- /test/source/mined/file/74.csv: -------------------------------------------------------------------------------- 1 | befd3ab1fb69962aa00f9a23aef8f6,ba80efeae39f6de1dc102fb0fd9df8e0,com/qvantel/jsonapi/client/akka/AkkaClient$$anon$1.class 2 | befd3ab1fb69962aa00f9a23aef8f7,ba80efeae39f6de1dc102fb0fd9df8e1,false/dup/entry/to/test/collate 3 | -------------------------------------------------------------------------------- /test/source/mined/file/85.csv: -------------------------------------------------------------------------------- 1 | 100000000000000000000000000001,ba80efeae39f6de1dc102fb0fd9df8e1,key/to/be/removed/with/delete/keys 2 | 2f517f6c65a25d4fa1f87092b893ff,ba80efeae39f6de1dc102fb0fd9df8e0,com/qvantel/jsonapi/client/akka/AkkaClient$.class 3 | -------------------------------------------------------------------------------- /test/source/mined/file/b3.csv: -------------------------------------------------------------------------------- 1 | 100000000000000000000000000001,ba80efeae39f6de1dc102fb0fd9df8e1,key/to/be/removed/with/delete/keys 2 | 37487d67c0307bb9d009e2b5e248c8,e7bc2c903379c1eda3ec7a12e88b933d,系列1-MyBatis源码解析/第10节课/mybatis-3-master/src/test/java/org/apache/ibatis/submitted/language/VelocitySqlSourceBuilder.java 3 | -------------------------------------------------------------------------------- /test/source/mined/license.csv: -------------------------------------------------------------------------------- 1 | c63c059270651b0149f5699b8e793ee2,0,Apache-2.0 2 | ff3069eee0efefc53a92f2726271162b,0,Apache-2.0 3 | -------------------------------------------------------------------------------- /test/source/mined/pivot/ba.csv: -------------------------------------------------------------------------------- 1 | 80efeae39f6de1dc102fb0fd9df8e0,74befd3ab1fb69962aa00f9a23aef8f6 2 | 80efeae39f6de1dc102fb0fd9df8e0,852f517f6c65a25d4fa1f87092b893ff 3 | 80efeae39f6de1dc102fb0fd9df8e0,8a51c06fbd50a41a35184c6b8a8d3e22 4 | 80efeae39f6de1dc102fb0fd9df8e0,8f2b281315bc8e923df7006b3d93e1cf 5 | 80efeae39f6de1dc102fb0fd9df8e0,b377b77401a8bfbef9ea316787ebda99 6 | -------------------------------------------------------------------------------- /test/source/mined/quality.csv: -------------------------------------------------------------------------------- 1 | b377b77401a8bfbef9ea316787ebda99,0,4 2 | -------------------------------------------------------------------------------- /test/source/mined/sources/8f2b.mz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scanoss/ldb/b2b2f350f24bcd27aa0b3a5dc93f17c8189b85e6/test/source/mined/sources/8f2b.mz -------------------------------------------------------------------------------- /test/source/mined/sources/b377.mz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scanoss/ldb/b2b2f350f24bcd27aa0b3a5dc93f17c8189b85e6/test/source/mined/sources/b377.mz -------------------------------------------------------------------------------- /test/source/mined/url.csv: -------------------------------------------------------------------------------- 1 | 00000000000000000000000000000001,test_vendor_00,test_component_00,1.1.1_test,2023-12-1,Apache-2.0,pkg:maven/com.thoughtworks.dsl/keywords-each_sjs0.6_2.12,https://repo1.maven.org/maven2/com/qvantel/jsonapi-scala-akka-client_2.12/6.0.1/jsonapi-scala-akka-client_2.12-6.0.1.jar 2 | 00000000000000000000000000000101,test_vendor_00,test_component_00,1.1.1_test,2023-12-1,Apache-2.0,pkg:maven/com.thoughtworks.dsl/keywords-each_sjs0.6_2.12,https://repo1.maven.org/maven2/com/qvantel/jsonapi-scala-akka-client_2.12/6.0.1/jsonapi-scala-akka-client_2.12-6.0.2.jar 3 | ba80efeae39f6de1dc102fb0fd9df8e0,com.thoughtworks.dsl,keywords-each_sjs0.6_2.12,1.5.5+1-7a249e3c,2021-12-10,Apache-2.0,pkg:maven/com.thoughtworks.dsl/keywords-each_sjs0.6_2.12,https://repo1.maven.org/maven2/com/qvantel/jsonapi-scala-akka-client_2.12/6.0.1/jsonapi-scala-akka-client_2.12-6.0.1.jar 4 | ba80efeae39f6de1dc102fb0fd9df8e1,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download1.zip 5 | ba80efeae39f6de1dc102fb0fd9df8e1,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download2.zip 6 | ba80efeae39f6de1dc102fb0fd9df8e1,test_vendor,test_component2,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download1.zip 7 | ca80efeae39f6de1dc102fb0fd9df8e2,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download3.zip 8 | -------------------------------------------------------------------------------- /test/source/mined/version.json: -------------------------------------------------------------------------------- 1 | {"monthly":"23.10", "daily":"23.10.23"} -------------------------------------------------------------------------------- /test/source/mined/wfp/05.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scanoss/ldb/b2b2f350f24bcd27aa0b3a5dc93f17c8189b85e6/test/source/mined/wfp/05.bin -------------------------------------------------------------------------------- /test/source/mined/wfp/20.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scanoss/ldb/b2b2f350f24bcd27aa0b3a5dc93f17c8189b85e6/test/source/mined/wfp/20.bin -------------------------------------------------------------------------------- /test/source/mined2/quality.csv: -------------------------------------------------------------------------------- 1 | b377b77401a8bfbef9ea316787ebda99,0,4 2 | -------------------------------------------------------------------------------- /test/source/mined2/version.json: -------------------------------------------------------------------------------- 1 | {"daily":"24.10.23"} -------------------------------------------------------------------------------- /test/source/mined3/quality.csv: -------------------------------------------------------------------------------- 1 | b377b77401a8bfbef9ea316787ebda99,0,4 2 | -------------------------------------------------------------------------------- /test/source/mined3/version.json: -------------------------------------------------------------------------------- 1 | {"daily":"25.10.23"} -------------------------------------------------------------------------------- /test/source/test_file_commands.txt: -------------------------------------------------------------------------------- 1 | dump test_kb/file hex 32 2 | dump test_kb/url hex 16 3 | dump test_kb/pivot hex 32 -------------------------------------------------------------------------------- /test/source/test_remove_record.txt: -------------------------------------------------------------------------------- 1 | ba80efeae39f6de1dc102fb0fd9df8e1,test_vendor,*,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download1.zip 2 | ca80efeae39f6de1dc102fb0fd9df8e2,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download3.zip -------------------------------------------------------------------------------- /test/test_kb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | test_01_kb_dir_exist() { 4 | dir="/var/lib/ldb/test_kb" 5 | assert "test -e $dir" 6 | } 7 | 8 | test_02_kb_content() { 9 | assert_equals "10" $(echo "dump test_kb/file hex 32" | ../ldb | wc -l) "file table fails" 10 | assert_equals "5" $(echo "dump keys from test_kb/url" | ../ldb | wc -l) "url table fails" 11 | assert_equals "5" $(echo "dump test_kb/pivot hex 32" |../ldb | wc -l) "pivot table fails" 12 | assert_equals "19" $(echo "dump test_kb/wfp hex -1" |../ldb | wc -l) "wfp table fails" 13 | assert_equals "2" $(echo "dump keys from test_kb/sources" |../ldb | wc -l) "sources table fails" 14 | } 15 | 16 | test_03_kb_query() { 17 | result=$(echo "select from test_kb/file key 00000000000000000000000000000001 csv hex 32" | ../ldb) 18 | assert_equals "00000000000000000000000000000001,ca80efeae39f6de1dc102fb0fd9df8e1,test1/file/to/see/if/the/fisrt/record/is/imported0.test" $result "file table query fails" 19 | result=$(echo "select from test_kb/url key ca80efeae39f6de1dc102fb0fd9df8e2 csv hex 16" | ../ldb) 20 | assert_equals "ca80efeae39f6de1dc102fb0fd9df8e2,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download3.zip" $result "url table quert fails" 21 | result=$(echo "select from test_kb/sources key b377b77401a8bfbef9ea316787ebda99 csv hex 16" | ../ldb | tr -cd '[:print:]') 22 | comp="Manifest-Version: 1.0Implementation-Title: jsonapi-scala-akka-clientImplementation-Version: 6.0.1Specification-Vendor: com.qvantelSpecification-Title: jsonapi-scala-akka-clientImplementation-Vendor-Id: com.qvantelSpecification-Version: 6.0.1Implementation-URL: https://github.com/qvantel/jsonapi-scalaImplementation-Vendor: com.qvantel" 23 | assert_equals "$comp" "$result" 24 | } 25 | 26 | test_04_kb_delete_record() { 27 | echo "delete from test_kb/url records from source/test_remove_record.txt" | ../ldb -q 28 | result=$(echo "select from test_kb/url key ba80efeae39f6de1dc102fb0fd9df8e1 csv hex 16" | ../ldb -q) 29 | assert_equals "ba80efeae39f6de1dc102fb0fd9df8e1,test_vendor,test_component,1.0.0_test,2023-11-03,my_license,pkg:test_vendor/test_component,https://testurl.com/download2.zip" $result 30 | result=$(echo "select from test_kb/url key ca80efeae39f6de1dc102fb0fd9df8e2 csv hex 16" | ../ldb -q) 31 | assert_equals "" $result 32 | echo "delete from test_kb/file record 397754ba49e9e0cf4e7c190da78dda05,06e47378dedd2ca50e4c7257a54de9de,prettyCheckable-1.1/jquery-1.9.1.min.js" | ../ldb -q 33 | result=$(echo "select from test_kb/file key 397754ba49e9e0cf4e7c190da78dda05 csv hex 32" | ../ldb -q) 34 | assert_equals "" $result 35 | } 36 | 37 | test_05_kb_delete_keys() { 38 | echo "delete from test_kb/file max 2048 keys 00100000000000000000000000000001,85100000000000000000000000000001,b3100000000000000000000000000001" | ../ldb -q 39 | result=$(echo "select from test_kb/file key 00100000000000000000000000000001 csv hex 32" | ../ldb) 40 | result+=$(echo "select from test_kb/file key 85100000000000000000000000000001 csv hex 32" | ../ldb) 41 | result+=$(echo "select from test_kb/file key b3100000000000000000000000000001 csv hex 32" | ../ldb) 42 | assert_equals "" $result 43 | echo "delete from test_kb/sources max 2048 keys b377b77401a8bfbef9ea316787ebda99,8f2b281315bc8e923df7006b3d93e1cf" | ../ldb -q 44 | result=$(echo "select from test_kb/sources key b377b77401a8bfbef9ea316787ebda99 csv hex 16" | ../ldb) 45 | result+=$(echo "select from test_kb/sources key 8f2b281315bc8e923df7006b3d93e1cf csv hex 16" | ../ldb) 46 | assert_equals "" $result 47 | } 48 | 49 | #import again, but with the collate option, so the duplicated will be removed. Then test again the content. 50 | test_06_kb_collate() { 51 | ../ldb -q -u source/mined -n test_kb -c 52 | test_02_kb_content 53 | } 54 | 55 | test_07_kb_query_from_file() { 56 | result=$(../ldb -f source/test_file_commands.txt | wc -l) 57 | assert_equals "22" $result 58 | } 59 | 60 | test_08_unicode_chars() { 61 | result=$(echo "select from test_kb/file key b337487d67c0307bb9d009e2b5e248c8 csv hex 32" | ../ldb) 62 | assert_equals "b337487d67c0307bb9d009e2b5e248c8,e7bc2c903379c1eda3ec7a12e88b933d,系列1-MyBatis源码解析/第10节课/mybatis-3-master/src/test/java/org/apache/ibatis/submitted/language/VelocitySqlSourceBuilder.java" $result 63 | } 64 | 65 | test_09_version_validation() { 66 | ../ldb -q -u source/mined2 -n test_kb -q 67 | result=$(cat /var/lib/ldb/test_kb/version.json) 68 | assert_equals "{\"monthly\":\"23.10\", \"daily\":\"24.10.23\"}" "$result" 69 | echo "bulk insert test_kb from source/mined3 with (FILE_DEL=0,VALIDATE_VERSION=0)" | ../ldb 70 | result=$(cat /var/lib/ldb/test_kb/version.json) 71 | assert_equals "{\"monthly\":\"23.10\", \"daily\":\"25.10.23\"}" "$result" 72 | } 73 | 74 | test_10_configuration() { 75 | config="GLOBAL: (KEYS=-1, VALIDATE_FIELDS=1, FIELDS=1, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=20, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/tmp)" 76 | sed -i "1s|.*|$config|" /usr/local/etc/scanoss/ldb/test_kb.conf 77 | 78 | ../ldb -q -u source/mined2 -n test_kb -q 79 | result=$(tac /var/log/scanoss/ldb/test_kb.log | grep -m 1 "GLOBAL configuration:") 80 | config="GLOBAL configuration: (KEYS=-1, VALIDATE_FIELDS=1, FIELDS=1, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=20, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/tmp)" 81 | assert_equals "$config" "$result" 82 | 83 | echo "bulk insert test_kb/file from source/mined/file with (TMP_PATH=/var,FILE_DEL=0,THREADS=30)" | ../ldb -q 84 | config="file configuration: (KEYS=2, VALIDATE_FIELDS=1, FIELDS=3, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=30, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/var)" 85 | result=$(tac /var/log/scanoss/ldb/test_kb.log | grep -m 1 "file configuration:") 86 | assert_equals "$config" "$result" 87 | 88 | config="GLOBAL: (COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, VALIDATE_VERSION=1, TMP_PATH=/tmp)" 89 | cores=$(nproc) 90 | threads=$((cores / 2)) 91 | sed -i "1s|.*|$config|" /usr/local/etc/scanoss/ldb/test_kb.conf 92 | 93 | echo "bulk insert test_kb/file from source/mined/file with (TMP_PATH=/var,FILE_DEL=0)" | ../ldb -q 94 | config="file configuration: (KEYS=2, VALIDATE_FIELDS=1, FIELDS=3, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=$threads, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/var)" 95 | result=$(tac /var/log/scanoss/ldb/test_kb.log | grep -m 1 "file configuration:") 96 | assert_equals "$config" "$result" 97 | 98 | echo "bulk insert test_kb from source/mined with (TMP_PATH=/var,FILE_DEL=0,VALIDATE_VERSION=0)" | ../ldb -q 99 | config="file configuration: (KEYS=2, VALIDATE_FIELDS=1, FIELDS=3, VALIDATE_VERSION=0, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=$threads, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/var)" 100 | result=$(tac /var/log/scanoss/ldb/test_kb.log | grep -m 1 "file configuration:") 101 | assert_equals "$config" "$result" 102 | } 103 | 104 | setup_suite () { 105 | ../ldb -u source/mined -n test_kb 106 | } 107 | teardown_suite () { 108 | rm -rf /var/lib/ldb/test_kb 109 | rm -rf /usr/local/etc/scanoss/ldb/test_kb.conf 110 | } -------------------------------------------------------------------------------- /version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ### 3 | # SPDX-License-Identifier: GPL-2.0-or-later 4 | # 5 | # Copyright (C) 2018-2023 SCANOSS.COM 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | ### 18 | # 19 | # Determine the latest tag associated with this repo and echo to stdout 20 | # 21 | version=$(git describe --tags --abbrev=0) 22 | if [[ -z "$version" ]] ; then 23 | version=$(git describe --tags "$(git rev-list --tags --max-count=1)") 24 | fi 25 | if [[ -z "$version" ]] ; then 26 | echo "Error: Failed to determine a valid version number" >&2 27 | exit 1 28 | fi 29 | echo "$version" | sed 's/^v//' 30 | exit 0 --------------------------------------------------------------------------------