├── cblite ├── cblite.def ├── cbliteTool+sql.cc ├── CMakeLists.txt ├── cbliteTool+encrypt.cc ├── cbliteTool+file.cc ├── cbliteTool+put.cc └── cbliteTool+revs.cc ├── misc ├── cloc_cbllog.txt └── cbllog_files.txt ├── .gitignore ├── gen_errors ├── .gitignore ├── java.py ├── README.md ├── android.py ├── gen_errors.py ├── csharp.py ├── objc.py └── cbl-errors.json ├── LargeDatasetGenerator ├── images │ ├── relationship.png │ └── relationship.puml ├── LargeDatasetGenerator.Abstractions │ ├── LargeDatasetGenerator.Abstractions.csproj │ ├── IDataGenerator.cs │ └── IJsonOutput.cs ├── LargeDatasetGenerator.Core │ ├── LargeDatasetGenerator.Core.csproj │ ├── Generator │ │ ├── BoolGenerator.cs │ │ ├── GuidGenerator.cs │ │ ├── Extensions.cs │ │ ├── ThreadSafeRandom.cs │ │ ├── DataGenerator.cs │ │ ├── RepeatGeneratorArgs.cs │ │ ├── PluginLoader.cs │ │ ├── GeneratorArgumentParser.cs │ │ ├── DateObjectArgs.cs │ │ ├── IntegerGenerator.cs │ │ ├── LoremGenerator.cs │ │ ├── GaussGenerator.cs │ │ ├── RandomGenerator.cs │ │ ├── RangeGenerator.cs │ │ └── FloatingGenerator.cs │ ├── Outputs │ │ └── JsonFileOutput.cs │ ├── PositionalConsole.cs │ └── Data │ │ └── LoremIpsum.cs ├── LargeDatasetGenerator.SyncGateway │ ├── LargeDatasetGenerator.SyncGateway.csproj │ ├── UrlValidation.cs │ └── SyncGatewayOutput.cs ├── LargeDatasetGenerator.CouchbaseServer │ ├── LargeDatasetGenerator.CouchbaseServer.csproj │ ├── UrlValidation.cs │ └── CouchbaseServerOutput.cs ├── LargeDatasetGenerator │ ├── example.json │ ├── LargeDatasetGenerator.csproj │ ├── Program_args.cs │ └── Program.cs ├── LargeDatasetGenerator.CouchbaseLite │ ├── LargeDatasetGenerator.CouchbaseLite.csproj │ └── CouchbaseLiteOutput.cs ├── README.md ├── README_generator.md ├── README_output.md └── LargeDatasetGenerator.sln ├── ci ├── cbl-log │ ├── packages │ │ └── fpm │ │ │ ├── cbl-log-post-remove.sh │ │ │ └── Makefile │ ├── ci-build.ps1 │ ├── ci-build.sh │ └── CMakeLists.txt └── cblite │ ├── ci-build.ps1 │ ├── ci-build.sh │ └── CMakeLists.txt ├── Xcode ├── Tools.xcodeproj │ ├── project.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ │ └── IDEWorkspaceChecks.plist │ └── xcshareddata │ │ └── xcschemes │ │ ├── cblite.xcscheme │ │ ├── cbl-log.xcscheme │ │ └── Tests.xcscheme └── xcconfigs │ ├── Project_Debug.xcconfig │ ├── Project_Release.xcconfig │ ├── Project_Debug_EE.xcconfig │ ├── Project_Release_EE.xcconfig │ ├── Project.xcconfig │ ├── linenoise.xcconfig │ ├── cblite.xcconfig │ ├── cbl-log.xcconfig │ └── tests.xcconfig ├── config.h.in ├── .gitmodules ├── cbl-log ├── LogDecoder_stub.cpp ├── CMakeLists.txt └── cbl-log.cc ├── README.cbl-log.md ├── tests ├── tests_main.cc └── LibC++Debug.cc ├── litecp ├── JSONEndpoint.hh ├── DirEndpoint.hh ├── RemoteEndpoint.hh ├── RemoteEndpoint.cc ├── Endpoint.cc ├── JSONEndpoint.cc ├── DBEndpoint.hh ├── DirEndpoint.cc └── Endpoint.hh ├── ArgumentTokenizer.hh ├── appveyor.yml ├── README.md ├── BUILDING.md ├── README.cblite.md ├── ArgumentTokenizer.cc ├── cmake └── common_setup.cmake └── Tool.cc /cblite/cblite.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | c4query_translateN1QL -------------------------------------------------------------------------------- /misc/cloc_cbllog.txt: -------------------------------------------------------------------------------- 1 | --list-file=misc/cbllog_files.txt --include-lang="C,C++" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | output/ 3 | bin/ 4 | obj/ 5 | launchSettings.json 6 | *.user 7 | .vs/ -------------------------------------------------------------------------------- /gen_errors/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Intellij stuff 3 | .idea 4 | *.iml 5 | 6 | # Python stuff 7 | *.pyc 8 | __pycache__ 9 | 10 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/images/relationship.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDDStudio/couchbase-mobile-tools/master/LargeDatasetGenerator/images/relationship.png -------------------------------------------------------------------------------- /ci/cbl-log/packages/fpm/cbl-log-post-remove.sh: -------------------------------------------------------------------------------- 1 | # Remove cbl-log directory 2 | if [ -d "/opt/cbl-log" ]; then 3 | rm -rf /opt/cbl-log > /dev/null 2>&1 4 | fi 5 | exit 0 6 | -------------------------------------------------------------------------------- /Xcode/Tools.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /misc/cbllog_files.txt: -------------------------------------------------------------------------------- 1 | Tool.cc 2 | ArgumentTokenizer.cc 3 | cbl-log/cbl-log.cc 4 | vendor/couchbase-lite-core/LiteCore/Support/LogDecoder.cc 5 | vendor/couchbase-lite-core/LiteCore/Support/LogEncoder.cc 6 | vendor/couchbase-lite-core/LiteCore/tests/LogEncoderTest.cc 7 | -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define TOOLS_VERSION_MAJOR @TOOLS_VERSION_MAJOR@ 4 | #define TOOLS_VERSION_MINOR @TOOLS_VERSION_MINOR@ 5 | #define TOOLS_VERSION @TOOLS_VERSION_MAJOR@.@TOOLS_VERSION_MINOR@.@TOOLS_VERSION_PATCH@ 6 | #define TOOLS_VERSION_STRING "@TOOLS_VERSION_STRING@" -------------------------------------------------------------------------------- /Xcode/xcconfigs/Project_Debug.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // Project_Debug.xcconfig 3 | // CBL_C 4 | // 5 | // Created by Jens Alfke on 12/28/18. 6 | // Copyright © 2018 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "Project.xcconfig" 10 | #include "../vendor/couchbase-lite-core/Xcode/xcconfigs/Project_Debug.xcconfig" 11 | 12 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/Project_Release.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // Project.xcconfig 3 | // CBL_C 4 | // 5 | // Created by Jens Alfke on 12/28/18. 6 | // Copyright © 2018 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "Project.xcconfig" 10 | #include "../vendor/couchbase-lite-core/Xcode/xcconfigs/Project_Release.xcconfig" 11 | 12 | -------------------------------------------------------------------------------- /Xcode/Tools.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/Project_Debug_EE.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // Project_Debug_EE.xcconfig 3 | // Tools 4 | // 5 | // Created by Jens Alfke on 5/3/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "Project_Debug.xcconfig" 10 | 11 | GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS) COUCHBASE_ENTERPRISE 12 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/Project_Release_EE.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // Project_Release_EE.xcconfig 3 | // Tools 4 | // 5 | // Created by Jens Alfke on 5/3/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "Project_Release.xcconfig" 10 | 11 | GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS) COUCHBASE_ENTERPRISE 12 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/linenoise-ng"] 2 | path = vendor/linenoise-ng 3 | url = https://github.com/arangodb/linenoise-ng 4 | [submodule "vendor/couchbase-lite-core"] 5 | path = vendor/couchbase-lite-core 6 | url = https://github.com/couchbase/couchbase-lite-core 7 | [submodule "vendor/peg"] 8 | path = vendor/peg 9 | url = https://github.com/snej/peg.git 10 | -------------------------------------------------------------------------------- /cbl-log/LogDecoder_stub.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // LogDecoder_stub.cpp 3 | // 4 | // Copyright © 2019 Couchbase. All rights reserved. 5 | // 6 | 7 | // Used for Xcode builds only. Since LogDecoder is inside the LiteCore subproject, Xcode won't 8 | // let us add it to the Tools project. So instead, add a new source file that simply includes it. 9 | 10 | #include "LogDecoder.cc" 11 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/Project.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // Project.xcconfig 3 | // CBL_C 4 | // 5 | // Created by Jens Alfke on 12/28/18. 6 | // Copyright © 2018 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "../vendor/couchbase-lite-core/Xcode/xcconfigs/Project.xcconfig" 10 | 11 | LITECORE = $(SRCROOT)/../vendor/couchbase-lite-core 12 | FLEECE = $(LITECORE)/vendor/fleece 13 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Abstractions/LargeDatasetGenerator.Abstractions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/linenoise.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // linenoise.xcconfig 3 | // cblite 4 | // 5 | // Created by Jens Alfke on 2/12/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | CLANG_WARN_DOCUMENTATION_COMMENTS = NO 10 | GCC_WARN_64_TO_32_BIT_CONVERSION = NO 11 | GCC_WARN_ABOUT_MISSING_PROTOTYPES = NO 12 | CLANG_WARN_SUSPICIOUS_IMPLICIT_CONVERSION = NO 13 | GCC_WARN_UNINITIALIZED_AUTOS = NO 14 | CLANG_WARN_UNREACHABLE_CODE = NO 15 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/cblite.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // cblite.xcconfig 3 | // cblite 4 | // 5 | // Created by Jens Alfke on 2/12/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | SDKROOT = macosx 10 | SUPPORTED_PLATFORMS = macosx 11 | 12 | HEADER_SEARCH_PATHS = $(inherited) $(FLEECE)/API $(inherited) $(FLEECE)/Fleece/Support $(LITECORE)/C/include $(LITECORE)/C $(LITECORE)/Replicator $(LITECORE)/LiteCore/Support 13 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/cbl-log.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // cbl-log.xcconfig 3 | // Tools 4 | // 5 | // Created by Jens Alfke on 4/23/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | SDKROOT = macosx 10 | SUPPORTED_PLATFORMS = macosx 11 | 12 | GCC_PREPROCESSOR_DEFINITIONS = $(inherited) CBLTOOL_NO_C_API 13 | 14 | HEADER_SEARCH_PATHS = $(inherited) $(FLEECE)/API $(inherited) $(FLEECE)/Fleece/Support $(LITECORE)/LiteCore/Support 15 | -------------------------------------------------------------------------------- /Xcode/xcconfigs/tests.xcconfig: -------------------------------------------------------------------------------- 1 | // 2 | // tests.xcconfig 3 | // Tools 4 | // 5 | // Created by Jens Alfke on 4/11/19. 6 | // Copyright © 2019 Couchbase. All rights reserved. 7 | // 8 | 9 | #include "../vendor/couchbase-lite-core/Xcode/xcconfigs/Tests.xcconfig" 10 | #include "cblite.xcconfig" 11 | 12 | ONLY_ACTIVE_ARCH = YES 13 | LLVM_LTO = NO // LTO makes tests very slow to link 14 | 15 | HEADER_SEARCH_PATHS = $(HEADER_SEARCH_PATHS) $(FLEECE)/vendor/catch/ 16 | -------------------------------------------------------------------------------- /README.cbl-log.md: -------------------------------------------------------------------------------- 1 | # The `cbl-log` Tool 2 | 3 | `cbl-log` is a command-line tool for decoding logs generated by LiteCore (or Couchbase Lite). 4 | 5 | For build instructions, see [BUILDING.md](BUILDING.md). 6 | 7 | ## Features 8 | 9 | | Command | Purpose | 10 | |----------------|---------| 11 | | `cbl-log help` | Display help text | 12 | | `cbl-log logcat` | Converts a binary log file to text and writes it to stdout or the given output path | 13 | 14 | 15 | 16 | [This script](https://gist.github.com/pkjvit/ef84a88b6a61b01291638f1c815f4af3) can convert multiple cbllog files at once. 17 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/LargeDatasetGenerator.Core.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | latest 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.SyncGateway/LargeDatasetGenerator.SyncGateway.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.CouchbaseServer/LargeDatasetGenerator.CouchbaseServer.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /ci/cblite/ci-build.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .Description 3 | Builds the cbl-log tool for Windows only on Couchbase build servers (It will technically work for Mac and Linux if the directory 4 | separator characters are flipped from backslash to slash) 5 | #> 6 | $CMakePath = "C:\Program Files\CMake\bin\cmake.exe" 7 | 8 | New-Item -ItemType Directory -Path "$PSScriptRoot\build" -ErrorAction Ignore 9 | Push-Location "$PSScriptRoot\build" 10 | 11 | & "$CMakePath" "$PSScriptRoot\..\..\cblite" 12 | & "$CMakePath" --build . --target cblite --config RelWithDebInfo 13 | & "$CMakePath" --build . --target cblitetest --config RelWithDebInfo 14 | 15 | Pop-Location 16 | -------------------------------------------------------------------------------- /ci/cbl-log/ci-build.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .Description 3 | Builds the cbl-log tool for Windows only on Couchbase build servers (It will technically work for Mac and Linux if the directory 4 | separator characters are flipped from backslash to slash) 5 | #> 6 | $CMakePath = "C:\Program Files\CMake\bin\cmake.exe" 7 | 8 | New-Item -ItemType Directory -Path "$PSScriptRoot\build" -ErrorAction Ignore 9 | Push-Location "$PSScriptRoot\build" 10 | 11 | & "$CMakePath" "$PSScriptRoot\..\..\cbl-log" 12 | & "$CMakePath" --build . --target cbl-log --config RelWithDebInfo 13 | & "$CMakePath" --build . --target cbl-logtest --config RelWithDebInfo 14 | 15 | Pop-Location 16 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.SyncGateway/UrlValidation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel.DataAnnotations; 3 | using McMaster.Extensions.CommandLineUtils; 4 | using McMaster.Extensions.CommandLineUtils.Validation; 5 | 6 | namespace LargeDatasetGenerator.SyncGateway 7 | { 8 | internal sealed class UrlValidation : IOptionValidator 9 | { 10 | public ValidationResult GetValidationResult(CommandOption option, ValidationContext context) 11 | { 12 | return Uri.TryCreate(option.Value(), UriKind.Absolute, out var uri) 13 | ? ValidationResult.Success 14 | : new ValidationResult("--url must be a valid URL"); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /ci/cblite/ci-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TOP="$( cd "$(dirname "$0")" ; pwd -P )/../.." 4 | pushd $TOP 5 | 6 | CMAKE_DIRECTORY=$1 7 | if [[ -z $CMAKE_DIRECTORY ]]; then 8 | echo "Error: No CMakeLists.txt directory specified, aborting..." 9 | exit 1 10 | fi 11 | 12 | mkdir -p ci/cblite/build 13 | pushd ci/cblite/build 14 | cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_DIRECTORY 15 | make -j8 cblite 16 | make -j8 cblitetest 17 | 18 | make install 19 | INSTALL_PREFIX=`cat CMakeCache.txt| grep CMAKE_INSTALL_PREFIX | cut -f 2 -d '='` 20 | 21 | popd 22 | 23 | if [[ ! -d $TOP/install ]]; then 24 | mkdir -p $TOP/install 25 | fi 26 | 27 | pushd $INSTALL_PREFIX/lib 28 | echo $INSTALL_PREFIX/lib 29 | rm -rf pkgconfig/ 30 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.CouchbaseServer/UrlValidation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel.DataAnnotations; 3 | using McMaster.Extensions.CommandLineUtils; 4 | using McMaster.Extensions.CommandLineUtils.Validation; 5 | 6 | namespace LargeDatasetGenerator.CouchbaseServer 7 | { 8 | internal sealed class UrlValidation : IOptionValidator 9 | { 10 | public ValidationResult GetValidationResult(CommandOption option, ValidationContext context) 11 | { 12 | return Uri.TryCreate(option.Value(), UriKind.Absolute, out var uri) 13 | ? ValidationResult.Success 14 | : new ValidationResult("--url must be a valid URL"); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /ci/cbl-log/ci-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TOP="$( cd "$(dirname "$0")" ; pwd -P )/../.." 4 | pushd $TOP 5 | 6 | CMAKE_DIRECTORY=$1 7 | if [[ -z $CMAKE_DIRECTORY ]]; then 8 | echo "Error: No CMakeLists.txt directory specified, aborting..." 9 | exit 1 10 | fi 11 | 12 | mkdir -p ci/cbl-log/build 13 | pushd ci/cbl-log/build 14 | cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_DIRECTORY 15 | make -j8 cbl-log 16 | make -j8 cbl-logtest 17 | 18 | make install 19 | INSTALL_PREFIX=`cat CMakeCache.txt| grep CMAKE_INSTALL_PREFIX | cut -f 2 -d '='` 20 | 21 | popd 22 | 23 | if [[ ! -d $TOP/install ]]; then 24 | mkdir -p $TOP/install 25 | fi 26 | 27 | pushd $INSTALL_PREFIX/lib 28 | echo $INSTALL_PREFIX/lib 29 | rm -rf libicu* pkgconfig/ icu/ 30 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id": "{{guid()}}", 3 | "traits": { 4 | "eye_color": "{{random('blue', 'brown', 'green', 'hazel', 'gray'}}", 5 | "hair_color": "{{random('blonde', 'brown', 'auburn', 'red', 'gray'}}", 6 | "age": "{{integer(18,65)}}", 7 | "has_driver_license": "{{bool()}}", 8 | "gpa": "{{floating(2.5, 4.0)}}" 9 | }, 10 | "children": [ 11 | { 12 | "repeat(0, 4)": { 13 | "name": "{{lorem(3, words}}", 14 | "birthday": "{{date(new Date(1974, 1, 1), new Date(2018, 12, 31))}}" 15 | } 16 | } 17 | ], 18 | "address" : { 19 | "line1" : "{{integer(100,999)}} {{lorem(1, word)}} St." 20 | }, 21 | "range": "{{range(1, 10)}}", 22 | "description": "{{lorem(3, paragraphs)}}", 23 | "job_performance": "{{gauss()}}", 24 | "constant_answer": 42 25 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/BoolGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace LargeDatasetGenerator.Generator 4 | { 5 | /// 6 | /// A generator that will randomly generate true or false 7 | /// 8 | public sealed class BoolGenerator : IDataGenerator 9 | { 10 | #region Properties 11 | 12 | /// 13 | public string Description { get; } = "Randomly generates true or false values"; 14 | 15 | /// 16 | public string Signature { get; } = "{{bool()}}"; 17 | 18 | #endregion 19 | 20 | #region IDataGenerator 21 | 22 | /// 23 | public Task GenerateValueAsync() 24 | { 25 | return Task.FromResult(ThreadSafeRandom.NextDouble() >= 0.5); 26 | } 27 | 28 | #endregion 29 | } 30 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/GuidGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace LargeDatasetGenerator.Generator 5 | { 6 | /// 7 | /// A generator that creates GUID objects and returns their string representation 8 | /// 9 | public sealed class GuidGenerator : IDataGenerator 10 | { 11 | #region Properties 12 | 13 | /// 14 | public string Description { get; } = "Creates a GUID object as a string"; 15 | 16 | /// 17 | public string Signature { get; } = "{{guid()}}"; 18 | 19 | #endregion 20 | 21 | #region IDataGenerator 22 | 23 | /// 24 | public Task GenerateValueAsync() 25 | { 26 | return Task.FromResult(Guid.NewGuid().ToString()); 27 | } 28 | 29 | #endregion 30 | } 31 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.CouchbaseLite/LargeDatasetGenerator.CouchbaseLite.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.1 5 | 6 | 7 | 8 | 2 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/Extensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Threading.Tasks; 5 | 6 | namespace LargeDatasetGenerator.Extensions 7 | { 8 | internal static class Extensions 9 | { 10 | #region Public Methods 11 | 12 | public static long NextInt64(this Random r, long min, long max) 13 | { 14 | if (max <= Int32.MaxValue && min >= Int32.MinValue) { 15 | return r.Next((int) min, (int) max); 16 | } 17 | 18 | var ret = (decimal) r.NextDouble(); 19 | ret *= (max - min); 20 | ret += min; 21 | return (long) ret; 22 | } 23 | 24 | public static void AddRange(this IList collection, IEnumerable range) 25 | { 26 | foreach (var item in range) { 27 | collection.Add(item); 28 | } 29 | } 30 | 31 | #endregion 32 | } 33 | } -------------------------------------------------------------------------------- /tests/tests_main.cc: -------------------------------------------------------------------------------- 1 | // 2 | // tests_main.cc 3 | // 4 | // Copyright © 2019 Couchbase. All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #define CATCH_CONFIG_MAIN 20 | #include "catch.hpp" 21 | #include "CaseListReporter.hh" 22 | 23 | extern bool gC4ExpectingExceptions; 24 | bool gC4ExpectingExceptions; 25 | 26 | bool C4ExpectingExceptions(); 27 | 28 | bool C4ExpectingExceptions() { 29 | return gC4ExpectingExceptions; 30 | } 31 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/ThreadSafeRandom.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | using LargeDatasetGenerator.Extensions; 4 | 5 | namespace LargeDatasetGenerator.Generator 6 | { 7 | internal static class ThreadSafeRandom 8 | { 9 | #region Constants 10 | 11 | private static readonly object Locker = new object(); 12 | private static readonly Random Rng = new Random(); 13 | 14 | #endregion 15 | 16 | #region Public Methods 17 | 18 | public static int Next(int min, int max) 19 | { 20 | lock (Locker) { 21 | return Rng.Next(min, max); 22 | } 23 | } 24 | 25 | public static double NextDouble() 26 | { 27 | lock (Locker) { 28 | return Rng.NextDouble(); 29 | } 30 | } 31 | 32 | public static long NextInt64(long min, long max) 33 | { 34 | lock (Locker) { 35 | return Rng.NextInt64(min, max); 36 | } 37 | } 38 | 39 | #endregion 40 | } 41 | } -------------------------------------------------------------------------------- /ci/cbl-log/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED (VERSION 3.1) 2 | CMAKE_POLICY (VERSION 3.1) 3 | 4 | # Tell CMake to use headers / frameworks from SDK inside XCode instead of 5 | # the ones found on the system (for weak linking). Ignored on non-Apple 6 | SET(CMAKE_OSX_SYSROOT macosx) 7 | 8 | # Top-level CMakeLists for Couchbase Lite Core 9 | PROJECT (couchbase-lite-log-build) 10 | 11 | # Provide reasonable default for CMAKE_INSTALL_PREFIX 12 | IF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) 13 | SET(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install" CACHE STRING 14 | "The install location" FORCE) 15 | LIST(APPEND CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}") 16 | ENDIF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) 17 | 18 | # Configure tlm the way it wants to be configured 19 | SET (CB_DOWNLOAD_DEPS 1) 20 | SET (CB_DOWNLOAD_DEPS_DEFAULT_MANIFEST 21 | "${PROJECT_SOURCE_DIR}/cbbuild/scripts/jenkins/lite-core/manifest.cmake") 22 | LIST (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/tlm/cmake/Modules") 23 | ADD_SUBDIRECTORY (tlm/deps) 24 | 25 | ADD_SUBDIRECTORY (couchbase-mobile-tools/cbl-log) 26 | -------------------------------------------------------------------------------- /ci/cblite/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED (VERSION 3.1) 2 | CMAKE_POLICY (VERSION 3.1) 3 | 4 | # Tell CMake to use headers / frameworks from SDK inside XCode instead of 5 | # the ones found on the system (for weak linking). Ignored on non-Apple 6 | SET(CMAKE_OSX_SYSROOT macosx) 7 | 8 | # Top-level CMakeLists for Couchbase Lite Core 9 | PROJECT (couchbase-lite-cli-build) 10 | 11 | # Provide reasonable default for CMAKE_INSTALL_PREFIX 12 | IF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) 13 | SET(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install" CACHE STRING 14 | "The install location" FORCE) 15 | LIST(APPEND CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}") 16 | ENDIF (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) 17 | 18 | # Configure tlm the way it wants to be configured 19 | SET (CB_DOWNLOAD_DEPS 1) 20 | SET (CB_DOWNLOAD_DEPS_DEFAULT_MANIFEST 21 | "${PROJECT_SOURCE_DIR}/cbbuild/scripts/jenkins/lite-core/manifest.cmake") 22 | LIST (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/tlm/cmake/Modules") 23 | ADD_SUBDIRECTORY (tlm/deps) 24 | 25 | ADD_SUBDIRECTORY (couchbase-mobile-tools/cblite) 26 | -------------------------------------------------------------------------------- /tests/LibC++Debug.cc: -------------------------------------------------------------------------------- 1 | // 2 | // LibC++Debug.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifdef __APPLE__ 20 | #ifdef _LIBCPP_DEBUG 21 | 22 | #include <__debug> 23 | 24 | namespace std { 25 | // Resolves a link error building with libc++ in debug mode. Apparently this symbol would be in 26 | // the debug version of libc++.dylib, but we don't have that on Apple platforms. 27 | __1::__libcpp_debug_function_type __1::__libcpp_debug_function; 28 | } 29 | 30 | #endif 31 | #endif 32 | -------------------------------------------------------------------------------- /litecp/JSONEndpoint.hh: -------------------------------------------------------------------------------- 1 | // 2 | // JSONEndpoint.hh 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include "Endpoint.hh" 21 | #include "FilePath.hh" 22 | #include 23 | 24 | 25 | class JSONEndpoint : public Endpoint { 26 | public: 27 | JSONEndpoint(const string &spec) 28 | :Endpoint(spec) 29 | { } 30 | 31 | virtual void prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint*) override; 32 | virtual void copyTo(Endpoint*, uint64_t limit) override; 33 | virtual void writeJSON(slice docID, slice json) override; 34 | 35 | private: 36 | unique_ptr _in; 37 | unique_ptr _out; 38 | }; 39 | -------------------------------------------------------------------------------- /gen_errors/java.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Couchbase, Inc All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | 18 | 19 | class JavaFormatter: 20 | """Formatter for pure Java properties 21 | Errors are formatted as a properties file to be read into a properties map""" 22 | name = "Java" 23 | 24 | msg_file_name = "errors.properties" 25 | message_def = "{0} = {1}\n" 26 | 27 | def __init__(self, out_dir): 28 | self.out_file = os.path.join(out_dir, JavaFormatter.msg_file_name) 29 | 30 | def format(self, errors): 31 | with open(self.out_file, "w") as message_file: 32 | for error in errors: 33 | message_file.write(JavaFormatter.message_def.format(error["name"], error["message"])) 34 | -------------------------------------------------------------------------------- /litecp/DirEndpoint.hh: -------------------------------------------------------------------------------- 1 | // 2 | // DirEndpoint.hh 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include "Endpoint.hh" 21 | #include "FilePath.hh" 22 | #include 23 | 24 | 25 | class DirectoryEndpoint : public Endpoint { 26 | public: 27 | DirectoryEndpoint(const string &spec) 28 | :Endpoint(spec) 29 | ,_dir(spec, "") 30 | { } 31 | 32 | virtual void prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint*) override; 33 | virtual void copyTo(Endpoint*, uint64_t limit) override; 34 | virtual void writeJSON(slice docID, slice json) override; 35 | 36 | private: 37 | slice readFile(const string &path, alloc_slice &buffer); 38 | 39 | FilePath _dir; 40 | }; 41 | 42 | -------------------------------------------------------------------------------- /ci/cbl-log/packages/fpm/Makefile: -------------------------------------------------------------------------------- 1 | # cbl-log packaging 2 | 3 | ifeq ($(strip $(VERSION)),) 4 | REL_VERSION = 1.0.0 5 | REVISION = 9999 6 | else 7 | REL_VERSION = $(VERSION) 8 | REVISION = $(BLD_NUM) 9 | endif 10 | 11 | NAME=cbl-log 12 | VENDOR='Couchbase Inc.' 13 | INSTALL_PREFIX=/opt/cbl-log 14 | PACKAGE_DESCRIPTION='The cbl-log Tool' 15 | URL='https://github.com/couchbaselabs/couchbase-mobile-tools/blob/master/README.cbl-log.md' 16 | POST_UNINSTALL_SCRIPT=cbl-log-post-remove.sh 17 | 18 | EXCLUDE_FILES='*logtest' 19 | FPM_COMMON_OPTIONS=fpm -s dir \ 20 | -n $(NAME) \ 21 | -v $(REL_VERSION) \ 22 | --prefix $(INSTALL_PREFIX) \ 23 | --chdir install \ 24 | --description $(PACKAGE_DESCRIPTION) \ 25 | --url $(URL) \ 26 | --vendor $(VENDOR) \ 27 | --after-remove $(POST_UNINSTALL_SCRIPT) \ 28 | --exclude $(EXCLUDE_FILES) --debug 29 | 30 | .PHONY: package 31 | package: clean rpm deb1804 deb1604 32 | 33 | rpm: 34 | $(FPM_COMMON_OPTIONS) -t rpm --iteration $(REVISION)_centos7 bin lib LICENSE.txt 35 | mv $(NAME)-$(REL_VERSION)*.rpm $(NAME)-$(REL_VERSION)-$(REVISION)-centos7.x86_64.rpm 36 | 37 | deb1804: 38 | $(FPM_COMMON_OPTIONS) -t deb --iteration $(REVISION)-ubuntu18.04 --depends "libc++1 >= 3.9" --depends "libc++abi1 >= 3.9" bin LICENSE.txt 39 | 40 | deb1604: 41 | $(FPM_COMMON_OPTIONS) -t deb --iteration $(REVISION)-ubuntu16.04 bin lib LICENSE.txt 42 | 43 | clean: 44 | rm -f $(NAME)*.rpm $(NAME)*.deb 45 | -------------------------------------------------------------------------------- /gen_errors/README.md: -------------------------------------------------------------------------------- 1 | # Couchbase Lite Error generation tool 2 | 3 | This little script generates error messages for the various platform implementation of CBLite 4 | from a single JSON source. 5 | 6 | ## Use 7 | python3 gen_errors.py 8 | 9 | The official source of error message truth is cbl-errors.json. 10 | 11 | ## Source 12 | cbl-errors.json should be an easy to follow the template. The file contains a list of 13 | objects, each of which is has a key (error) and a value (the message). The keys are 14 | used as variable names in some of the generated files. Don't use keys that will cause 15 | one of the compilers to barf. 16 | 17 | The source supports template strings. Templates are represented as a small integer 18 | surrounded by curly braces: '{n}'. The presumption is that, within a message, distinct 19 | values of n will be replaced by distinct texts and that identical values of n will be 20 | replaced by identical strings. 21 | 22 | Platforms code is on its own assuring that it provided the right number of arguments 23 | to any error message template, assuring that they are appropriate, and for handling 24 | any errors that arise from formatting. 25 | 26 | ## Generation 27 | The code generation for each platfom is handled by a separate python module. It should 28 | be possible to change the generation for any one platform without affecting any other. 29 | 30 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/images/relationship.puml: -------------------------------------------------------------------------------- 1 | @startuml relationship 2 | skinparam Handwritten true 3 | skinparam DefaultFontColor Blue 4 | skinparam ArrowColor White 5 | skinparam ObjectBorderColor Blue 6 | skinparam BackgroundColor #4444FF 7 | skinparam ArrowFontColor White 8 | skinparam object { 9 | BackgroundColor White 10 | } 11 | 12 | object LargeDatasetGenerator { 13 | Program 14 | } 15 | 16 | object LargeDatasetGenerator.Core { 17 | Library 18 | } 19 | 20 | object LargeDatasetGenerator.Abstractions { 21 | Library 22 | } 23 | object LargeDatasetGenerator.CouchbaseLite { 24 | Library 25 | } 26 | 27 | object LargeDatasetGenerator.SyncGateway { 28 | Library 29 | } 30 | 31 | object LargeDatasetGenerator.CouchbaseServer { 32 | Library 33 | } 34 | 35 | LargeDatasetGenerator.Core <|-- LargeDatasetGenerator : Uses 36 | LargeDatasetGenerator.Abstractions *-- LargeDatasetGenerator.Core : References 37 | LargeDatasetGenerator.Abstractions *-- LargeDatasetGenerator.CouchbaseLite : References 38 | LargeDatasetGenerator.Abstractions *-- LargeDatasetGenerator.SyncGateway : References 39 | LargeDatasetGenerator.Abstractions *-- LargeDatasetGenerator.CouchbaseServer : References 40 | LargeDatasetGenerator.CouchbaseLite *-- LargeDatasetGenerator : References 41 | LargeDatasetGenerator.SyncGateway *-- LargeDatasetGenerator : References 42 | LargeDatasetGenerator.CouchbaseServer *-- LargeDatasetGenerator : References 43 | @enduml -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator/LargeDatasetGenerator.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.1 6 | latest 7 | 2.1.8 8 | true 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /gen_errors/android.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2017 Couchbase, Inc All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import json 17 | import os 18 | 19 | 20 | class AndroidFormatter: 21 | """Formatter for Android resources 22 | Android will read a map from a JSON document 23 | This formatter will handle strings with up to 5 arguments""" 24 | name = "Android" 25 | 26 | msg_file_name = "errors.json" 27 | 28 | def __init__(self, out_dir): 29 | self.out_file = os.path.join(out_dir, AndroidFormatter.msg_file_name) 30 | 31 | def format(self, errors): 32 | errors_dict = dict() 33 | for error in errors: 34 | message = error["message"] 35 | errors_dict[error["name"]] = message.format("%1$s", "%2$s", "%3$s", "%4$s", "%5$s") 36 | with open(self.out_file, "w") as message_file: 37 | message_file.write(json.dumps(errors_dict)) 38 | -------------------------------------------------------------------------------- /ArgumentTokenizer.hh: -------------------------------------------------------------------------------- 1 | // 2 | // ArgumentTokenizer.hh 3 | // 4 | // Copyright (c) 2018 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include 21 | #include 22 | 23 | class ArgumentTokenizer { 24 | public: 25 | void reset(); 26 | void reset(const char *input); 27 | void reset(std::vector args); 28 | 29 | bool hasArgument() const {return _hasArgument;} 30 | const std::string& argument() const {return _argument;} 31 | bool next(); 32 | 33 | std::string restOfInput(); 34 | 35 | bool tokenize(const char *input, std::vector &outArgs); 36 | 37 | private: 38 | std::vector _args; 39 | std::string _input; 40 | const char* _current {nullptr}; 41 | const char* _startOfArg {nullptr}; 42 | bool _hasArgument; 43 | std::string _argument; 44 | }; 45 | 46 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 1.0.0.{build} 2 | branches: 3 | only: 4 | - dev 5 | 6 | skip_tags: true 7 | 8 | init: 9 | - cmd: git config --global core.autocrlf true 10 | 11 | # Environment 12 | image: Visual Studio 2017 13 | clone_depth: 1 14 | 15 | before_build: 16 | - dotnet restore LargeDatasetGenerator/LargeDatasetGenerator.sln 17 | 18 | build_script: 19 | - ps: dotnet build -c release LargeDatasetGenerator/LargeDatasetGenerator/LargeDatasetGenerator.csproj 20 | - ps: dotnet publish -r win10-x64 -c release LargeDatasetGenerator/LargeDatasetGenerator/LargeDatasetGenerator.csproj 21 | - ps: Add-AppveyorMessage -Message "Windows publish completed" 22 | - ps: dotnet publish -r osx-x64 -c release LargeDatasetGenerator/LargeDatasetGenerator/LargeDatasetGenerator.csproj 23 | - ps: Add-AppveyorMessage -Message "macOS publish completed" 24 | - ps: dotnet publish -r linux-x64 -c release LargeDatasetGenerator/LargeDatasetGenerator/LargeDatasetGenerator.csproj 25 | - ps: Add-AppveyorMessage -Message "Linux publish completed" 26 | 27 | after_build: 28 | - ps: Move-Item .\LargeDatasetGenerator\LargeDatasetGenerator\bin\release\netcoreapp2.1\win10-x64\publish\ win10-x64 29 | - ps: Move-Item .\LargeDatasetGenerator\LargeDatasetGenerator\bin\release\netcoreapp2.1\osx-x64\publish\ macos-x64 30 | - ps: Move-Item .\LargeDatasetGenerator\LargeDatasetGenerator\bin\release\netcoreapp2.1\linux-x64\publish\ linux-x64 31 | 32 | artifacts: 33 | - path: win10-x64 34 | - path: macos-x64 35 | - path: linux-x64 36 | -------------------------------------------------------------------------------- /litecp/RemoteEndpoint.hh: -------------------------------------------------------------------------------- 1 | // 2 | // RemoteEndpoint.hh 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include "Endpoint.hh" 21 | 22 | 23 | class RemoteEndpoint : public Endpoint { 24 | public: 25 | RemoteEndpoint(const string &spec) 26 | :Endpoint(spec) 27 | { } 28 | 29 | virtual bool isDatabase() const override {return true;} 30 | virtual bool isRemote() const override {return true;} 31 | 32 | virtual void prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint*) override; 33 | virtual void copyTo(Endpoint *dst, uint64_t limit) override; 34 | virtual void writeJSON(slice docID, slice json) override; 35 | virtual void finish() override; 36 | 37 | const C4Address& url() const {return _address;} 38 | C4String databaseName() const {return _dbName;} 39 | 40 | private: 41 | C4Address _address; 42 | C4String _dbName; 43 | }; 44 | 45 | 46 | -------------------------------------------------------------------------------- /litecp/RemoteEndpoint.cc: -------------------------------------------------------------------------------- 1 | // 2 | // RemoteEndpoint.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "RemoteEndpoint.hh" 20 | #include "DBEndpoint.hh" 21 | 22 | 23 | void RemoteEndpoint::prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint *other) { 24 | Endpoint::prepare(isSource, mustExist, docIDProperty, other); 25 | 26 | if (!c4address_fromURL(slice(_spec), &_address, &_dbName)) 27 | Tool::instance->fail("Invalid database URL"); 28 | } 29 | 30 | 31 | // As source (i.e. pull): 32 | void RemoteEndpoint::copyTo(Endpoint *dst, uint64_t limit) { 33 | auto dstDB = dynamic_cast(dst); 34 | if (dstDB) 35 | dstDB->replicateWith(*this, false); 36 | else 37 | Tool::instance->fail("Sorry, this mode isn't supported."); 38 | } 39 | 40 | 41 | // As destination: 42 | void RemoteEndpoint::writeJSON(slice docID, slice json) { 43 | Tool::instance->fail("Sorry, this mode isn't supported."); 44 | } 45 | 46 | 47 | void RemoteEndpoint::finish() { 48 | } 49 | -------------------------------------------------------------------------------- /cblite/cbliteTool+sql.cc: -------------------------------------------------------------------------------- 1 | // 2 | // cbliteTool+sql.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "cbliteTool.hh" 20 | #include "c4Private.h" 21 | 22 | 23 | void CBLiteTool::sqlUsage() { 24 | writeUsageCommand("sql", false, "QUERY"); 25 | cerr << 26 | " Runs a raw SQL query on the database file. This is NOT a way to query your documents!\n" 27 | " Rather, it's a very low-level diagnostic tool that will not be useful unless you know the\n" 28 | " underlying SQLite schema used by LiteCore.\n" 29 | " NOTE: Query must be a single (quoted) argument. Sorry.\n" 30 | ; 31 | } 32 | 33 | 34 | void CBLiteTool::sqlQuery() { 35 | // Read params: 36 | if (_showHelp) { 37 | sqlUsage(); 38 | return; 39 | } 40 | openDatabaseFromNextArg(); 41 | string sql = restOfInput("sql statement"); 42 | 43 | C4Error error; 44 | alloc_slice fleeceResult = c4db_rawQuery(_db, slice(sql), &error); 45 | if (!fleeceResult) 46 | fail("Query failed", error); 47 | 48 | prettyPrint(Value::fromData(fleeceResult)); 49 | cout << '\n'; 50 | } 51 | 52 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Abstractions/IDataGenerator.cs: -------------------------------------------------------------------------------- 1 | // 2 | // IDataGenerator.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System.Threading.Tasks; 20 | 21 | namespace LargeDatasetGenerator.Generator 22 | { 23 | /// 24 | /// An interface for generating values to insert into a set of output data. 25 | /// 26 | public interface IDataGenerator 27 | { 28 | #region Properties 29 | 30 | /// 31 | /// Gets a description of what this generator does for informational purposes 32 | /// 33 | string Description { get; } 34 | 35 | /// 36 | /// Gets the signature expected for this generator for informational purposes 37 | /// 38 | string Signature { get; } 39 | 40 | #endregion 41 | 42 | #region Public Methods 43 | 44 | /// 45 | /// Generates a new arbitrary value depending on the object's functionality 46 | /// 47 | /// An awaitable task holding the result 48 | Task GenerateValueAsync(); 49 | 50 | #endregion 51 | } 52 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/DataGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Reflection; 5 | 6 | namespace LargeDatasetGenerator.Generator 7 | { 8 | public static class DataGenerator 9 | { 10 | #region Constants 11 | 12 | public static readonly IReadOnlyDictionary TypeMap; 13 | 14 | #endregion 15 | 16 | #region Constructors 17 | 18 | /// 19 | /// Static Constructor 20 | /// 21 | static DataGenerator() 22 | { 23 | TypeMap = PluginLoader.GeneratorTypes.ToDictionary(x => x.Name.Replace("Generator", "").ToLowerInvariant(), 24 | v => v); 25 | } 26 | 27 | #endregion 28 | 29 | #region Public Methods 30 | 31 | /// 32 | /// Gets the generator registered for the given identifier, if it exists. A new object 33 | /// is created for every call. 34 | /// 35 | /// The identifier of the registered generator 36 | /// The input to create it with (not always used) 37 | /// The instantiated generator object 38 | /// Thrown if the given key has no registration 39 | public static IDataGenerator Generator(string identifier, string input) 40 | { 41 | if (!TypeMap.ContainsKey(identifier)) { 42 | throw new KeyNotFoundException($"Generator not found for {identifier}"); 43 | } 44 | 45 | var type = TypeMap[identifier]; 46 | if (type.GetConstructor(new[] {typeof(string)}) != null) { 47 | return Activator.CreateInstance(type, input) as IDataGenerator; 48 | } 49 | 50 | return Activator.CreateInstance(type) as IDataGenerator; 51 | } 52 | 53 | #endregion 54 | } 55 | } -------------------------------------------------------------------------------- /litecp/Endpoint.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Endpoint.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "Endpoint.hh" 20 | #include "DBEndpoint.hh" 21 | #include "RemoteEndpoint.hh" 22 | #include "JSONEndpoint.hh" 23 | #include "DirEndpoint.hh" 24 | #include "Error.hh" 25 | 26 | 27 | Endpoint* Endpoint::create(string str) { 28 | if (hasPrefix(str, "ws://") || hasPrefix(str, "wss://")) { 29 | return new RemoteEndpoint(str); 30 | } 31 | 32 | #ifndef _MSC_VER 33 | if (hasPrefix(str, "~/")) { 34 | str.erase(str.begin(), str.begin()+1); 35 | str.insert(0, getenv("HOME")); 36 | } 37 | #endif 38 | 39 | if (hasSuffix(str, kC4DatabaseFilenameExtension)) { 40 | return new DbEndpoint(str); 41 | } else if (hasSuffix(str, ".json")) { 42 | return new JSONEndpoint(str); 43 | } else if (hasSuffix(str, FilePath::kSeparator)) { 44 | return new DirectoryEndpoint(str); 45 | } else { 46 | if (str.find("://") != string::npos) 47 | cerr << "HINT: Replication URLs must use the 'ws:' or 'wss:' schemes.\n"; 48 | else if (FilePath(str).existsAsDir() || str.find('.') == string::npos) 49 | cerr << "HINT: If you are trying to copy to/from a directory of JSON files, append a '/' to the path.\n"; 50 | return nullptr; 51 | } 52 | } 53 | 54 | 55 | Endpoint* Endpoint::create(C4Database *db) { 56 | Assert(db != nullptr); 57 | return new DbEndpoint(db); 58 | } 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The Couchbase Mobile Tool Repo 2 | 3 | This repo is a collection of tools developed by the Couchbase mobile team. Official support is not guaranteed by presence in this repo, and each tool gains official support (i.e. paid support help, etc) on a case-by-case basis. For more information, open one of the following pages: 4 | 5 | - [cblite](README.cblite.md) 6 | - [cbl-log](README.cbl-log.md) 7 | - [LargeDatasetGenerator](LargeDatasetGenerator/README.md) 8 | 9 | ## Building the Tools 10 | 11 | First: if you haven't already checked out submodules, run `git submodule update --init --recursive`. 12 | 13 | There is a python script at the root of the repo called `build.py` which if run with no arguments will guide you through the build process (caveat: It might not work on "X weird system"). At the end of any interactive run, the script will print the non-interactive invocation to the screen for future reference. For a more advanced description of how to build without the script see [this doc](BUILDING.md) 14 | 15 | ### Compiler Requirements 16 | 17 | The recommended configuration is clang and libc++. Here is a table of versions: 18 | 19 | **Compiler (Linux)** 20 | 21 | | Name | Minimum | Recommended | 22 | |-------|---------|-------------| 23 | | GCC | 7 | 7 or higher | 24 | | clang | 3.9* | 5 or higher | 25 | \* Requires libstdc++-7 or libc++ from clang 5 or higher 26 | 27 | **Compiler (Windows)** 28 | 29 | Compiler versions on Windows have recently taken a turn beginning with Visual Studio 2017. However, it appears that compiler versions from here forward will be contained within a ten digit minor version range. Visual Studio 2017 was released with MSVC 14.10.25008 and is currently at MSVC 14.16.27027 while Visual Studio 2019 was released with MSVC 14.20.27508. For the purposes of these tools, anything 14.10 or above is fine. The C++ runtime installer for end users has been combined into one big installer for all of version 14 which overwrites the previous one instead of an installer per runtime like it used to be, so that makes checking if it is installed in an installer a bit more of a hassle. If you've installed VS 2017, you are good to go for compiling. For distribution you will need to check the MSVC version and install the VS2015-2019 C++ runtime distributable if needed. 30 | -------------------------------------------------------------------------------- /gen_errors/gen_errors.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2017 Couchbase, Inc All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | """Generate platform specific implementations 17 | of cross-platform error messages for Couchbase Lite. 18 | """ 19 | 20 | import json 21 | import os 22 | import sys 23 | 24 | from android import AndroidFormatter 25 | from csharp import CSharpFormatter 26 | from java import JavaFormatter 27 | from objc import ObjCFormatter 28 | 29 | 30 | def read_errors(file_name): 31 | with open(file_name, "r") as json_file: 32 | return json.load(json_file) 33 | 34 | 35 | def format_errors(error_defs, out_dir): 36 | formatters = [ 37 | AndroidFormatter(out_dir), 38 | CSharpFormatter(out_dir), 39 | JavaFormatter(out_dir), 40 | ObjCFormatter(out_dir)] 41 | 42 | json_errors = read_errors(error_defs) 43 | for formatter in formatters: 44 | print("== Formatting for {0}".format(formatter.name)) 45 | formatter.format(json_errors) 46 | 47 | 48 | def usage(): 49 | print("Usage: python gen_errors.py ") 50 | print(" errors.json - the error message definition file") 51 | print(" output directory - an existing, writable directory") 52 | exit(1) 53 | 54 | 55 | def main(): 56 | if len(sys.argv) != 3: 57 | usage() 58 | error_defs = sys.argv[1] 59 | if not os.path.exists(error_defs) or not os.path.isfile(error_defs) or (not os.access(error_defs, os.R_OK)): 60 | usage() 61 | out_dir = sys.argv[2] 62 | if not os.path.exists(out_dir) or os.path.isfile(out_dir) or (not os.access(out_dir, os.W_OK)): 63 | usage() 64 | 65 | format_errors(error_defs, out_dir) 66 | 67 | 68 | if __name__ == "__main__": 69 | main() 70 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Abstractions/IJsonOutput.cs: -------------------------------------------------------------------------------- 1 | // 2 | // IJsonOutput.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System.Collections.Generic; 20 | using System.Threading.Tasks; 21 | 22 | namespace LargeDatasetGenerator.Output 23 | { 24 | /// 25 | /// An interface representing an object that can accept JSON to write 26 | /// to an arbitrary output 27 | /// 28 | public interface IJsonOutput 29 | { 30 | #region Properties 31 | 32 | /// 33 | /// Gets the arguments that this object expects when it is invoked 34 | /// 35 | string ExpectedArgs { get; } 36 | 37 | /// 38 | /// Gets the name of this object so that it can be identified via 39 | /// passed argument 40 | /// 41 | string Name { get; } 42 | 43 | #endregion 44 | 45 | #region Public Methods 46 | 47 | /// 48 | /// Preload the arguments from the command line to verify that they are 49 | /// all present and correct 50 | /// 51 | /// The arguments to process (in the form --[argname] [value]) 52 | /// true if the arguments are all present and valid, otherwise false 53 | bool PreloadArgs(string[] args); 54 | 55 | /// 56 | /// Writes a batch of JSON objects to the output 57 | /// 58 | /// The set of JSON objects to write 59 | /// An awaitable task 60 | Task WriteBatchAsync(IEnumerable> json); 61 | 62 | #endregion 63 | } 64 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Outputs/JsonFileOutput.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | using McMaster.Extensions.CommandLineUtils; 8 | 9 | using Newtonsoft.Json; 10 | 11 | namespace LargeDatasetGenerator.Output 12 | { 13 | /// 14 | /// An output object that writes to JSON files. Each file will contain the batch size 15 | /// number of entries. 16 | /// 17 | public sealed class JsonFileOutput : IJsonOutput 18 | { 19 | #region Constants 20 | 21 | /// 22 | /// Used by the core library for a default output type 23 | /// 24 | public const string NameConst = "file"; 25 | 26 | #endregion 27 | 28 | #region Variables 29 | 30 | private int _currentBatch = 1; 31 | private string _filename; 32 | 33 | #endregion 34 | 35 | #region Properties 36 | 37 | /// 38 | public string ExpectedArgs { get; } = "--filename "; 39 | 40 | /// 41 | public string Name { get; } = NameConst; 42 | 43 | #endregion 44 | 45 | #region IJsonOutput 46 | 47 | /// 48 | public bool PreloadArgs(string[] args) 49 | { 50 | var subApp = new CommandLineApplication(); 51 | 52 | var filename = subApp.Option("--filename|-f ", "The file template to write JSON data into", 53 | CommandOptionType.SingleValue); 54 | filename.IsRequired(); 55 | 56 | subApp.OnExecute(() => { _filename = Path.GetFileNameWithoutExtension(filename.ParsedValue); }); 57 | 58 | return subApp.Execute(args) == 0; 59 | } 60 | 61 | /// 62 | public Task WriteBatchAsync(IEnumerable> json) 63 | { 64 | var nextBatch = Interlocked.Increment(ref _currentBatch); 65 | var filename = $"{_filename}_{nextBatch}.json"; 66 | return Task.Factory.StartNew(() => 67 | { 68 | var serialized = JsonConvert.SerializeObject(json, Formatting.Indented); 69 | File.WriteAllText(filename, serialized, Encoding.UTF8); 70 | }); 71 | } 72 | 73 | #endregion 74 | } 75 | } -------------------------------------------------------------------------------- /cblite/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | cmake_policy(VERSION 3.1) 3 | project (cblite) 4 | include("../cmake/common_setup.cmake") 5 | 6 | set(LITECORE ${PROJECT_SOURCE_DIR}/../vendor/couchbase-lite-core/) 7 | common_setup() 8 | get_platform_libs(PLATFORM_LIBS) 9 | 10 | set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS 11 | $<$:DEBUG> 12 | ) 13 | 14 | aux_source_directory("../cblite" CBLITE_SRC) 15 | aux_source_directory("../litecp" LITECP_SRC) 16 | aux_source_directory("../vendor/linenoise-ng/src" LINENOISE_SRC) 17 | add_subdirectory(${LITECORE} LiteCore) 18 | get_directory_property(LITECORE_LIBRARIES_PRIVATE DIRECTORY ${LITECORE} DEFINITION LITECORE_LIBRARIES_PRIVATE) 19 | 20 | fix_cpp_runtime() 21 | 22 | add_executable( 23 | cblite 24 | ${CBLITE_SRC} 25 | ${LITECP_SRC} 26 | ${LINENOISE_SRC} 27 | ../Tool.cc 28 | ../ArgumentTokenizer.cc 29 | ) 30 | 31 | add_executable( 32 | cblitetest 33 | ../ArgumentTokenizer.cc 34 | ../tests/tests_main.cc 35 | ../tests/TokenizerTest.cc 36 | ) 37 | 38 | target_include_directories( 39 | cblite PRIVATE 40 | ${PROJECT_SOURCE_DIR}/.. 41 | ${PROJECT_SOURCE_DIR}/../litecp 42 | ${PROJECT_SOURCE_DIR}/../vendor/linenoise-ng/include/ 43 | ${LITECORE}C/ 44 | ${LITECORE}C/include/ 45 | ${LITECORE}LiteCore/Support/ 46 | ${LITECORE}Replicator/ # for CivetWebSocket.hh 47 | ${LITECORE}vendor/fleece/API/ 48 | ${LITECORE}vendor/fleece/Fleece/Support/ # PlatformCompat.hh 49 | ${PROJECT_BINARY_DIR}/generated_headers/ 50 | ) 51 | 52 | target_compile_definitions(cblite PRIVATE -DCMAKE) 53 | target_link_libraries(cblite ${LITECORE_LIBRARIES_PRIVATE} LiteCoreREST_Static ${PLATFORM_LIBS}) 54 | if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") 55 | target_link_libraries(cblite ${ICU4C_COMMON} ${ICU4C_I18N}) 56 | endif() 57 | 58 | target_include_directories( 59 | cblitetest PRIVATE 60 | ${LITECORE}vendor/fleece/vendor/catch 61 | ${LITECORE}LiteCore/Support/ 62 | ${LITECORE}LiteCore/Storage/ 63 | ${LITECORE}vendor/fleece/Fleece/Support/ 64 | ${LITECORE}vendor/fleece/API/ 65 | ${LITECORE}vendor/BLIP-Cpp/src/util 66 | ${LITECORE}C/include/ 67 | ${PROJECT_SOURCE_DIR}/.. 68 | ) 69 | 70 | target_link_libraries( 71 | cblitetest PRIVATE 72 | ${PLATFORM_LIBS} 73 | ) 74 | 75 | install ( 76 | TARGETS cblite cblitetest 77 | RUNTIME DESTINATION bin 78 | LIBRARY DESTINATION lib 79 | ) -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.SyncGateway/SyncGatewayOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net.Http; 5 | using System.Net.Http.Headers; 6 | using System.Threading.Tasks; 7 | using LargeDatasetGenerator.Output; 8 | using McMaster.Extensions.CommandLineUtils; 9 | using Newtonsoft.Json; 10 | 11 | namespace LargeDatasetGenerator.SyncGateway 12 | { 13 | public sealed class SyncGatewayOutput : IJsonOutput, IDisposable 14 | { 15 | #region Variables 16 | 17 | private readonly HttpClient _httpClient = new HttpClient(); 18 | 19 | private Uri _url; 20 | 21 | #endregion 22 | 23 | #region Properties 24 | 25 | public string ExpectedArgs { get; } = "--url "; 26 | public string Name { get; } = "sync_gateway"; 27 | 28 | #endregion 29 | 30 | #region IDisposable 31 | 32 | public void Dispose() 33 | { 34 | _httpClient.Dispose(); 35 | } 36 | 37 | #endregion 38 | 39 | #region IJsonOutput 40 | 41 | public bool PreloadArgs(string[] args) 42 | { 43 | var subApp = new CommandLineApplication(); 44 | 45 | var url = subApp.Option("--url ", "The file template to write JSON data into", 46 | CommandOptionType.SingleValue); 47 | url.IsRequired(); 48 | url.Validators.Add(new UrlValidation()); 49 | 50 | subApp.OnExecute(() => 51 | { 52 | var urlString = url.ParsedValue; 53 | if (!urlString.EndsWith("/")) { 54 | urlString += "/"; 55 | } 56 | 57 | _url = new Uri(urlString, UriKind.Absolute); 58 | }); 59 | 60 | return subApp.Execute(args) == 0; 61 | } 62 | 63 | public async Task WriteBatchAsync(IEnumerable> json) 64 | { 65 | var postBody = new Dictionary 66 | { 67 | ["docs"] = json.ToList() 68 | }; 69 | 70 | var content = new StringContent(JsonConvert.SerializeObject(postBody)); 71 | content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); 72 | await _httpClient.PostAsync(new Uri(_url, "_bulk_docs"), 73 | content).ConfigureAwait(false); 74 | } 75 | 76 | #endregion 77 | } 78 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/README.md: -------------------------------------------------------------------------------- 1 | # LargeDatasetGenerator 2 | 3 | A .NET Core application designed to generate an arbitrarily large set of JSON data based on a template and write it to a predefined output. Examples of the output include: 4 | 5 | - JSON Files 6 | - Couchbase Lite 7 | - Couchbase Sync Gateway 8 | - Couchbase Server 9 | 10 | The input file format is heavily inspired by [JSON Generator](https://next.json-generator.com/) and involves a moustache style of functions written into JSON strings. The template itself is also valid JSON. 11 | 12 | ## Usage 13 | 14 | ``` 15 | LargeDatasetGenerator [options] 16 | 17 | Options: 18 | -? Show help information 19 | -i|--input The file to read the document template from 20 | -o|--output The way to save the resulting JSON 21 | -c|--count The number of entries to generate from the template (default 1000) 22 | -b|--batch-size The number of entries to process as one unit (default 100) 23 | --list-output List the available output methods for writing JSON 24 | --list-generators List the available generator functions for the template JSON 25 | ``` 26 | 27 | The output types provided by this repo are as follows: 28 | 29 | ``` 30 | file 31 | --filename 32 | 33 | couchbase_lite 34 | --name [--path ] 35 | 36 | couchbase_server 37 | --url --username --password --bucket 38 | 39 | sync_gateway 40 | --url 41 | ``` 42 | 43 | ## Extension 44 | 45 | This application is architected so that all of the logic is contained in a reusable library, combined with plugin functionality. Here is a diagram of the architecture: 46 | 47 | ![](images/relationship.png) 48 | 49 | To explain a bit further, the 'LargeDatasetGenerator' program references and uses classes from within 'LargeDatasetGeneratore.Core'. It also references, but does not directly use, three plugin assemblies that provide output functionality. If extending the functionality of the core library, you need only reference the 'LargeDatasetGenerator.Abstractions' assembly, built your plugin types, and then reference the plugin assembly from the main program when building. The core library will take care of the rest. 50 | 51 | To learn more about writing custom output types, see [README_output.md](README_output.md). 52 | 53 | To learn more about writing custom generator types, see [README_generator.md](README_generator.md) -------------------------------------------------------------------------------- /BUILDING.md: -------------------------------------------------------------------------------- 1 | # Guide to Building 2 | 3 | The first thing you should try is the interactive python script (which will tell you the non-interactive command at the end of its run). If you are here it is because it is either not working or you are extra curious. Each app in here is different (you can think of this repo as containing, basically, several repos in one) and they are organized into their own folders. 4 | 5 | ## cbl-log 6 | 7 | This is a distributed and officially licensed product. It is a CMake project so if you don't know CMake well, it would be good to know it fairly well before attempting to build. The CMake project itself is in the `cbl-log` folder. It contains two targets: 8 | 9 | - cbl-log : The final executable 10 | - cbl-logtest : The test harness for cbl-log 11 | 12 | The product is buildable via standard CMake commands, so the same as any other CMake based solution you've come across. There are no CMake options to worry about. 13 | 14 | ## cblite 15 | 16 | This is a command line tool similar to the sqlite CLI. It is *not* an officially licensed product (open source license only), but provided for convenience. It is a CMake project so if you don't know CMake well, it would be good to know it fairly well before attempting to build. The CMake project itself is in the `cblite` folder. It contains two targets: 17 | 18 | - cblite : The final executable 19 | - cblite : The test harness for cblite 20 | 21 | The product is buildable via standard CMake commands, so the same as any other CMake based solution you've come across. There are no CMake options to worry about. 22 | 23 | ## LargeDatasetGenerator 24 | 25 | This is a tool for generating a large set of proceduraly generated data to be inserted into various endpoints. It can be useful for load testing various components. It is *not* an officially licensed product (open source license only). It is a dotnet CLI project, which means to build it you will need to install the [.NET Core SDK](https://dotnet.microsoft.com/download). After that, the CLI is pretty similar to any other build system. `dotnet build` builds, `dotnet build -c Release` builds Release, etc. If you want to build the program without any dependencies on the .NET runtime (native dependencies of the runtime itself still required, beware), then follow the guide for [.NET Core 2.x](https://dotnetthoughts.net/how-to-create-a-self-contained-dotnet-core-application/) (good) or [.NET Core 3.x](https://gunnarpeipman.com/dotnet-core-self-contained-executable/) much better. -------------------------------------------------------------------------------- /gen_errors/csharp.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2019 Couchbase, Inc All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | from datetime import datetime 18 | 19 | class CSharpFormatter: 20 | """Formatter for C# symbol definitions 21 | Definitions are formatted as string constants""" 22 | name = "C#" 23 | year = datetime.today().year 24 | 25 | msg_file_name = "CouchbaseLiteErrorMessage.cs" 26 | 27 | message_def = " internal const string {0} = \"{1}\";\n" 28 | 29 | trailer = """ } 30 | } 31 | 32 | """ 33 | 34 | header = """// CouchbaseLiteErrorMessage.cs 35 | // 36 | // Copyright (c) {0} Couchbase, Inc All rights reserved. 37 | // 38 | // Licensed under the Apache License, Version 2.0 (the "License"); 39 | // you may not use this file except in compliance with the License. 40 | // You may obtain a copy of the License at 41 | // 42 | // http://www.apache.org/licenses/LICENSE-2.0 43 | // 44 | // Unless required by applicable law or agreed to in writing, software 45 | // distributed under the License is distributed on an "AS IS" BASIS, 46 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 47 | // See the License for the specific language governing permissions and 48 | // limitations under the License. 49 | // 50 | using System; 51 | using System.Collections.Generic; 52 | using System.Text; 53 | 54 | namespace Couchbase.Lite 55 | {{ 56 | internal static partial class CouchbaseLiteErrorMessage 57 | {{ 58 | """.format(year) 59 | 60 | def __init__(self, out_dir): 61 | self.out_file = os.path.join(out_dir, CSharpFormatter.msg_file_name) 62 | 63 | def format(self, errors_json): 64 | with open(self.out_file, "w") as message_file: 65 | message_file.write(CSharpFormatter.header) 66 | for error in errors_json: 67 | message_file.write(CSharpFormatter.message_def.format(error["name"], error["message"])) 68 | message_file.write(CSharpFormatter.trailer) 69 | -------------------------------------------------------------------------------- /litecp/JSONEndpoint.cc: -------------------------------------------------------------------------------- 1 | // 2 | // JSONEndpoint.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "JSONEndpoint.hh" 20 | 21 | 22 | void JSONEndpoint::prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint *other) { 23 | Endpoint::prepare(isSource, mustExist, docIDProperty, other); 24 | bool err; 25 | if (isSource) { 26 | _in.reset(new ifstream(_spec, ios_base::in)); 27 | err = _in->fail(); 28 | if (!err && _in->peek() != '{') 29 | Tool::instance->fail("Source file does not appear to contain JSON objects (does not start with '{')."); 30 | } else { 31 | if (mustExist && remove(_spec.c_str()) != 0) 32 | Tool::instance->fail(format("Destination JSON file %s doesn't exist or is not writeable [--existing]", 33 | _spec.c_str())); 34 | _out.reset(new ofstream(_spec, ios_base::trunc | ios_base::out)); 35 | err = _out->fail(); 36 | } 37 | if (err) 38 | Tool::instance->fail(format("Couldn't open JSON file %s", _spec.c_str())); 39 | } 40 | 41 | 42 | // As source: 43 | void JSONEndpoint::copyTo(Endpoint *dst, uint64_t limit) { 44 | if (Tool::instance->verbose()) 45 | cout << "Importing JSON file...\n"; 46 | string line; 47 | unsigned lineNo; 48 | for (lineNo = 0; lineNo < limit && getline(*_in, line); ++lineNo) { 49 | dst->writeJSON(nullslice, c4str(line)); 50 | } 51 | if (_in->bad()) 52 | Tool::instance->errorOccurred("Couldn't read JSON file"); 53 | else if (lineNo == limit) 54 | cout << "Stopped after " << limit << " documents.\n"; 55 | } 56 | 57 | 58 | // As destination: 59 | void JSONEndpoint::writeJSON(slice docID, slice json) { 60 | if (docID && _docIDProperty) { 61 | *_out << "{\"" << _docIDProperty << "\":\"" << docID << "\","; 62 | json.moveStart(1); 63 | } 64 | *_out << json << '\n'; 65 | logDocument(docID); 66 | } 67 | -------------------------------------------------------------------------------- /cbl-log/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | cmake_policy(VERSION 3.1) 3 | project (cbl-log) 4 | include("../cmake/common_setup.cmake") 5 | 6 | set(LITECORE ${PROJECT_SOURCE_DIR}/../vendor/couchbase-lite-core/) 7 | set(LITECORE_DISABLE_ICU CACHE BOOL ON) 8 | common_setup() 9 | get_platform_libs(PLATFORM_LIBS) 10 | 11 | set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS 12 | $<$:DEBUG> 13 | ) 14 | 15 | aux_source_directory("../vendor/linenoise-ng/src" LINENOISE_SRC) 16 | set( 17 | LOGCAT_SRC 18 | "../cbl-log/cbl-log.cc" 19 | "${LITECORE}LiteCore/Support/LogDecoder.cc" 20 | "${LITECORE}LiteCore/Support/LogEncoder.cc" 21 | ) 22 | 23 | add_subdirectory(${LITECORE} LiteCore) 24 | 25 | add_executable( 26 | cbl-log 27 | ${LOGCAT_SRC} 28 | ${LINENOISE_SRC} 29 | ../Tool.cc 30 | ../ArgumentTokenizer.cc 31 | ) 32 | 33 | target_include_directories( 34 | cbl-log PRIVATE 35 | ${PROJECT_SOURCE_DIR}/.. 36 | ${LITECORE}C/include/ 37 | ${PROJECT_SOURCE_DIR}/../vendor/linenoise-ng/include/ 38 | ${LITECORE}LiteCore/Support/ 39 | ${LITECORE}vendor/fleece/API/ 40 | ${LITECORE}vendor/fleece/Fleece/Support/ # PlatformCompat.hh 41 | ${LITECORE}vendor/BLIP-Cpp/src/util 42 | ${PROJECT_BINARY_DIR}/generated_headers/ 43 | ) 44 | 45 | target_compile_definitions(cbl-log PRIVATE -DCBLTOOL_NO_C_API -DCMAKE) 46 | target_link_libraries(cbl-log FleeceStatic Support BLIPStatic ${PLATFORM_LIBS}) 47 | 48 | add_executable( 49 | cbl-logtest 50 | ../tests/TokenizerTest.cc 51 | ../ArgumentTokenizer.cc 52 | ../tests/tests_main.cc 53 | ${LITECORE}LiteCore/tests/LogEncoderTest.cc 54 | ${LITECORE}LiteCore/Storage/UnicodeCollator_Stub.cc 55 | ) 56 | 57 | target_include_directories( 58 | cbl-logtest PRIVATE 59 | ${LITECORE}vendor/fleece/vendor/catch 60 | ${LITECORE}LiteCore/Support/ 61 | ${LITECORE}LiteCore/Storage/ 62 | ${LITECORE}vendor/fleece/Fleece/Support/ 63 | ${LITECORE}vendor/fleece/API/ 64 | ${LITECORE}vendor/BLIP-Cpp/src/util 65 | ${LITECORE}C/include/ 66 | ${PROJECT_SOURCE_DIR}/.. 67 | ) 68 | 69 | if(APPLE) 70 | set(CRYPTO_LIB "-framework Security") 71 | else() 72 | set(CRYPTO_LIB "mbedcrypto") 73 | endif() 74 | 75 | target_link_libraries( 76 | cbl-logtest PRIVATE 77 | LiteCoreStatic 78 | FleeceStatic 79 | Support 80 | SQLite3_UnicodeSN 81 | BLIPStatic 82 | CivetWeb 83 | ${CRYPTO_LIB} 84 | ${PLATFORM_LIBS} 85 | ) 86 | 87 | install ( 88 | TARGETS cbl-log cbl-logtest 89 | RUNTIME DESTINATION bin 90 | LIBRARY DESTINATION lib 91 | ) -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/RepeatGeneratorArgs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | using sly.lexer; 4 | 5 | namespace LargeDatasetGenerator.Generator 6 | { 7 | internal sealed class RepeatGeneratorArgs : GeneratorArgumentParser<(int, int)> 8 | { 9 | #region Variables 10 | 11 | private int _currentArg; 12 | private int _max; 13 | private int _min; 14 | 15 | #endregion 16 | 17 | #region Constructors 18 | 19 | public RepeatGeneratorArgs(string input) : base(input) 20 | { 21 | } 22 | 23 | #endregion 24 | 25 | #region Overrides 26 | 27 | protected override void AcceptToken(Token token, Token previous) 28 | { 29 | switch (token.TokenID) { 30 | case GeneratorTokenType.OpenParen: 31 | if (previous.TokenID != GeneratorTokenType.None) { 32 | ThrowSyntaxError(token); 33 | } 34 | 35 | _currentArg++; 36 | break; 37 | case GeneratorTokenType.CloseParen: 38 | if (_currentArg < 2) { 39 | throw new InvalidOperationException("Missing argument 'min'"); 40 | } 41 | 42 | if (previous.TokenID != GeneratorTokenType.Int) { 43 | ThrowSyntaxError(token); 44 | } 45 | 46 | break; 47 | case GeneratorTokenType.Int: 48 | if (previous.TokenID != GeneratorTokenType.OpenParen && 49 | previous.TokenID != GeneratorTokenType.Comma) { 50 | ThrowSyntaxError(token); 51 | } 52 | 53 | if (_currentArg == 1) { 54 | _min = _max = token.IntValue; 55 | } else { 56 | _max = token.IntValue; 57 | } 58 | 59 | break; 60 | case GeneratorTokenType.Comma: 61 | if (previous.TokenID != GeneratorTokenType.Int) { 62 | ThrowSyntaxError(token); 63 | } 64 | 65 | _currentArg++; 66 | if (_currentArg > 2) { 67 | ThrowSyntaxError(token); 68 | } 69 | 70 | break; 71 | default: 72 | ThrowSyntaxError(token); 73 | break; 74 | } 75 | } 76 | 77 | protected override (int, int) Finish() 78 | { 79 | return (_min, _max); 80 | } 81 | 82 | #endregion 83 | } 84 | } -------------------------------------------------------------------------------- /README.cblite.md: -------------------------------------------------------------------------------- 1 | # The `cblite` Tool 2 | 3 | `cblite` is a command-line tool for inspecting and querying [LiteCore][LITECORE] and [Couchbase Lite][CBL] databases (which are directories with a `.cblite2` extension.) 4 | 5 | For build instructions, see [BUILDING.md](BUILDING.md). 6 | 7 | ## Features 8 | 9 | | Command | Purpose | 10 | |----------------|---------| 11 | | `cblite cat` | Display the body of one or more documents | 12 | | `cblite cp` | Replicate, import or export a database | 13 | | `cblite file` | Display information about the database | 14 | | `cblite help` | Display help text | 15 | | `cblite ls` | List the documents in the database | 16 | | `cblite put` | Create or update a document | 17 | | `cblite query` | Run queries, using the [JSON Query Schema][QUERY] | 18 | | `cblite revs` | List the revisions of a document | 19 | | `cblite rm` | Delete a document | 20 | | `cblite select`| Run queries, using [N1QL][N1QL] syntax | 21 | | `cblite serve` | Starts a (rudimentary) REST API listener | 22 | 23 | See the [full documentation](Documentation.md)... 24 | 25 | ## Example 26 | 27 | ``` 28 | $ cblite file travel-sample.cblite2 29 | Database: travel-sample.cblite2/ 30 | Total size: 34MB 31 | Documents: 31591, last sequence 31591 32 | 33 | $ cblite ls -l --limit 10 travel-sample.cblite2 34 | Document ID Rev ID Flags Seq Size 35 | airline_10 1-d70614ae --- 1 0.1K 36 | airline_10123 1-091f80f6 --- 2 0.1K 37 | airline_10226 1-928c43f4 --- 3 0.1K 38 | airline_10642 1-5cb6252c --- 4 0.1K 39 | airline_10748 1-630b0443 --- 5 0.1K 40 | airline_10765 1-e7999661 --- 6 0.1K 41 | airline_109 1-bd546abb --- 7 0.1K 42 | airline_112 1-ca955c69 --- 8 0.1K 43 | airline_1191 1-28dbba6e --- 9 0.1K 44 | airline_1203 1-045b6947 --- 10 0.1K 45 | (Stopping after 10 docs) 46 | 47 | $ cblite travel-sample.cblite2 48 | (cblite) query --limit 10 ["=", [".type"], "airline"] 49 | ["_id": "airline_10"] 50 | ["_id": "airline_10123"] 51 | ["_id": "airline_10226"] 52 | ["_id": "airline_10642"] 53 | ["_id": "airline_10748"] 54 | ["_id": "airline_10765"] 55 | ["_id": "airline_109"] 56 | ["_id": "airline_112"] 57 | ["_id": "airline_1191"] 58 | ["_id": "airline_1203"] 59 | (Limit was 10 rows) 60 | (cblite) query --limit 10 {WHAT: [[".name"]], WHERE: ["=", [".type"], "airline"], ORDER_BY: [[".name"]]} 61 | ["40-Mile Air"] 62 | ["AD Aviation"] 63 | ["ATA Airlines"] 64 | ["Access Air"] 65 | ["Aigle Azur"] 66 | ["Air Austral"] 67 | ["Air Caledonie International"] 68 | ["Air Caraïbes"] 69 | ["Air Cargo Carriers"] 70 | ["Air Cudlua"] 71 | (Limit was 10 rows) 72 | (cblite) ^D 73 | $ 74 | ``` 75 | 76 | [LITECORE]: https://github.com/couchbase/couchbase-lite-core 77 | [CBL]: https://www.couchbase.com/products/lite 78 | [QUERY]: https://github.com/couchbase/couchbase-lite-core/wiki/JSON-Query-Schema 79 | [N1QL]: https://docs.couchbase.com/server/6.0/n1ql/n1ql-language-reference/index.html 80 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/README_generator.md: -------------------------------------------------------------------------------- 1 | # Adding Generators 2 | 3 | The generator types are the moustache formatted values inside the template JSON which dictate how the values will be created in the output JSON. A quick example is `{{bool()}}` which will randomly output either `true` or `false`. The interface to implement for this is [IDataGenerator](LargeDatasetGenerator.Abstractions/IDataGenerator.cs). Your plugin will reference that assembly, compile, and then the executing program will reference your plugin. Here is a walkthrough of a fairly simply generator that is included in the core library: 4 | 5 | ```c# 6 | public sealed class IntegerGenerator : IDataGenerator 7 | { 8 | private readonly long _max; 9 | private readonly long _min; 10 | 11 | // IDataGenerator properties 12 | public string Description { get; } = "Generates random 64-bit integers between min and max"; 13 | 14 | public string Signature { get; } = "{{integer(min: int64 = int64.min / 2, max: int64 = int64.max / 2)}}"; 15 | 16 | public IntegerGenerator(string input) 17 | { 18 | // This constructor is optional in general, but required for 19 | // generators that accept arguments 20 | 21 | // See link below for this class 22 | var parser = new IntegerGeneratorArgs(input); 23 | var (min, max) = parser.Parse(); 24 | _min = min; 25 | _max = max; 26 | } 27 | 28 | public IntegerGenerator() 29 | { 30 | // This constructor is not used in general, but rather for the 31 | // --list-generators mode so its functionality can be arbitrary 32 | var parser = new IntegerGeneratorArgs("()"); 33 | var (min, max) = parser.Parse(); 34 | _min = min; 35 | _max = max; 36 | } 37 | 38 | public Task GenerateValueAsync() 39 | { 40 | // ThreadSafeRandom is just a locked wrapper around a static Random 41 | // instance and NextInt64 is an extension method for creating 42 | // 64-bit random numbers. 43 | return Task.FromResult(ThreadSafeRandom.NextInt64(_min, _max)); 44 | } 45 | } 46 | ``` 47 | 48 | The `IntegerGeneratorArgs` class is somewhat large and complex looking so it is not included above but you can find it [here](LargeDatasetGenerator.Core/Generator/IntegerGenerator.cs). It makes use of the [csly](https://github.com/b3b00/csly) lexer/parser library to parse the arguments out of the string. 49 | 50 | There are two properties and one method to implement in the interface: 51 | 52 | - Description: Used for the `--list-generators` argument of the program to give an indication of what the generator does. 53 | - Signature: Also used for the `--list-generators` argument to show the arguments used. Optional arguments will have `= ` after them. 54 | - GenerateValueAsync: Generates the next value for use in the output JSON. Most are non-async and will return `Task.FromResult` but for future-proofing the ability to run asynchronously was used (random numbers from an Internet hardware source?) -------------------------------------------------------------------------------- /litecp/DBEndpoint.hh: -------------------------------------------------------------------------------- 1 | // 2 | // DBEndpoint.hh 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include "Endpoint.hh" 21 | #include "c4Replicator.h" 22 | #include "Stopwatch.hh" 23 | #include "fleece/slice.hh" 24 | 25 | class JSONEndpoint; 26 | class RemoteEndpoint; 27 | 28 | 29 | class DbEndpoint : public Endpoint { 30 | public: 31 | DbEndpoint(const string &spec) 32 | :Endpoint(spec) 33 | { } 34 | 35 | DbEndpoint(C4Database* db); 36 | 37 | virtual bool isDatabase() const override {return true;} 38 | alloc_slice path() const; 39 | 40 | virtual void prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint*) override; 41 | void setBidirectional(bool bidi) {_bidirectional = bidi;} 42 | void setContinuous(bool cont) {_continuous = cont;} 43 | 44 | using credentials = std::pair; 45 | void setCredentials(const credentials &cred) {_credentials = cred;} 46 | 47 | virtual void copyTo(Endpoint *dst, uint64_t limit) override; 48 | virtual void writeJSON(slice docID, slice json) override; 49 | virtual void finish() override; 50 | 51 | void pushToLocal(DbEndpoint&); 52 | void replicateWith(RemoteEndpoint&, bool pushing =true); 53 | 54 | void exportTo(JSONEndpoint*); 55 | void importFrom(JSONEndpoint*); 56 | 57 | void onStateChanged(C4ReplicatorStatus status); 58 | void onDocError(bool pushing, 59 | size_t count, 60 | const C4DocumentEnded* docs[]); 61 | 62 | private: 63 | void enterTransaction(); 64 | void commit(); 65 | void startLine(); 66 | 67 | void exportTo(Endpoint *dst, uint64_t limit); 68 | C4ReplicatorParameters replicatorParameters(C4ReplicatorMode push, C4ReplicatorMode pull); 69 | void replicate(C4Replicator*, C4Error&); 70 | 71 | c4::ref _db; 72 | unsigned _transactionSize {0}; 73 | bool _inTransaction {false}; 74 | Endpoint* _otherEndpoint; 75 | Stopwatch _stopwatch; 76 | double _lastElapsed {0}; 77 | uint64_t _lastDocCount {0}; 78 | bool _needNewline {false}; 79 | 80 | // Replication mode only: 81 | bool _bidirectional {false}; 82 | bool _continuous {false}; 83 | credentials _credentials; 84 | fleece::alloc_slice _options; 85 | 86 | static constexpr unsigned kMaxTransactionSize = 1000; 87 | }; 88 | 89 | 90 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/README_output.md: -------------------------------------------------------------------------------- 1 | # Adding Output Types 2 | 3 | The interface for writing JSON to output is [IJsonOutput](LargeDatasetGenerator.Abstractions/IJsonOutput.cs) in the LargetDatasetGenerator.Abstractions assembly. Your plugin will reference that assembly, compile, and then the executing program will reference your plugin. Here is a walkthrough of how the default one works: 4 | 5 | ```c# 6 | public sealed class JsonFileOutput : IJsonOutput 7 | { 8 | // Use to keep track of the current filename 9 | // (one file per batch) 10 | private int _currentBatch = 1; 11 | 12 | // The filename that was passed in by the user 13 | private string _filename; 14 | 15 | // IJsonOutput properties 16 | public string ExpectedArgs { get; } = "--filename "; 17 | public string Name { get; } = "file"; 18 | 19 | // IJsonOutput methods 20 | public bool PreloadArgs(string[] args) 21 | { 22 | // Not necessary to use this library, but CommandLineUtils is setup 23 | // already to process the format passed in through 'args'. 24 | var subApp = new CommandLineApplication(); 25 | 26 | // Marked as "single value" to indicate it should only appear once 27 | // in the input 28 | var filename = subApp.Option("--filename|-f ", "The file template to write JSON data into", 29 | CommandOptionType.SingleValue); 30 | 31 | // This call ensures the argument is present, or an exception 32 | // is thrown 33 | filename.IsRequired(); 34 | 35 | subApp.OnExecute(() => 36 | { 37 | // By this point all arguments have been parsed correctly, 38 | // so record the value 39 | _filename = Path.GetFileNameWithoutExtensio(filename.ParsedValue); 40 | }); 41 | 42 | //This triggers the argument parsing and 'OnExecute' callback 43 | return subApp.Execute(args) == 0; 44 | } 45 | 46 | public Task WriteBatchAsync(IEnumerable> json) 47 | { 48 | // One file per batch 49 | var filename = $"{_filename}_{_currentBatch++}.json"; 50 | 51 | // Make both the serialization and write to file async 52 | return Task.Factory.StartNew(() => 53 | { 54 | var serialized = JsonConvert.SerializeObject(json, Formatting.Indented); 55 | File.WriteAllText(filename, serialized, Encoding.UTF8); 56 | }); 57 | } 58 | } 59 | ``` 60 | 61 | There are two properties and two methods to implement in the interface. The properties are 62 | 63 | - Name: Used for registering the output type. This will be used to find the type via the `--input` argument on the application. 64 | - ExpectedArgs: Used for the `--list-output` argument of the program to let the user know what arguments to pass 65 | 66 | The methods are: 67 | 68 | - PreloadArgs: Read and parse the provided arguments. This allows quick failure in case invalid arguments are given. 69 | - WriteBatchAsync: The bulk of the logic will be here. This will be HTTP calls, database writes, etc. They should be asynchronous. -------------------------------------------------------------------------------- /cblite/cbliteTool+encrypt.cc: -------------------------------------------------------------------------------- 1 | // 2 | // cbliteTool+encrypt.cc 3 | // 4 | // Copyright © 2019 Couchbase. All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "cbliteTool.hh" 20 | 21 | #ifdef COUCHBASE_ENTERPRISE 22 | 23 | void CBLiteTool::encryptUsage() { 24 | writeUsageCommand("encrypt", false); 25 | cerr << 26 | " --raw : Take an AES256 key (64 hex digits) instead of a password\n" 27 | " Encrypts the database, or changes the current encryption.\n" 28 | " Prompts securely for a new password/key.\n" 29 | ; 30 | } 31 | 32 | 33 | void CBLiteTool::decryptUsage() { 34 | writeUsageCommand("decrypt", false); 35 | cerr << 36 | " Removes database encryption.\n" 37 | ; 38 | } 39 | 40 | 41 | static bool decodeHex(const string &hex, uint8_t* bytes, size_t byteCount) { 42 | if (hex.size() != 2*byteCount) 43 | return false; 44 | for (size_t i=0; i 0) { 63 | cout << "Indexes: "; 64 | int n = 0; 65 | for (Array::iterator i(indexes); i; ++i) { 66 | if (n++) 67 | cout << ", "; 68 | cout << i.value().asString(); 69 | } 70 | cout << "\n"; 71 | } 72 | 73 | if (nBlobs > 0) { 74 | cout << "Blobs: " << nBlobs << ", "; writeSize(dbSize); cerr << "\n"; 75 | } 76 | 77 | C4UUID publicUUID, privateUUID; 78 | if (c4db_getUUIDs(_db, &publicUUID, &privateUUID, nullptr)) { 79 | cout << "UUIDs: public " 80 | << slice(&publicUUID, sizeof(publicUUID)).hexString().c_str() 81 | << ", private " << slice(&privateUUID, sizeof(privateUUID)).hexString().c_str() 82 | << "\n"; 83 | } 84 | } 85 | 86 | 87 | void CBLiteTool::writeSize(uint64_t n) { 88 | static const char* kScales[] = {" bytes", "KB", "MB", "GB"}; 89 | int scale = 0; 90 | while (n >= 1024 && scale < 3) { 91 | n = (n + 512) / 1024; 92 | ++scale; 93 | } 94 | cout << n << kScales[scale]; 95 | } 96 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator/Program_args.cs: -------------------------------------------------------------------------------- 1 | // 2 | // Program_args.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System; 20 | using System.ComponentModel.DataAnnotations; 21 | 22 | using LargeDatasetGenerator.Output; 23 | 24 | using McMaster.Extensions.CommandLineUtils; 25 | 26 | namespace LargeDatasetGenerator 27 | { 28 | [ProgramArgumentValidation] 29 | partial class Program 30 | { 31 | #region Properties 32 | 33 | [Option("--advanced ", Description = "An ini file containing advanced configuration options")] 34 | public string AdvancedFile { get; set; } 35 | 36 | [Option("--batch-size ", Description = "The number of entries to process as one unit (default 100)", 37 | ShortName = "b")] 38 | [Range(1, Int32.MaxValue)] 39 | public int BatchSize { get; set; } = 100; 40 | 41 | [Option("--count ", Description = "The number of entries to generate from the template (default 1000)", 42 | ShortName = "c")] 43 | [Range(1, Int32.MaxValue)] 44 | public int Count { get; set; } = 1000; 45 | 46 | [FileExists] 47 | [Option("--input ", Description = "The file to read the document template from", ShortName = "i")] 48 | public string Input { get; set; } 49 | 50 | [Option("--list-generators", Description = "List the available generator functions for the template JSON")] 51 | public bool ListGenerators { get; set; } 52 | 53 | [Option("--list-output", Description = "List the available output methods for writing JSON")] 54 | public bool ListOutputs { get; set; } 55 | 56 | [Option("--output ", Description = "The way to save the resulting JSON", ShortName = "o")] 57 | public string OutputType { get; set; } = JsonFileOutput.NameConst; 58 | 59 | public string[] RemainingArguments { get; } 60 | 61 | #endregion 62 | } 63 | 64 | internal sealed class ProgramArgumentValidationAttribute : ValidationAttribute 65 | { 66 | #region Overrides 67 | 68 | protected override ValidationResult IsValid(object value, ValidationContext validationContext) 69 | { 70 | if (value is Program p) { 71 | if (p.ListOutputs || p.ListGenerators) { 72 | return ValidationResult.Success; 73 | } 74 | 75 | if (p.Input == null) { 76 | return new ValidationResult("Required argument --input missing"); 77 | } 78 | } 79 | 80 | return ValidationResult.Success; 81 | } 82 | 83 | #endregion 84 | } 85 | } -------------------------------------------------------------------------------- /litecp/DirEndpoint.cc: -------------------------------------------------------------------------------- 1 | // 2 | // DirEndpoint.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "DirEndpoint.hh" 20 | 21 | 22 | void DirectoryEndpoint::prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint *other) { 23 | Endpoint::prepare(isSource, mustExist, 24 | (docIDProperty ? docIDProperty : "_id"_sl), 25 | other); 26 | if (_dir.exists()) { 27 | if (!_dir.existsAsDir()) 28 | Tool::instance->fail(format("%s is not a directory", _spec.c_str())); 29 | } else { 30 | if (isSource || mustExist) 31 | Tool::instance->fail(format("Directory %s doesn't exist", _spec.c_str())); 32 | else 33 | _dir.mkdir(); 34 | } 35 | } 36 | 37 | 38 | // As source: 39 | void DirectoryEndpoint::copyTo(Endpoint *dst, uint64_t limit) { 40 | if (Tool::instance->verbose()) 41 | cout << "Importing JSON files...\n"; 42 | alloc_slice buffer(10000); 43 | _dir.forEachFile([&](const FilePath &file) { 44 | string filename = file.fileName(); 45 | if (!hasSuffix(filename, ".json") || hasPrefix(filename, ".")) 46 | return; 47 | string docID = filename.substr(0, filename.size() - 5); 48 | 49 | slice json = readFile(file.path(), buffer); 50 | if (json) 51 | dst->writeJSON(docID, json); 52 | }); 53 | } 54 | 55 | 56 | // As destination: 57 | void DirectoryEndpoint::writeJSON(slice docID, slice json) { 58 | alloc_slice docIDBuf; 59 | if (!docID) { 60 | docID = docIDBuf = docIDFromJSON(json); 61 | if (!docID) 62 | return; 63 | } 64 | 65 | if (docID.size == 0 || docID[0] == '.' || docID.findByte(FilePath::kSeparator[0])) { 66 | Tool::instance->errorOccurred(format("writing doc \"%.*s\": doc ID cannot be used as a filename", SPLAT(docID))); 67 | return; 68 | } 69 | 70 | FilePath jsonFile = _dir[docID.asString() + ".json"]; 71 | ofstream out(jsonFile.path(), ios_base::trunc | ios_base::out); 72 | out << json << '\n'; 73 | logDocument(docID); 74 | } 75 | 76 | 77 | slice DirectoryEndpoint::readFile(const string &path, alloc_slice &buffer) { 78 | size_t readBytes = 0; 79 | ifstream in(path, ios_base::in); 80 | do { 81 | if (readBytes == buffer.size) 82 | buffer.resize(2*buffer.size); 83 | in.read((char*)buffer.buf + readBytes, buffer.size - readBytes); 84 | readBytes += in.gcount(); 85 | } while (in.good()); 86 | if (in.bad()) { 87 | Tool::instance->errorOccurred(format("reading file %s", path.c_str())); 88 | return nullslice; 89 | } 90 | return {buffer.buf, readBytes}; 91 | } 92 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/PluginLoader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Reflection; 6 | 7 | using LargeDatasetGenerator.Output; 8 | 9 | namespace LargeDatasetGenerator.Generator 10 | { 11 | internal static class PluginLoader 12 | { 13 | #region Constants 14 | 15 | public static readonly IReadOnlyList GeneratorTypes; 16 | public static readonly IReadOnlyList OutputTypes; 17 | 18 | #endregion 19 | 20 | #region Constructors 21 | 22 | static PluginLoader() 23 | { 24 | var generatorTypes = new List(); 25 | var outputTypes = new List(); 26 | var totalCount = 0; 27 | var invalidCount = 0; 28 | var assemblyDir = Path.GetDirectoryName(typeof(PluginLoader).Assembly.Location); 29 | var alreadyLoaded = 30 | new HashSet(AppDomain.CurrentDomain.GetAssemblies().Select(x => x.GetName().Name)); 31 | 32 | outputTypes.AddRange(LoadOutputTypes(Assembly.GetExecutingAssembly())); 33 | generatorTypes.AddRange(LoadGeneratorTypes(Assembly.GetExecutingAssembly())); 34 | foreach (var dll in Directory.EnumerateFiles(assemblyDir, "*.dll") 35 | .Where(x => !alreadyLoaded.Contains(Path.GetFileNameWithoutExtension(x)))) { 36 | try { 37 | var assembly = Assembly.LoadFile(Path.GetFullPath(dll)); 38 | outputTypes.AddRange(LoadOutputTypes(assembly)); 39 | generatorTypes.AddRange(LoadGeneratorTypes(assembly)); 40 | totalCount++; 41 | } catch (BadImageFormatException) { 42 | invalidCount++; 43 | } catch (Exception e) { 44 | Console.ForegroundColor = ConsoleColor.DarkYellow; 45 | PositionalConsole.WriteLine($"Skipping {dll} because it was not loadable: {e}"); 46 | Console.ResetColor(); 47 | } 48 | } 49 | 50 | if (invalidCount > 0) { 51 | PositionalConsole.WriteLine($"Skipped {invalidCount} of {totalCount} assemblies because they were not valid .NET", 52 | ConsoleColor.DarkYellow); 53 | } else { 54 | PositionalConsole.WriteLine($"Processed {totalCount} assemblies for plugin types"); 55 | } 56 | 57 | GeneratorTypes = generatorTypes; 58 | OutputTypes = outputTypes; 59 | } 60 | 61 | #endregion 62 | 63 | #region Private Methods 64 | 65 | private static IEnumerable LoadOutputTypes(Assembly assembly) 66 | { 67 | return assembly.GetExportedTypes() 68 | .Select(x => x.GetTypeInfo()) 69 | .Where(x => !x.IsAbstract && x.ImplementedInterfaces.Contains(typeof(IJsonOutput))); 70 | } 71 | 72 | private static IEnumerable LoadGeneratorTypes(Assembly assembly) 73 | { 74 | return assembly.GetExportedTypes() 75 | .Select(x => x.GetTypeInfo()) 76 | .Where(x => !x.IsAbstract && x.ImplementedInterfaces.Contains(typeof(IDataGenerator))); 77 | } 78 | 79 | #endregion 80 | } 81 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/GeneratorArgumentParser.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using sly.buildresult; 3 | using sly.lexer; 4 | 5 | namespace LargeDatasetGenerator.Generator 6 | { 7 | internal enum GeneratorTokenType 8 | { 9 | None, 10 | 11 | [Lexeme(GenericToken.KeyWord, "new")] New, 12 | 13 | [Lexeme(GenericToken.SugarToken, ",")] Comma, 14 | 15 | [Lexeme(GenericToken.SugarToken, "(")] OpenParen, 16 | 17 | [Lexeme(GenericToken.SugarToken, ")")] CloseParen, 18 | 19 | [Lexeme(GenericToken.Identifier, IdentifierType.Alpha)] 20 | Identifier, 21 | 22 | [Lexeme(GenericToken.String, "'", "\\")] [Lexeme(GenericToken.String, "\"", "\\")] 23 | String, 24 | 25 | [Lexeme(GenericToken.Int)] Int, 26 | 27 | [Lexeme(GenericToken.Double)] Double 28 | } 29 | 30 | internal abstract class GeneratorArgumentParser 31 | { 32 | #region Variables 33 | 34 | private readonly string _input; 35 | private readonly ILexer _lexer; 36 | 37 | #endregion 38 | 39 | #region Constructors 40 | 41 | protected GeneratorArgumentParser(string input) 42 | { 43 | _input = input; 44 | var buildResult = LexerBuilder.BuildLexer(new BuildResult>()); 45 | if (buildResult.IsError) { 46 | throw new ApplicationException("Lexer failed to build"); 47 | } 48 | 49 | _lexer = buildResult.Result; 50 | } 51 | 52 | #endregion 53 | 54 | #region Public Methods 55 | 56 | public T Parse() 57 | { 58 | var parenthesisCount = 0; 59 | Token previous = new Token {TokenID = GeneratorTokenType.None}; 60 | foreach (var token in _lexer.Tokenize(_input)) { 61 | switch (token.TokenID) { 62 | case GeneratorTokenType.OpenParen: 63 | parenthesisCount++; 64 | break; 65 | case GeneratorTokenType.CloseParen: 66 | parenthesisCount--; 67 | if (parenthesisCount < 0) { 68 | throw new InvalidOperationException($"Syntax error ')' at {token.Position}"); 69 | } 70 | 71 | break; 72 | default: 73 | if (!token.End && parenthesisCount == 0) { 74 | throw new InvalidOperationException($"Syntax error '{token.StringWithoutQuotes} at {token.Position}"); 75 | } 76 | 77 | break; 78 | } 79 | 80 | AcceptToken(token, previous); 81 | previous = token; 82 | } 83 | 84 | return Finish(); 85 | } 86 | 87 | #endregion 88 | 89 | #region Protected Methods 90 | 91 | protected abstract void AcceptToken(Token token, Token previous); 92 | 93 | protected abstract T Finish(); 94 | 95 | protected void ThrowSyntaxError(Token token) 96 | { 97 | if (token.End) { 98 | return; 99 | } 100 | 101 | throw new InvalidOperationException($"Syntax error '{token.StringWithoutQuotes}' at {token.Position}"); 102 | } 103 | 104 | #endregion 105 | } 106 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/PositionalConsole.cs: -------------------------------------------------------------------------------- 1 | // 2 | // PositionalConsole.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System; 20 | using System.Text; 21 | 22 | namespace LargeDatasetGenerator 23 | { 24 | internal static class PositionalConsole 25 | { 26 | #region Constants 27 | 28 | private static readonly object Locker = new object(); 29 | 30 | #endregion 31 | 32 | #region Public Methods 33 | 34 | public static void SetReserved(int lineCount) 35 | { 36 | Console.SetCursorPosition(0, Math.Max(Console.CursorTop, lineCount)); 37 | } 38 | 39 | public static void WriteLine(string line, ConsoleColor? color = null) 40 | { 41 | lock (Locker) { 42 | if (color != null) { 43 | Console.ForegroundColor = color.Value; 44 | } 45 | 46 | Console.WriteLine(line); 47 | 48 | if (color != null) { 49 | Console.ResetColor(); 50 | } 51 | } 52 | } 53 | 54 | public static void WriteLineAt(int left, int top, string line) 55 | { 56 | lock (Locker) { 57 | var oldLeft = Console.CursorLeft; 58 | var oldTop = Console.CursorTop; 59 | Console.SetCursorPosition(left, top); 60 | Console.WriteLine(line); 61 | Console.SetCursorPosition(oldLeft, oldTop); 62 | } 63 | } 64 | 65 | #endregion 66 | } 67 | 68 | internal sealed class ProgressOutput 69 | { 70 | #region Constants 71 | 72 | private static readonly char[] Indicators = new[] {'|', '/', '-', '\\'}; 73 | 74 | #endregion 75 | 76 | #region Variables 77 | 78 | private readonly int _step; 79 | private int _callCount; 80 | 81 | #endregion 82 | 83 | #region Constructors 84 | 85 | public ProgressOutput(int step) 86 | { 87 | _step = step; 88 | } 89 | 90 | #endregion 91 | 92 | #region Public Methods 93 | 94 | public string Render(int completed, int total) 95 | { 96 | if (completed == total) { 97 | return " 　[==========] 100%"; 98 | } 99 | 100 | _callCount++; 101 | char indicator = Indicators[(_callCount / _step) % Indicators.Length]; 102 | int blockCount = (int) Math.Floor((double) completed / total * 10.0); 103 | var sb = new StringBuilder("["); 104 | for (int i = 0; i < 10; i++) { 105 | sb.Append(i < blockCount ? "=" : " "); 106 | } 107 | 108 | sb.AppendFormat("] {0}%", (int)((double)completed / total * 100.0)); 109 | return $"{indicator} {sb}"; 110 | } 111 | 112 | #endregion 113 | } 114 | } -------------------------------------------------------------------------------- /Xcode/Tools.xcodeproj/xcshareddata/xcschemes/cblite.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 32 | 33 | 39 | 40 | 41 | 42 | 43 | 44 | 54 | 56 | 62 | 63 | 64 | 65 | 66 | 67 | 73 | 75 | 81 | 82 | 83 | 84 | 86 | 87 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /litecp/Endpoint.hh: -------------------------------------------------------------------------------- 1 | // 2 | // Endpoint.hh 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | #include "Tool.hh" 21 | #include 22 | 23 | using namespace fleece; 24 | 25 | 26 | /** Abstract base class for a source or target of copying/replication. */ 27 | class Endpoint { 28 | public: 29 | Endpoint(const string &spec) 30 | :_spec(spec) 31 | { } 32 | 33 | static Endpoint* create(string str); 34 | static Endpoint* create(C4Database*); 35 | virtual ~Endpoint() { } 36 | 37 | virtual bool isDatabase() const {return false;} 38 | virtual bool isRemote() const {return false;} 39 | 40 | virtual void prepare(bool isSource, bool mustExist, slice docIDProperty, const Endpoint *other) { 41 | _docIDProperty = docIDProperty; 42 | if (_docIDProperty) { 43 | _docIDPath.reset(new KeyPath(_docIDProperty, nullptr)); 44 | if (!*_docIDPath) 45 | Tool::instance->fail("Invalid docID"); 46 | } 47 | } 48 | 49 | virtual void copyTo(Endpoint*, uint64_t limit) =0; 50 | 51 | virtual void writeJSON(slice docID, slice json) = 0; 52 | 53 | virtual void finish() { } 54 | 55 | uint64_t docCount() {return _docCount;} 56 | void setDocCount(uint64_t n) {_docCount = n;} 57 | 58 | void logDocument(slice docID) { 59 | ++_docCount; 60 | if (Tool::instance->verbose() >= 2) 61 | cout << to_string(docID) << '\n'; 62 | else if (Tool::instance->verbose() == 1 && (_docCount % 1000) == 0) 63 | cout << _docCount << '\n'; 64 | } 65 | 66 | void logDocuments(unsigned n) { 67 | _docCount += n; 68 | if (Tool::instance->verbose() >= 2) 69 | cout << n << " more documents\n"; 70 | else if (Tool::instance->verbose() == 1 && (_docCount % 1000) < n) 71 | cout << _docCount << '\n'; 72 | } 73 | 74 | protected: 75 | 76 | alloc_slice docIDFromJSON(slice json) { 77 | alloc_slice body = Doc::fromJSON(json, nullptr).allocedData(); 78 | return docIDFromFleece(body, json); 79 | } 80 | 81 | alloc_slice docIDFromFleece(slice body, slice json) { 82 | alloc_slice docIDBuf; 83 | Dict root = Value::fromData(body).asDict(); 84 | Value docIDProp = root[*_docIDPath]; 85 | if (docIDProp) { 86 | docIDBuf = docIDProp.toString(); 87 | if (!docIDBuf) 88 | Tool::instance->fail(format("Property \"%.*s\" is not a scalar in JSON: %.*s", SPLAT(_docIDProperty), SPLAT(json))); 89 | } else { 90 | Tool::instance->errorOccurred(format("No property \"%.*s\" in JSON: %.*s", SPLAT(_docIDProperty), SPLAT(json))); 91 | } 92 | return docIDBuf; 93 | } 94 | 95 | const string _spec; 96 | Encoder _encoder; 97 | alloc_slice _docIDProperty; 98 | uint64_t _docCount {0}; 99 | unique_ptr _docIDPath; 100 | }; 101 | 102 | 103 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator/Program.cs: -------------------------------------------------------------------------------- 1 | // 2 | // Program.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System; 20 | using System.IO; 21 | 22 | using McMaster.Extensions.CommandLineUtils; 23 | 24 | using Microsoft.Extensions.Configuration; 25 | 26 | namespace LargeDatasetGenerator 27 | { 28 | [Command(Name = "LargeDatasetGenerator", Description = "Generates data into a JSON accepting endpoint", 29 | ThrowOnUnexpectedArgument = false)] 30 | [HelpOption("-?")] 31 | partial class Program 32 | { 33 | #region Private Methods 34 | 35 | static void Main(string[] args) 36 | { 37 | try { 38 | CommandLineApplication.Execute(args); 39 | } finally { 40 | Console.CursorVisible = true; 41 | } 42 | } 43 | 44 | private void ExtractMultiplierValue(IConfiguration config, string key, Action processor) 45 | { 46 | var value = config[key]; 47 | if (value == null) { 48 | return; 49 | } 50 | 51 | var numericPortion = Double.Parse(value.TrimEnd('*')); 52 | if (value.EndsWith('*')) { 53 | numericPortion *= Environment.ProcessorCount; 54 | } 55 | 56 | Console.WriteLine($"Using custom value {key}: {value}..."); 57 | processor(numericPortion); 58 | } 59 | 60 | private void ExtractValue(IConfiguration config, string key, Action processor) where T : IConvertible 61 | { 62 | var value = config[key]; 63 | if (value == null) { 64 | return; 65 | } 66 | 67 | Console.WriteLine($"Using custom value {key}: {value}..."); 68 | processor((T) Convert.ChangeType(value, typeof(T))); 69 | } 70 | 71 | private void OnExecute() 72 | { 73 | var configBuilder = new ConfigurationBuilder().AddEnvironmentVariables("LDG_"); 74 | if (AdvancedFile != null) { 75 | configBuilder.AddIniFile(Path.GetFullPath(AdvancedFile), false); 76 | } 77 | 78 | var passedConfig = configBuilder.Build(); 79 | var generatorConfig = LargeDatasetConfiguration.CreateDefault(); 80 | ExtractMultiplierValue(passedConfig, "create_job_count", x => generatorConfig.CreateJobCount = (int) x); 81 | ExtractMultiplierValue(passedConfig, "write_job_count", x => generatorConfig.WriteJobCount = (int) x); 82 | ExtractValue(passedConfig, "queue_size_multiplier", x => generatorConfig.QueueSizeMultiplier = x); 83 | 84 | var generator = new LargeDatasetGenerator(generatorConfig) 85 | { 86 | BatchSize = BatchSize, 87 | Count = Count 88 | }; 89 | 90 | if (ListOutputs) { 91 | generator.ListOutput(); 92 | return; 93 | } 94 | 95 | if (ListGenerators) { 96 | generator.ListGenerators(); 97 | return; 98 | } 99 | 100 | generator.Generate(Input, OutputType, RemainingArguments); 101 | } 102 | 103 | #endregion 104 | } 105 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/DateObjectArgs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | using LargeDatasetGenerator.Generator; 4 | 5 | using sly.lexer; 6 | 7 | namespace LargeDatasetGenerator.Core.Generator 8 | { 9 | internal sealed class DateObjectArgs 10 | { 11 | #region Variables 12 | 13 | private DateTimeComponent _currentComponent; 14 | 15 | #endregion 16 | 17 | #region Properties 18 | 19 | public DateTimeOffset DateTime { get; private set; } 20 | 21 | #endregion 22 | 23 | #region Constructors 24 | 25 | public DateObjectArgs(DateTimeOffset defaultValue) 26 | { 27 | DateTime = defaultValue; 28 | } 29 | 30 | #endregion 31 | 32 | #region Public Methods 33 | 34 | public bool Accept(Token token, Token previous) 35 | { 36 | if (token.TokenID == GeneratorTokenType.Int) { 37 | DateTime = SetDateTimeComponent(token.IntValue); 38 | return previous.TokenID == GeneratorTokenType.OpenParen || previous.TokenID == GeneratorTokenType.Comma; 39 | } 40 | 41 | if (token.TokenID == GeneratorTokenType.Comma) { 42 | _currentComponent++; 43 | return (previous.TokenID == GeneratorTokenType.Int || previous.TokenID == GeneratorTokenType.OpenParen) 44 | && _currentComponent <= DateTimeComponent.Second; 45 | } 46 | 47 | if (token.TokenID == GeneratorTokenType.CloseParen) { 48 | return previous.TokenID == GeneratorTokenType.OpenParen || previous.TokenID == GeneratorTokenType.Int; 49 | } 50 | 51 | return false; 52 | } 53 | 54 | #endregion 55 | 56 | #region Private Methods 57 | 58 | private DateTimeOffset SetDateTimeComponent(int value) 59 | { 60 | switch (_currentComponent) { 61 | case DateTimeComponent.Year: 62 | return new DateTimeOffset(value, DateTime.Month, DateTime.Day, DateTime.Hour, DateTime.Minute, 63 | DateTime.Second, 64 | DateTime.Offset); 65 | case DateTimeComponent.Month: 66 | return new DateTimeOffset(DateTime.Year, value, DateTime.Day, DateTime.Hour, DateTime.Minute, 67 | DateTime.Second, 68 | DateTime.Offset); 69 | case DateTimeComponent.Day: 70 | return new DateTimeOffset(DateTime.Year, DateTime.Month, value, DateTime.Hour, DateTime.Minute, 71 | DateTime.Second, 72 | DateTime.Offset); 73 | case DateTimeComponent.Hour: 74 | return new DateTimeOffset(DateTime.Year, DateTime.Month, DateTime.Day, value, DateTime.Minute, 75 | DateTime.Second, 76 | DateTime.Offset); 77 | case DateTimeComponent.Minute: 78 | return new DateTimeOffset(DateTime.Year, DateTime.Month, DateTime.Day, DateTime.Hour, value, 79 | DateTime.Second, 80 | DateTime.Offset); 81 | case DateTimeComponent.Second: 82 | return new DateTimeOffset(DateTime.Year, DateTime.Month, DateTime.Day, DateTime.Hour, 83 | DateTime.Minute, value, 84 | DateTime.Offset); 85 | default: 86 | throw new ArgumentException("Invalid date time received"); 87 | } 88 | } 89 | 90 | #endregion 91 | 92 | private enum DateTimeComponent 93 | { 94 | Year, 95 | Month, 96 | Day, 97 | Hour, 98 | Minute, 99 | Second 100 | } 101 | } 102 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.421 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LargeDatasetGenerator.CouchbaseServer", "LargeDatasetGenerator.CouchbaseServer\LargeDatasetGenerator.CouchbaseServer.csproj", "{9C88D042-7375-4601-9DA2-D71579E532E0}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LargeDatasetGenerator.Abstractions", "LargeDatasetGenerator.Abstractions\LargeDatasetGenerator.Abstractions.csproj", "{B00DD073-EDB1-4BD4-9AC4-11384881BD2E}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LargeDatasetGenerator.SyncGateway", "LargeDatasetGenerator.SyncGateway\LargeDatasetGenerator.SyncGateway.csproj", "{9EFE3E6A-66CB-4DF4-B33D-C6E94AABB0CD}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LargeDatasetGenerator.CouchbaseLite", "LargeDatasetGenerator.CouchbaseLite\LargeDatasetGenerator.CouchbaseLite.csproj", "{4347B62E-6F20-40D3-8E5D-3D644153E2D3}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LargeDatasetGenerator.Core", "LargeDatasetGenerator.Core\LargeDatasetGenerator.Core.csproj", "{89E7CC78-6C22-469C-987A-31F41B7DCBE8}" 15 | EndProject 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LargeDatasetGenerator", "LargeDatasetGenerator\LargeDatasetGenerator.csproj", "{142D0C0C-7F10-4481-B337-4567E90C820F}" 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|Any CPU = Debug|Any CPU 21 | Release|Any CPU = Release|Any CPU 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {9C88D042-7375-4601-9DA2-D71579E532E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {9C88D042-7375-4601-9DA2-D71579E532E0}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {9C88D042-7375-4601-9DA2-D71579E532E0}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {9C88D042-7375-4601-9DA2-D71579E532E0}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {B00DD073-EDB1-4BD4-9AC4-11384881BD2E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {B00DD073-EDB1-4BD4-9AC4-11384881BD2E}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {B00DD073-EDB1-4BD4-9AC4-11384881BD2E}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {B00DD073-EDB1-4BD4-9AC4-11384881BD2E}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {9EFE3E6A-66CB-4DF4-B33D-C6E94AABB0CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {9EFE3E6A-66CB-4DF4-B33D-C6E94AABB0CD}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {9EFE3E6A-66CB-4DF4-B33D-C6E94AABB0CD}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {9EFE3E6A-66CB-4DF4-B33D-C6E94AABB0CD}.Release|Any CPU.Build.0 = Release|Any CPU 36 | {4347B62E-6F20-40D3-8E5D-3D644153E2D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {4347B62E-6F20-40D3-8E5D-3D644153E2D3}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {4347B62E-6F20-40D3-8E5D-3D644153E2D3}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {4347B62E-6F20-40D3-8E5D-3D644153E2D3}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {89E7CC78-6C22-469C-987A-31F41B7DCBE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {89E7CC78-6C22-469C-987A-31F41B7DCBE8}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {89E7CC78-6C22-469C-987A-31F41B7DCBE8}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {89E7CC78-6C22-469C-987A-31F41B7DCBE8}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {142D0C0C-7F10-4481-B337-4567E90C820F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {142D0C0C-7F10-4481-B337-4567E90C820F}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {142D0C0C-7F10-4481-B337-4567E90C820F}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {142D0C0C-7F10-4481-B337-4567E90C820F}.Release|Any CPU.Build.0 = Release|Any CPU 48 | EndGlobalSection 49 | GlobalSection(SolutionProperties) = preSolution 50 | HideSolutionNode = FALSE 51 | EndGlobalSection 52 | GlobalSection(ExtensibilityGlobals) = postSolution 53 | SolutionGuid = {E8DDB52F-4423-412D-B8C1-0BA55C9C010B} 54 | EndGlobalSection 55 | EndGlobal 56 | -------------------------------------------------------------------------------- /Xcode/Tools.xcodeproj/xcshareddata/xcschemes/cbl-log.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 32 | 33 | 39 | 40 | 41 | 42 | 43 | 44 | 57 | 59 | 65 | 66 | 67 | 68 | 72 | 73 | 74 | 75 | 81 | 83 | 89 | 90 | 91 | 92 | 94 | 95 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /gen_errors/objc.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2017 Couchbase, Inc All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | from datetime import datetime 18 | 19 | 20 | class ObjCFormatter: 21 | """Formatter for Obj-C symbol definitions""" 22 | name = "Obj-C" 23 | year = datetime.today().year 24 | 25 | m_msg_file_name = "CBLErrorMessage.m" 26 | h_msg_file_name = "CBLErrorMessage.h" 27 | 28 | m_message_def = "NSString* const kCBLErrorMessage{0} = @\"{1}\";\n" 29 | h_message_def = "extern NSString* const kCBLErrorMessage{0};\n" 30 | 31 | m_trailer = """ 32 | @end 33 | 34 | """ 35 | 36 | h_trailer = """ 37 | @end 38 | 39 | NS_ASSUME_NONNULL_END 40 | 41 | """ 42 | 43 | m_header = """// 44 | // CBLErrorMessage.m 45 | // CouchbaseLite 46 | // 47 | // Copyright (c) {0} Couchbase, Inc All rights reserved. 48 | // 49 | // Licensed under the Apache License, Version 2.0 (the "License"); 50 | // you may not use this file except in compliance with the License. 51 | // You may obtain a copy of the License at 52 | // 53 | // http://www.apache.org/licenses/LICENSE-2.0 54 | // 55 | // Unless required by applicable law or agreed to in writing, software 56 | // distributed under the License is distributed on an "AS IS" BASIS, 57 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 58 | // See the License for the specific language governing permissions and 59 | // limitations under the License. 60 | // 61 | 62 | #import "CBLErrorMessage.h" 63 | 64 | @implementation CBLErrorMessage 65 | 66 | """.format(year) 67 | 68 | h_header = """// 69 | // CBLErrorMessage.h 70 | // CouchbaseLite 71 | // 72 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 73 | // 74 | // Licensed under the Apache License, Version 2.0 (the "License"); 75 | // you may not use this file except in compliance with the License. 76 | // You may obtain a copy of the License at 77 | // 78 | // http://www.apache.org/licenses/LICENSE-2.0 79 | // 80 | // Unless required by applicable law or agreed to in writing, software 81 | // distributed under the License is distributed on an "AS IS" BASIS, 82 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 83 | // See the License for the specific language governing permissions and 84 | // limitations under the License. 85 | // 86 | 87 | #import 88 | 89 | NS_ASSUME_NONNULL_BEGIN 90 | 91 | @interface CBLErrorMessage : NSObject 92 | 93 | """ 94 | 95 | def __init__(self, out_dir): 96 | self.m_out_file = os.path.join(out_dir, ObjCFormatter.m_msg_file_name) 97 | self.h_out_file = os.path.join(out_dir, ObjCFormatter.h_msg_file_name) 98 | 99 | def format(self, errors): 100 | with open(self.m_out_file, "w") as message_file: 101 | message_file.write(ObjCFormatter.m_header) 102 | for error in errors: 103 | message = error["message"].format("%1$@", "%2$@", "%3$@", "%4$@", "%4$@") 104 | message_file.write(ObjCFormatter.m_message_def.format(error["name"], message)) 105 | message_file.write(ObjCFormatter.m_trailer) 106 | 107 | with open(self.h_out_file, "w") as message_file: 108 | message_file.write(ObjCFormatter.h_header) 109 | for error in errors: 110 | message_file.write(ObjCFormatter.h_message_def.format(error["name"])) 111 | message_file.write(ObjCFormatter.h_trailer) 112 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.CouchbaseLite/CouchbaseLiteOutput.cs: -------------------------------------------------------------------------------- 1 | // 2 | // CouchbaseLiteOutput.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Threading.Tasks; 22 | 23 | using Couchbase.Lite; 24 | 25 | using LargeDatasetGenerator.Output; 26 | 27 | using McMaster.Extensions.CommandLineUtils; 28 | 29 | namespace LargeDatasetGenerator.CouchbaseLite 30 | { 31 | /// 32 | /// An implementation that writes to a Couchbase 33 | /// Lite database 34 | /// 35 | public class CouchbaseLiteOutput : IJsonOutput, IDisposable 36 | { 37 | #region Variables 38 | 39 | private Database _database; 40 | 41 | #endregion 42 | 43 | #region Properties 44 | 45 | /// 46 | public string ExpectedArgs { get; } = "--name [--path ]"; 47 | 48 | /// 49 | public string Name { get; } = "couchbase_lite"; 50 | 51 | #endregion 52 | 53 | #region Constructors 54 | 55 | static CouchbaseLiteOutput() 56 | { 57 | Couchbase.Lite.Support.NetDesktop.Activate(); 58 | } 59 | 60 | #endregion 61 | 62 | #region IDisposable 63 | 64 | /// 65 | public void Dispose() 66 | { 67 | _database.Dispose(); 68 | _database = null; 69 | } 70 | 71 | #endregion 72 | 73 | #region IJsonOutput 74 | 75 | /// 76 | public bool PreloadArgs(string[] args) 77 | { 78 | var subApp = new CommandLineApplication(); 79 | 80 | var name = subApp.Option("--name ", "The database name to use for saving", 81 | CommandOptionType.SingleValue); 82 | name.IsRequired(); 83 | 84 | var path = subApp.Option("--path ", "The path to save the database in (optional)", 85 | CommandOptionType.SingleValue); 86 | 87 | subApp.OnExecute(() => 88 | { 89 | var dbName = name.ParsedValue; 90 | DatabaseConfiguration config = null; 91 | if (path.HasValue()) { 92 | config = new DatabaseConfiguration {Directory = path.ParsedValue}; 93 | } 94 | 95 | _database = new Database(dbName, config); 96 | }); 97 | 98 | return subApp.Execute(args) == 0; 99 | } 100 | 101 | /// 102 | public Task WriteBatchAsync(IEnumerable> json) 103 | { 104 | return Task.Factory.StartNew(() => 105 | { 106 | _database.InBatch(() => 107 | { 108 | foreach (var content in json) { 109 | var contentCopy = new Dictionary(content); 110 | var id = contentCopy["_id"] as string; 111 | contentCopy.Remove("_id"); 112 | 113 | using (var doc = new MutableDocument(id, contentCopy)) { 114 | _database.Save(doc); 115 | } 116 | } 117 | }); 118 | }); 119 | } 120 | 121 | #endregion 122 | } 123 | } -------------------------------------------------------------------------------- /ArgumentTokenizer.cc: -------------------------------------------------------------------------------- 1 | // 2 | // ArgumentTokenizer.cc 3 | // 4 | // Copyright (c) 2018 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "ArgumentTokenizer.hh" 20 | #include 21 | using namespace std; 22 | 23 | 24 | void ArgumentTokenizer::reset(const char *input) { 25 | _input = input; 26 | _current = _input.c_str(); 27 | _args.clear(); 28 | next(); 29 | } 30 | 31 | 32 | void ArgumentTokenizer::reset(std::vector args) { 33 | _args = args; 34 | _current = nullptr; 35 | _startOfArg = nullptr; 36 | next(); 37 | } 38 | 39 | 40 | bool ArgumentTokenizer::next() { 41 | _hasArgument = true; 42 | if (!_args.empty()) { 43 | _argument = _args[0]; 44 | _args.erase(_args.begin()); 45 | return true; 46 | } 47 | 48 | if (_current) { 49 | _startOfArg = _current; 50 | char quoteChar = 0; 51 | bool inQuote = false; 52 | bool forceAppend = false; 53 | string nextArg; 54 | while(*_current) { 55 | char c = *_current; 56 | _current++; 57 | if(c == '\r' || c == '\n') { 58 | continue; 59 | } 60 | 61 | if(!forceAppend) { 62 | if(c == '\\') { 63 | forceAppend = true; 64 | continue; 65 | } else if(c == '"' || c == '\'') { 66 | if(quoteChar != 0 && c == quoteChar) { 67 | inQuote = false; 68 | quoteChar = 0; 69 | continue; 70 | } else if(quoteChar == 0) { 71 | inQuote = true; 72 | quoteChar = c; 73 | continue; 74 | } 75 | } else if(c == ' ' && !inQuote) { 76 | if (!nextArg.empty()) { 77 | _argument = nextArg; 78 | return true; 79 | } 80 | continue; 81 | } 82 | } else { 83 | forceAppend = false; 84 | } 85 | 86 | nextArg.append(1, c); 87 | } 88 | 89 | if(inQuote) 90 | throw runtime_error("Invalid input line: Unclosed quote"); 91 | if (forceAppend) 92 | throw runtime_error("Invalid input line: missing character after '\\'"); 93 | 94 | _current = nullptr; 95 | if(nextArg.length() > 0) { 96 | _argument = nextArg; 97 | return true; 98 | } 99 | } 100 | reset(); 101 | return false; 102 | } 103 | 104 | 105 | void ArgumentTokenizer::reset() { 106 | _args.clear(); 107 | _input.clear(); 108 | _current = nullptr; 109 | _hasArgument = false; 110 | _startOfArg = nullptr; 111 | _argument = ""; 112 | } 113 | 114 | 115 | string ArgumentTokenizer::restOfInput() { 116 | string result; 117 | if (_startOfArg) { 118 | result = string(_startOfArg); 119 | } else if (_hasArgument) { 120 | stringstream rest; 121 | rest << _argument; 122 | for (const string &arg : _args) 123 | rest << ' ' << arg; 124 | result = rest.str(); 125 | } 126 | reset(); 127 | return result; 128 | } 129 | 130 | 131 | bool ArgumentTokenizer::tokenize(const char *input, std::vector &outArgs) { 132 | if (!input) 133 | return false; 134 | try { 135 | reset(input); 136 | for (outArgs.clear(); hasArgument(); next()) 137 | outArgs.push_back(argument()); 138 | return true; 139 | } catch (const runtime_error &x) { 140 | return false; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /cblite/cbliteTool+put.cc: -------------------------------------------------------------------------------- 1 | // 2 | // cbliteTool+put.cc 3 | // 4 | // Copyright (c) 2018 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "cbliteTool.hh" 20 | #include 21 | 22 | 23 | const Tool::FlagSpec CBLiteTool::kPutFlags[] = { 24 | {"--create", (FlagHandler)&CBLiteTool::createDocFlag}, 25 | {"--update", (FlagHandler)&CBLiteTool::updateDocFlag}, 26 | {"--delete", (FlagHandler)&CBLiteTool::deleteDocFlag}, 27 | {nullptr, nullptr} 28 | }; 29 | 30 | void CBLiteTool::putUsage() { 31 | writeUsageCommand("put", true, "DOCID \"JSON\""); 32 | cerr << 33 | " Updates a document.\n" 34 | " --create : Document must not exist\n" 35 | " --delete : Deletes the document (and JSON is optional); same as `rm` subcommand\n" 36 | " --update : Document must already exist\n" 37 | " -- : End of arguments (use if DOCID starts with '-')\n" 38 | " " << it("DOCID") << " : Document ID\n" 39 | " " << it("JSON") << " : Document body as JSON (JSON5 syntax allowed.) Must be quoted.\n" 40 | ; 41 | 42 | writeUsageCommand("rm", false, "DOCID"); 43 | cerr << 44 | " Deletes a document. (Same as `put --delete`)\n" 45 | " " << it("DOCID") << " : Document ID\n" 46 | ; 47 | } 48 | 49 | 50 | void CBLiteTool::putDoc() { 51 | _putMode = (_currentCommand == "rm") ? kDelete : kPut; 52 | 53 | // Read params: 54 | processFlags(kPutFlags); 55 | if (_showHelp) { 56 | putUsage(); 57 | return; 58 | } 59 | 60 | openWriteableDatabaseFromNextArg(); 61 | 62 | string docID = nextArg("document ID"); 63 | string json5; 64 | if (_putMode != kDelete) 65 | json5 = nextArg("document body as JSON"); 66 | endOfArgs(); 67 | 68 | C4Error error; 69 | c4::Transaction t(_db); 70 | if (!t.begin(&error)) 71 | fail("Couldn't open database transaction"); 72 | 73 | c4::ref doc = c4doc_get(_db, slice(docID), false, &error); 74 | if (!doc) 75 | fail("Couldn't read document", error); 76 | bool existed = (doc->flags & kDocExists) != 0 && (doc->selectedRev.flags & kRevDeleted) == 0; 77 | if (!existed && (_putMode == kUpdate || _putMode == kDelete)) { 78 | if (doc->flags & kDocExists) 79 | fail("Document is already deleted"); 80 | else 81 | fail("Document doesn't exist"); 82 | } 83 | if (existed && _putMode == kCreate) 84 | fail("Document already exists"); 85 | 86 | alloc_slice body; 87 | if (_putMode != kDelete) { 88 | FLStringResult errMsg; 89 | alloc_slice json = FLJSON5_ToJSON(slice(json5), &errMsg, nullptr, nullptr); 90 | if (!json) { 91 | string message = string(alloc_slice(errMsg)); 92 | FLSliceResult_Free(errMsg); 93 | fail("Invalid JSON: " + message); 94 | } 95 | body = c4db_encodeJSON(_db, json, &error); 96 | if (!body) 97 | fail("Couldn't encode body", error); 98 | } 99 | 100 | doc = c4doc_update(doc, body, (_putMode == kDelete ? kRevDeleted : 0), &error); 101 | if (!doc) 102 | fail("Couldn't save document", error); 103 | 104 | if (!t.commit(&error)) 105 | fail("Couldn't commit database transaction", error); 106 | 107 | const char *verb; 108 | if (_putMode == kDelete) 109 | verb = "Deleted"; 110 | else if (existed) 111 | verb = "Updated"; 112 | else 113 | verb = "Created"; 114 | string revID = slice(doc->selectedRev.revID).asString(); 115 | if (revID.size() > 10) 116 | revID.resize(10); 117 | cout << verb << " `" << docID 118 | << "`, new revision " << revID << " (sequence " << doc->sequence << ")\n"; 119 | } 120 | -------------------------------------------------------------------------------- /Xcode/Tools.xcodeproj/xcshareddata/xcschemes/Tests.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 32 | 33 | 39 | 40 | 41 | 42 | 43 | 44 | 57 | 59 | 65 | 66 | 67 | 68 | 71 | 72 | 75 | 76 | 77 | 78 | 82 | 83 | 84 | 85 | 91 | 93 | 99 | 100 | 101 | 102 | 104 | 105 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /cbl-log/cbl-log.cc: -------------------------------------------------------------------------------- 1 | // 2 | // cbl-log.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "LogDecoder.hh" 20 | #include "Tool.hh" 21 | #include 22 | #include 23 | using namespace std; 24 | 25 | class CBLLogCat : public Tool 26 | { 27 | public: 28 | CBLLogCat() : Tool("cbl-log") {} 29 | void usage() override; 30 | int run() override; 31 | private: 32 | void logcatUsage(); 33 | void logcat(); 34 | void helpCommand(); 35 | 36 | static const FlagSpec kSubcommands[]; 37 | 38 | string _currentCommand; 39 | bool _showHelp {false}; 40 | }; 41 | 42 | const Tool::FlagSpec CBLLogCat::kSubcommands[] = { 43 | {"help", (FlagHandler)&CBLLogCat::helpCommand}, 44 | {"logcat", (FlagHandler)&CBLLogCat::logcat}, 45 | {nullptr, nullptr} 46 | }; 47 | 48 | int main(int argc, const char * argv[]) { 49 | CBLLogCat tool; 50 | return tool.main(argc, argv); 51 | } 52 | 53 | void CBLLogCat::usage() { 54 | cerr << 55 | ansiBold() << "cbl-log: Couchbase Lite / LiteCore log decoder\n" << ansiReset() << 56 | "Usage: cbl-log help " << it("[SUBCOMMAND]") << "\n" 57 | " cbl-log logcat " << it("LOG_FILE [OUTPUT_FILE]") << "]\n" 58 | "For information about subcommand parameters/flags, run `cbl-log help SUBCOMMAND`.\n" 59 | ; 60 | } 61 | 62 | int CBLLogCat::run() { 63 | if (!hasArgs()) { 64 | cerr << ansiBold() 65 | << "cbl-log: Couchbase Lite / LiteCore log decoder\n" << ansiReset() 66 | << "Missing subcommand.\n" 67 | << "For a list of subcommands, run " << ansiBold() << "cbl-log help" << ansiReset() << ".\n"; 68 | fail(); 69 | } 70 | 71 | string cmd = nextArg("subcommand or database path"); 72 | if (!processFlag(cmd, kSubcommands)) { 73 | _currentCommand = ""; 74 | failMisuse(format("Unknown subcommand '%s'", cmd.c_str())); 75 | } 76 | 77 | return 0; 78 | } 79 | 80 | void CBLLogCat::logcatUsage() { 81 | cerr << ansiBold(); 82 | cerr << "cbl-log logcat" << ' ' << it("LOG_FILE [OUTPUT_FILE]") << "\n"; 83 | cerr << 84 | " Converts a binary log file to text and writes it to stdout or the given output path\n" 85 | ; 86 | } 87 | 88 | void CBLLogCat::logcat() { 89 | // Read params: 90 | processFlags(nullptr); 91 | if (_showHelp) { 92 | logcatUsage(); 93 | return; 94 | } 95 | string logPath = nextArg("log file path"); 96 | string outputPath = peekNextArg(); 97 | 98 | vector kLevels = {"***", "", "", "WARNING", "ERROR"}; 99 | if(outputPath.empty()) { 100 | kLevels[3] = ansiBold() + ansiRed() + kLevels[3] + ansiReset(); 101 | kLevels[4] = ansiBold() + ansiRed() + kLevels[4] + ansiReset(); 102 | } 103 | 104 | ifstream in(logPath, ifstream::in | ifstream::binary); 105 | if (!in) 106 | fail(format("Couldn't open '%s'", logPath.c_str())); 107 | in.exceptions(std::ifstream::badbit); 108 | 109 | try { 110 | LogDecoder decoder(in); 111 | if(outputPath.empty()) { 112 | decoder.decodeTo(cout, kLevels); 113 | } else { 114 | ofstream fout(outputPath); 115 | decoder.decodeTo(fout, kLevels); 116 | } 117 | } catch (const LogDecoder::error &x) { 118 | fail(format("reading log file: %s", x.what())); 119 | } 120 | } 121 | 122 | void CBLLogCat::helpCommand() { 123 | if (!hasArgs()) { 124 | _showHelp = true; // forces command to show help and return 125 | string cmd = nextArg("subcommand"); 126 | if (!processFlag(cmd, kSubcommands)) 127 | cerr << format("Unknown subcommand '%s'\n", cmd.c_str()); 128 | } else { 129 | usage(); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/IntegerGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | using sly.lexer; 5 | 6 | namespace LargeDatasetGenerator.Generator 7 | { 8 | internal sealed class IntegerGeneratorArgs : GeneratorArgumentParser<(long, long)> 9 | { 10 | #region Variables 11 | 12 | private int _currentArgument; 13 | private int? _max; 14 | private int? _min; 15 | 16 | #endregion 17 | 18 | #region Constructors 19 | 20 | public IntegerGeneratorArgs(string input) : base(input) 21 | { 22 | } 23 | 24 | #endregion 25 | 26 | #region Overrides 27 | 28 | protected override void AcceptToken(Token token, Token previous) 29 | { 30 | switch (token.TokenID) { 31 | case GeneratorTokenType.OpenParen: 32 | if (previous.TokenID != GeneratorTokenType.None) { 33 | ThrowSyntaxError(token); 34 | } 35 | 36 | break; 37 | case GeneratorTokenType.CloseParen: 38 | if (previous.TokenID != GeneratorTokenType.Int && 39 | previous.TokenID != GeneratorTokenType.OpenParen) { 40 | ThrowSyntaxError(token); 41 | } 42 | 43 | break; 44 | case GeneratorTokenType.Int: 45 | if (previous.TokenID != GeneratorTokenType.OpenParen && 46 | previous.TokenID != GeneratorTokenType.Comma) { 47 | ThrowSyntaxError(token); 48 | } 49 | 50 | if (!_min.HasValue) { 51 | _min = token.IntValue; 52 | } else if (!_max.HasValue) { 53 | _max = token.IntValue; 54 | } 55 | 56 | break; 57 | case GeneratorTokenType.Comma: 58 | if (previous.TokenID != GeneratorTokenType.Int && 59 | previous.TokenID != GeneratorTokenType.OpenParen) { 60 | ThrowSyntaxError(token); 61 | } 62 | 63 | _currentArgument++; 64 | if (_currentArgument >= 2) { 65 | ThrowSyntaxError(token); 66 | } 67 | 68 | break; 69 | default: 70 | ThrowSyntaxError(token); 71 | break; 72 | } 73 | } 74 | 75 | protected override (long, long) Finish() 76 | { 77 | return (_min ?? Int64.MinValue / 2, _max ?? Int64.MaxValue / 2); 78 | } 79 | 80 | #endregion 81 | } 82 | 83 | /// 84 | /// A generator that generates random 64-bit integers 85 | /// 86 | public sealed class IntegerGenerator : IDataGenerator 87 | { 88 | #region Variables 89 | 90 | private readonly long _max; 91 | private readonly long _min; 92 | 93 | #endregion 94 | 95 | #region Properties 96 | 97 | /// 98 | public string Description { get; } = "Generates random 64-bit integers between min and max"; 99 | 100 | /// 101 | public string Signature { get; } = "{{integer(min: int64 = int64.min / 2, max: int64 = int64.max / 2)}}"; 102 | 103 | #endregion 104 | 105 | #region Constructors 106 | 107 | /// 108 | /// Constructor 109 | /// 110 | /// The arguments to the integer function 111 | public IntegerGenerator(string input) 112 | { 113 | var parser = new IntegerGeneratorArgs(input); 114 | var (min, max) = parser.Parse(); 115 | _min = min; 116 | _max = max; 117 | } 118 | 119 | /// 120 | /// Default constructor 121 | /// 122 | public IntegerGenerator() : this("()") 123 | { 124 | } 125 | 126 | #endregion 127 | 128 | #region IDataGenerator 129 | 130 | /// 131 | public Task GenerateValueAsync() 132 | { 133 | return Task.FromResult(ThreadSafeRandom.NextInt64(_min, _max)); 134 | } 135 | 136 | #endregion 137 | } 138 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/LoremGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | using LargeDatasetGenerator.Data; 5 | 6 | using sly.lexer; 7 | 8 | namespace LargeDatasetGenerator.Generator 9 | { 10 | internal sealed class LoremGeneratorArgs : GeneratorArgumentParser<(int, LoremUnit)> 11 | { 12 | #region Variables 13 | 14 | private int? _count; 15 | private int _currentArgument; 16 | private LoremUnit? _unit; 17 | 18 | #endregion 19 | 20 | #region Constructors 21 | 22 | public LoremGeneratorArgs(string input) : base(input) 23 | { 24 | } 25 | 26 | #endregion 27 | 28 | #region Overrides 29 | 30 | protected override void AcceptToken(Token token, Token previous) 31 | { 32 | switch (token.TokenID) { 33 | case GeneratorTokenType.OpenParen: 34 | if (previous.TokenID != GeneratorTokenType.None) { 35 | ThrowSyntaxError(token); 36 | } 37 | 38 | break; 39 | case GeneratorTokenType.CloseParen: 40 | if (previous.TokenID != GeneratorTokenType.Int && 41 | previous.TokenID != GeneratorTokenType.Identifier) { 42 | ThrowSyntaxError(token); 43 | } 44 | 45 | break; 46 | case GeneratorTokenType.Comma: 47 | _currentArgument++; 48 | if (_currentArgument > 1) { 49 | ThrowSyntaxError(token); 50 | } 51 | 52 | break; 53 | case GeneratorTokenType.Int: 54 | if (_currentArgument != 0 || previous.TokenID != GeneratorTokenType.OpenParen) { 55 | ThrowSyntaxError(token); 56 | } 57 | 58 | _count = token.IntValue; 59 | break; 60 | case GeneratorTokenType.Identifier: 61 | if (_currentArgument != 1) { 62 | ThrowSyntaxError(token); 63 | } 64 | 65 | if (!Enum.TryParse(token.StringWithoutQuotes, true, out var unit) && 66 | !Enum.TryParse(token.StringWithoutQuotes.TrimEnd('s'), true, out unit)) { 67 | ThrowSyntaxError(token); 68 | } 69 | 70 | _unit = unit; 71 | break; 72 | default: 73 | ThrowSyntaxError(token); 74 | break; 75 | } 76 | } 77 | 78 | protected override (int, LoremUnit) Finish() 79 | { 80 | return (_count ?? 1, _unit ?? LoremUnit.Sentence); 81 | } 82 | 83 | #endregion 84 | } 85 | 86 | public sealed class LoremGenerator : IDataGenerator 87 | { 88 | #region Variables 89 | 90 | private readonly int _count; 91 | private readonly LoremUnit _unit; 92 | 93 | #endregion 94 | 95 | #region Properties 96 | 97 | /// 98 | public string Description { get; } = "Generates lorem ipsum text of the specified amount"; 99 | 100 | /// 101 | public string Signature { get; } = "{{lorem(count: int32, unit: word(s)|sentence(s)|paragraph(s))}}"; 102 | 103 | #endregion 104 | 105 | #region Constructors 106 | 107 | /// 108 | /// Constructor 109 | /// 110 | /// The arguments to the lorem function 111 | public LoremGenerator(string input) 112 | { 113 | var parser = new LoremGeneratorArgs(input); 114 | (int count, LoremUnit unit) result = parser.Parse(); 115 | _count = result.count; 116 | _unit = result.unit; 117 | } 118 | 119 | /// 120 | /// Default constructor 121 | /// 122 | public LoremGenerator() : this("(1, word)") 123 | { 124 | } 125 | 126 | #endregion 127 | 128 | #region IDataGenerator 129 | 130 | /// 131 | public Task GenerateValueAsync() 132 | { 133 | return Task.FromResult(LoremIpsum.Generate(_count, _unit)); 134 | } 135 | 136 | #endregion 137 | } 138 | } -------------------------------------------------------------------------------- /gen_errors/cbl-errors.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "name": "CreateDBDirectoryFailed", "message": "Unable to create database directory." }, 3 | { "name": "CloseDBFailedReplications", "message": "Cannot close the database. Please stop all of the replicators before closing the database." }, 4 | { "name": "CloseDBFailedQueryListeners", "message": "Cannot close the database. Please remove all of the query listeners before closing the database." }, 5 | { "name": "DeleteDBFailedReplications", "message": "Cannot delete the database. Please stop all of the replicators before closing the database." }, 6 | { "name": "DeleteDBFailedQueryListeners", "message": "Cannot delete the database. Please remove all of the query listeners before closing the database." }, 7 | { "name": "DeleteDocFailedNotSaved", "message": "Cannot delete a document that has not yet been saved." }, 8 | { "name": "DocumentNotFound", "message": "The document doesn't exist in the database." }, 9 | { "name": "DocumentAnotherDatabase", "message": "Cannot operate on a document from another database." }, 10 | { "name": "BlobDifferentDatabase", "message": "A document contains a blob that was saved to a different database. The save operation cannot complete." }, 11 | { "name": "BlobContentNull", "message": "No data available to write for install. Please ensure that all blobs in the document have non-null content." }, 12 | { "name": "ResolvedDocContainsNull", "message": "Resolved document has a null body." }, 13 | { "name": "ResolvedDocFailedLiteCore", "message": "LiteCore failed resolving conflict." }, 14 | { "name": "ResolvedDocWrongDb", "message": "Resolved document's database {0} is different from expected database {1}." }, 15 | { "name": "DBClosed", "message": "Attempt to perform an operation on a closed database." }, 16 | { "name": "NoDocumentRevision", "message": "No revision data on the document!" }, 17 | { "name": "FragmentPathNotExist", "message": "Specified fragment path does not exist in object; cannot set value." }, 18 | { "name": "InvalidCouchbaseObjType", "message": "{0} is not a valid type. You may only pass {1}, Blob, a one-dimensional array or a dictionary whose members are one of the preceding types." }, 19 | { "name": "InvalidValueToBeDeserialized", "message": "Non-string or null key in data to be deserialized." }, 20 | { "name": "BlobContainsNoData", "message": "Blob has no data available." }, 21 | { "name": "NotFileBasedURL", "message": "{0} must be a file-based URL." }, 22 | { "name": "BlobReadStreamNotOpen", "message": "Stream is not open." }, 23 | { "name": "CannotSetLogLevel", "message": "Cannot set logging level without a configuration." }, 24 | { "name": "InvalidSchemeURLEndpoint", "message": "Invalid scheme for URLEndpoint url ({0}). It must be either 'ws:' or 'wss:'." }, 25 | { "name": "InvalidEmbeddedCredentialsInURL", "message": "Embedded credentials in a URL (username:password@url) are not allowed. Use the BasicAuthenticator class instead." }, 26 | { "name": "ReplicatorNotStopped", "message": "Replicator is not stopped. Resetting checkpoint is only allowed when the replicator is in the stopped state." }, 27 | { "name": "QueryParamNotAllowedContainCollections", "message": "Query parameters are not allowed to contain collections." }, 28 | { "name": "MissASforJoin", "message": "Missing AS clause for JOIN." }, 29 | { "name": "MissONforJoin", "message": "Missing ON statement for JOIN." }, 30 | { "name": "ExpressionsMustBeIExpressionOrString", "message": "Expressions must either be {0} or String." }, 31 | { "name": "InvalidExpressionValueBetween", "message": "Invalid expression value for expression of Between({0})." }, 32 | { "name": "ResultSetAlreadyEnumerated", "message": "This result set has already been enumerated. Please re-execute the original query." }, 33 | { "name": "ExpressionsMustContainOnePlusElement", "message": "{0} expressions must contain at least one element." }, 34 | { "name": "DuplicateSelectResultName", "message": "Duplicate select result named {0}." }, 35 | { "name": "NoAliasInJoin", "message": "The default database must have an alias in order to use a JOIN statement (Make sure your data source uses the As() function)." }, 36 | { "name": "InvalidQueryDBNull", "message": "Invalid query: The database is null." }, 37 | { "name": "InvalidQueryMissingSelectOrFrom", "message": "Invalid query: missing Select or From." }, 38 | { "name": "PullOnlyPendingDocIDs", "message": "Pending Document IDs are not supported on pull-only replicators." }, 39 | { "name": "NoDocEditInReplicationFilter", "message": "Documents from a replication filter cannot be edited."} 40 | ] 41 | 42 | -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.CouchbaseServer/CouchbaseServerOutput.cs: -------------------------------------------------------------------------------- 1 | // 2 | // CouchbaseServerOutput.cs 3 | // 4 | // Copyright (c) 2019 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Threading.Tasks; 23 | 24 | using Couchbase; 25 | using Couchbase.Authentication; 26 | using Couchbase.Configuration.Client; 27 | using Couchbase.Core; 28 | 29 | using LargeDatasetGenerator.Output; 30 | 31 | using McMaster.Extensions.CommandLineUtils; 32 | 33 | namespace LargeDatasetGenerator.CouchbaseServer 34 | { 35 | public sealed class CouchbaseServerOutput : IJsonOutput, IDisposable 36 | { 37 | #region Variables 38 | 39 | private IBucket _bucket; 40 | private string _bucketName; 41 | 42 | private string _password; 43 | private Uri _url; 44 | private string _username; 45 | 46 | #endregion 47 | 48 | #region Properties 49 | 50 | public string ExpectedArgs { get; } = "--url --username --password --bucket "; 51 | 52 | public string Name { get; } = "couchbase_server"; 53 | 54 | #endregion 55 | 56 | #region IDisposable 57 | 58 | public void Dispose() 59 | { 60 | _bucket.Dispose(); 61 | } 62 | 63 | #endregion 64 | 65 | #region IJsonOutput 66 | 67 | public bool PreloadArgs(string[] args) 68 | { 69 | var subApp = new CommandLineApplication(); 70 | 71 | var url = subApp.Option("--url ", "The file template to write JSON data into", 72 | CommandOptionType.SingleValue); 73 | url.IsRequired(); 74 | url.Validators.Add(new UrlValidation()); 75 | 76 | var username = subApp.Option("--username ", "The username to log in as", 77 | CommandOptionType.SingleValue); 78 | username.IsRequired(); 79 | 80 | var password = subApp.Option("--password ", "The password to log in with", 81 | CommandOptionType.SingleValue); 82 | password.IsRequired(); 83 | 84 | var bucket = subApp.Option("--bucket ", "The bucket to use for writing data", 85 | CommandOptionType.SingleValue); 86 | bucket.IsRequired(); 87 | 88 | subApp.OnExecute(() => 89 | { 90 | _url = new Uri(url.ParsedValue, UriKind.Absolute); 91 | _username = username.ParsedValue; 92 | _password = password.ParsedValue; 93 | _bucketName = bucket.ParsedValue; 94 | }); 95 | 96 | return subApp.Execute(args) == 0; 97 | } 98 | 99 | public Task WriteBatchAsync(IEnumerable> json) 100 | { 101 | if (_bucket == null) { 102 | var cluster = new Cluster(new ClientConfiguration 103 | { 104 | Servers = new List {_url} 105 | }); 106 | 107 | var authenticator = new PasswordAuthenticator(_username, _password); 108 | cluster.Authenticate(authenticator); 109 | _bucket = cluster.OpenBucket(_bucketName); 110 | } 111 | 112 | IEnumerable>> documents = json.Select(x => 113 | { 114 | var retVal = new Document> 115 | { 116 | Id = x["_id"] as string, 117 | Content = new Dictionary(x) 118 | }; 119 | 120 | retVal.Content.Remove("_id"); 121 | 122 | return retVal; 123 | }); 124 | 125 | 126 | return _bucket.InsertAsync(documents.ToList()); 127 | } 128 | 129 | #endregion 130 | } 131 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/GaussGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | using sly.lexer; 5 | 6 | namespace LargeDatasetGenerator.Generator 7 | { 8 | internal sealed class GaussGeneratorArgs : GeneratorArgumentParser<(double, double)> 9 | { 10 | #region Variables 11 | 12 | private int _index; 13 | private double? _mean; 14 | private double? _stddev; 15 | 16 | #endregion 17 | 18 | #region Constructors 19 | 20 | public GaussGeneratorArgs(string input) : base(input) 21 | { 22 | } 23 | 24 | #endregion 25 | 26 | #region Overrides 27 | 28 | protected override void AcceptToken(Token token, Token previous) 29 | { 30 | switch (token.TokenID) { 31 | case GeneratorTokenType.OpenParen: 32 | if (previous.TokenID != GeneratorTokenType.None) { 33 | ThrowSyntaxError(token); 34 | } 35 | 36 | break; 37 | case GeneratorTokenType.CloseParen: 38 | if (previous.TokenID != GeneratorTokenType.Int) { 39 | ThrowSyntaxError(token); 40 | } 41 | 42 | break; 43 | case GeneratorTokenType.Int: 44 | if (previous != null && previous.TokenID != GeneratorTokenType.Comma) { 45 | ThrowSyntaxError(token); 46 | } 47 | 48 | if (_index == 0) { 49 | _mean = token.IntValue; 50 | } else { 51 | _stddev = token.IntValue; 52 | } 53 | 54 | break; 55 | case GeneratorTokenType.Double: 56 | if (previous != null && previous.TokenID != GeneratorTokenType.Comma) { 57 | ThrowSyntaxError(token); 58 | } 59 | 60 | if (_index == 0) { 61 | _mean = token.DoubleValue; 62 | } else { 63 | _stddev = token.DoubleValue; 64 | } 65 | 66 | break; 67 | case GeneratorTokenType.Comma: 68 | _index++; 69 | if (_index > 1) { 70 | ThrowSyntaxError(token); 71 | } 72 | 73 | break; 74 | default: 75 | ThrowSyntaxError(token); 76 | break; 77 | } 78 | } 79 | 80 | protected override (double, double) Finish() 81 | { 82 | return (_mean ?? 0.0, _stddev ?? 1.0); 83 | } 84 | 85 | #endregion 86 | } 87 | 88 | public sealed class GaussGenerator : IDataGenerator 89 | { 90 | #region Variables 91 | 92 | private readonly double _mean; 93 | private readonly double _stddev; 94 | 95 | #endregion 96 | 97 | #region Properties 98 | 99 | /// 100 | public string Description { get; } = "Generates normally distributed random floating point values"; 101 | 102 | /// 103 | public string Signature { get; } = "{{gauss(mu: number = 0.0, sigma: number = 1.0)}}"; 104 | 105 | #endregion 106 | 107 | #region Constructors 108 | 109 | /// 110 | /// Constructor 111 | /// 112 | /// The arguments to the gauss function 113 | public GaussGenerator(string input) 114 | { 115 | var parser = new GaussGeneratorArgs(input); 116 | (double mean, double stddev) result = parser.Parse(); 117 | _mean = result.mean; 118 | _stddev = result.stddev; 119 | } 120 | 121 | /// 122 | /// Default constructor 123 | /// 124 | public GaussGenerator() : this("()") 125 | { 126 | } 127 | 128 | #endregion 129 | 130 | #region IDataGenerator 131 | 132 | /// 133 | public Task GenerateValueAsync() 134 | { 135 | var x1 = 1 - ThreadSafeRandom.NextDouble(); 136 | var x2 = 1 - ThreadSafeRandom.NextDouble(); 137 | 138 | var y1 = Math.Sqrt(-2.0 * Math.Log(x1)) * Math.Cos(2.0 * Math.PI * x2); 139 | return Task.FromResult(y1 * _stddev + _mean); 140 | } 141 | 142 | #endregion 143 | } 144 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/RandomGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | 5 | using LargeDatasetGenerator.Generator; 6 | 7 | using sly.lexer; 8 | 9 | namespace LargeDatasetGenerator.Core.Generator 10 | { 11 | internal sealed class RandomGeneratorArgs : GeneratorArgumentParser> 12 | { 13 | #region Variables 14 | 15 | private readonly List _choices = new List(); 16 | 17 | #endregion 18 | 19 | #region Constructors 20 | 21 | public RandomGeneratorArgs(string input) : base(input) 22 | { 23 | } 24 | 25 | #endregion 26 | 27 | #region Overrides 28 | 29 | protected override void AcceptToken(Token token, Token previous) 30 | { 31 | switch (token.TokenID) { 32 | case GeneratorTokenType.OpenParen: 33 | if (previous.TokenID != GeneratorTokenType.None) { 34 | ThrowSyntaxError(token); 35 | } 36 | 37 | break; 38 | case GeneratorTokenType.CloseParen: 39 | if (previous.TokenID == GeneratorTokenType.OpenParen) { 40 | throw new InvalidOperationException("random() must have at least two items"); 41 | } 42 | 43 | if (previous.TokenID == GeneratorTokenType.Comma) { 44 | ThrowSyntaxError(token); 45 | } 46 | 47 | break; 48 | case GeneratorTokenType.String: 49 | if (previous.TokenID != GeneratorTokenType.OpenParen && 50 | previous.TokenID != GeneratorTokenType.Comma) { 51 | ThrowSyntaxError(token); 52 | } 53 | 54 | _choices.Add(token.StringWithoutQuotes.Trim('\'')); 55 | break; 56 | case GeneratorTokenType.Int: 57 | if (previous.TokenID != GeneratorTokenType.OpenParen && 58 | previous.TokenID != GeneratorTokenType.Comma) { 59 | ThrowSyntaxError(token); 60 | } 61 | 62 | _choices.Add(token.IntValue); 63 | break; 64 | case GeneratorTokenType.Double: 65 | if (previous.TokenID != GeneratorTokenType.OpenParen && 66 | previous.TokenID != GeneratorTokenType.Comma) { 67 | ThrowSyntaxError(token); 68 | } 69 | 70 | _choices.Add(token.DoubleValue); 71 | break; 72 | default: 73 | ThrowSyntaxError(token); 74 | break; 75 | } 76 | } 77 | 78 | protected override IReadOnlyList Finish() 79 | { 80 | return _choices; 81 | } 82 | 83 | #endregion 84 | } 85 | 86 | /// 87 | /// A generator that chooses a random item from a list of provided ones 88 | /// 89 | public sealed class RandomGenerator : IDataGenerator 90 | { 91 | #region Variables 92 | 93 | private readonly IReadOnlyList _choices; 94 | 95 | #endregion 96 | 97 | #region Properties 98 | 99 | /// 100 | public string Description { get; } = "Choose a random value from a provided list of values"; 101 | 102 | /// 103 | public string Signature { get; } = "{{random(, , ...)}}"; 104 | 105 | #endregion 106 | 107 | #region Constructors 108 | 109 | /// 110 | /// Constructor 111 | /// 112 | /// The raw arguments from the command line 113 | public RandomGenerator(string input) 114 | { 115 | var parser = new RandomGeneratorArgs(input); 116 | _choices = parser.Parse(); 117 | } 118 | 119 | /// 120 | /// Default constructor 121 | /// 122 | public RandomGenerator() : this("(\"foo\", \"bar\")") 123 | { 124 | 125 | } 126 | 127 | #endregion 128 | 129 | #region IDataGenerator 130 | 131 | /// 132 | public Task GenerateValueAsync() 133 | { 134 | var nextIndex = ThreadSafeRandom.Next(0, _choices.Count); 135 | return Task.FromResult(_choices[nextIndex]); 136 | } 137 | 138 | #endregion 139 | } 140 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/RangeGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | 5 | using sly.lexer; 6 | 7 | namespace LargeDatasetGenerator.Generator 8 | { 9 | internal sealed class RangeGeneratorArgs : GeneratorArgumentParser<(int, int, int)> 10 | { 11 | #region Variables 12 | 13 | private int _currentArgument; 14 | private int _start; 15 | private int _step = 1; 16 | private int _stop; 17 | 18 | #endregion 19 | 20 | #region Constructors 21 | 22 | public RangeGeneratorArgs(string input) : base(input) 23 | { 24 | } 25 | 26 | #endregion 27 | 28 | #region Overrides 29 | 30 | protected override void AcceptToken(Token token, Token previous) 31 | { 32 | switch (token.TokenID) { 33 | case GeneratorTokenType.OpenParen: 34 | if (previous.TokenID != GeneratorTokenType.None) { 35 | ThrowSyntaxError(token); 36 | } 37 | 38 | _currentArgument++; 39 | break; 40 | case GeneratorTokenType.CloseParen: 41 | if (previous.TokenID != GeneratorTokenType.Int) { 42 | if (previous.TokenID == GeneratorTokenType.OpenParen) { 43 | throw new InvalidOperationException("Missing argument 'stop'"); 44 | } 45 | 46 | ThrowSyntaxError(token); 47 | } 48 | 49 | break; 50 | case GeneratorTokenType.Int: 51 | if (previous.TokenID != GeneratorTokenType.Comma && 52 | previous.TokenID != GeneratorTokenType.OpenParen) { 53 | ThrowSyntaxError(token); 54 | } 55 | 56 | if (_currentArgument == 1) { 57 | _stop = token.IntValue; 58 | } else if (_currentArgument == 2) { 59 | _start = _stop; 60 | _stop = token.IntValue; 61 | } else { 62 | _step = token.IntValue; 63 | } 64 | 65 | break; 66 | case GeneratorTokenType.Comma: 67 | if (previous.TokenID != GeneratorTokenType.Int) { 68 | ThrowSyntaxError(token); 69 | } 70 | 71 | _currentArgument++; 72 | if (_currentArgument > 3) { 73 | ThrowSyntaxError(token); 74 | } 75 | 76 | break; 77 | default: 78 | ThrowSyntaxError(token); 79 | break; 80 | } 81 | } 82 | 83 | protected override (int, int, int) Finish() 84 | { 85 | return (_start, _stop, _step); 86 | } 87 | 88 | #endregion 89 | } 90 | 91 | /// 92 | /// A generator that creates an array of numbers in a range 93 | /// 94 | public sealed class RangeGenerator : IDataGenerator 95 | { 96 | #region Variables 97 | 98 | private readonly int _start, _stop, _step; 99 | 100 | #endregion 101 | 102 | #region Properties 103 | 104 | /// 105 | public string Description { get; } = "Generates an array of numbers between min and max, increasing by step."; 106 | 107 | /// 108 | public string Signature { get; } = "{{range(start: int32 = 0, stop: int32, step: int32 = 1)}}"; 109 | 110 | #endregion 111 | 112 | #region Constructors 113 | 114 | /// 115 | /// Constructor 116 | /// 117 | /// The arguments to the range function 118 | public RangeGenerator(string input) 119 | { 120 | var parser = new RangeGeneratorArgs(input); 121 | (int start, int stop, int step) result = parser.Parse(); 122 | _start = result.start; 123 | _stop = result.stop; 124 | _step = result.step; 125 | } 126 | 127 | /// 128 | /// Default constructor 129 | /// 130 | public RangeGenerator() : this("(0)") 131 | { 132 | } 133 | 134 | #endregion 135 | 136 | #region IDataGenerator 137 | 138 | /// 139 | public Task GenerateValueAsync() 140 | { 141 | var retVal = new List(); 142 | for (int i = _start; i < _stop; i += _step) { 143 | retVal.Add(i); 144 | } 145 | 146 | return Task.FromResult(retVal); 147 | } 148 | 149 | #endregion 150 | } 151 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Generator/FloatingGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | using sly.lexer; 5 | 6 | namespace LargeDatasetGenerator.Generator 7 | { 8 | internal sealed class FloatingGeneratorParser : GeneratorArgumentParser<(double, double, int?)> 9 | { 10 | #region Variables 11 | 12 | private int _currentArgument; 13 | private int? _digits; 14 | private double? _max; 15 | private double? _min; 16 | 17 | #endregion 18 | 19 | #region Constructors 20 | 21 | public FloatingGeneratorParser(string input) : base(input) 22 | { 23 | } 24 | 25 | #endregion 26 | 27 | #region Overrides 28 | 29 | protected override void AcceptToken(Token token, Token previous) 30 | { 31 | switch (token.TokenID) { 32 | case GeneratorTokenType.OpenParen: 33 | if (previous.TokenID != GeneratorTokenType.None) { 34 | ThrowSyntaxError(token); 35 | } 36 | 37 | break; 38 | case GeneratorTokenType.CloseParen: 39 | if (previous.TokenID != GeneratorTokenType.Double && 40 | previous.TokenID != GeneratorTokenType.Int) { 41 | ThrowSyntaxError(token); 42 | } 43 | 44 | break; 45 | case GeneratorTokenType.Double: 46 | if (!_min.HasValue) { 47 | _min = token.DoubleValue; 48 | } else { 49 | if (_currentArgument > 1) { 50 | ThrowSyntaxError(token); 51 | } 52 | 53 | _max = token.DoubleValue; 54 | } 55 | 56 | break; 57 | case GeneratorTokenType.Int: 58 | if (!_min.HasValue) { 59 | _min = token.IntValue; 60 | } else if (!_max.HasValue) { 61 | _max = token.IntValue; 62 | } else { 63 | if (_currentArgument > 2) { 64 | ThrowSyntaxError(token); 65 | } 66 | 67 | _digits = token.IntValue; 68 | } 69 | 70 | break; 71 | case GeneratorTokenType.Comma: 72 | _currentArgument++; 73 | if (_currentArgument >= 3) { 74 | ThrowSyntaxError(token); 75 | } 76 | 77 | break; 78 | default: 79 | ThrowSyntaxError(token); 80 | break; 81 | } 82 | } 83 | 84 | protected override (double, double, int?) Finish() 85 | { 86 | return (_min ?? Double.MinValue / 2, _max ?? Double.MaxValue / 2, _digits); 87 | } 88 | 89 | #endregion 90 | } 91 | 92 | public sealed class FloatingGenerator : IDataGenerator 93 | { 94 | #region Variables 95 | 96 | private readonly int? _digits; 97 | private readonly double _max; 98 | private readonly double _min; 99 | 100 | #endregion 101 | 102 | #region Properties 103 | 104 | /// 105 | public string Description { get; } = 106 | "Randomly generates a number between min and max, rounded to X digits after the decimal point"; 107 | 108 | /// 109 | public string Signature { get; } = "{{floating(min: number = (double.min / 2.0), max: number = (double.max / 2.0), digits: number = (no value))}}"; 110 | 111 | #endregion 112 | 113 | #region Constructors 114 | 115 | /// 116 | /// Constructor 117 | /// 118 | /// The arguments to the floating method 119 | public FloatingGenerator(string parameters) 120 | { 121 | var parser = new FloatingGeneratorParser(parameters); 122 | (double min, double max, int? digits) result = parser.Parse(); 123 | _min = result.min; 124 | _max = result.max; 125 | _digits = result.digits; 126 | } 127 | 128 | /// 129 | /// Default constructor 130 | /// 131 | public FloatingGenerator() : this("()") 132 | { 133 | 134 | } 135 | 136 | #endregion 137 | 138 | #region IDataGenerator 139 | 140 | /// 141 | public Task GenerateValueAsync() 142 | { 143 | var value = ThreadSafeRandom.NextDouble() * (_max - _min) + _min; 144 | if (_digits.HasValue) { 145 | value = Math.Round(value, _digits.Value); 146 | } 147 | 148 | return Task.FromResult(value); 149 | } 150 | 151 | #endregion 152 | } 153 | } -------------------------------------------------------------------------------- /LargeDatasetGenerator/LargeDatasetGenerator.Core/Data/LoremIpsum.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading; 6 | using LargeDatasetGenerator.Generator; 7 | 8 | namespace LargeDatasetGenerator.Data 9 | { 10 | public enum LoremUnit 11 | { 12 | Word, 13 | Sentence, 14 | Paragraph 15 | } 16 | 17 | internal static class LoremIpsum 18 | { 19 | public static readonly IReadOnlyList Words = new[]{"lorem", "ipsum", "dolor", "sit", "amet", "consectetuer", 20 | "adipiscing", "elit", "sed", "diam", "nonummy", "nibh", "euismod", 21 | "tincidunt", "ut", "laoreet", "dolore", "magna", "aliquam", "erat"}; 22 | 23 | private const int MinWordCount = 10; 24 | private const int MaxWordCount = 25; 25 | private const int MinSentenceCount = 3; 26 | private const int MaxSentenceCount = 8; 27 | 28 | public static string Generate(int count, LoremUnit unit) 29 | { 30 | var result = new StringBuilder(); 31 | var paragraphs = new List(); 32 | 33 | if (unit == LoremUnit.Word) { 34 | paragraphs.Add(new Paragraph(1, count)); 35 | } else if (unit == LoremUnit.Sentence) { 36 | paragraphs.Add(new Paragraph(count, 0)); 37 | } else { 38 | for (int i = 0; i < count; i++) { 39 | var sc = ThreadSafeRandom.Next(MinSentenceCount, MaxSentenceCount); 40 | paragraphs.Add(new Paragraph(sc, 0)); 41 | } 42 | } 43 | 44 | var first = true; 45 | foreach (var p in paragraphs) { 46 | if (!first) { 47 | result.AppendLine(); 48 | } 49 | 50 | foreach (var sentence in p) { 51 | if (!first) { 52 | result.Append(" "); 53 | } 54 | 55 | first = false; 56 | for (int i = 0; i < sentence.WordCount; i++) { 57 | var nextWord = Words[ThreadSafeRandom.Next(0, Words.Count)]; 58 | if (i == 0) { 59 | nextWord = nextWord.Substring(0, 1).ToUpperInvariant() + nextWord.Substring(1); 60 | } 61 | 62 | result.Append(nextWord); 63 | if (i != sentence.WordCount - 1) { 64 | result.Append(" "); 65 | } 66 | } 67 | 68 | if (unit != LoremUnit.Word) { 69 | result.Append("."); 70 | } 71 | } 72 | } 73 | 74 | return result.ToString(); 75 | } 76 | 77 | private sealed class Paragraph : IEnumerable 78 | { 79 | private readonly int _sentenceCount; 80 | private readonly int _wordCount; 81 | 82 | public Paragraph(int sentenceCount, int wordCount) 83 | { 84 | _sentenceCount = sentenceCount; 85 | _wordCount = wordCount; 86 | } 87 | 88 | private sealed class Enumerator : IEnumerator 89 | { 90 | private readonly int _length; 91 | private readonly int _wordCount; 92 | private int _current; 93 | 94 | public Enumerator(int length, int wordCount) 95 | { 96 | _length = length; 97 | _wordCount = wordCount; 98 | } 99 | 100 | public bool MoveNext() 101 | { 102 | return ++_current <= _length; 103 | } 104 | 105 | public void Reset() 106 | { 107 | throw new NotSupportedException(); 108 | } 109 | 110 | public Sentence Current 111 | { 112 | get 113 | { 114 | var wordCount = _wordCount == 0 ? ThreadSafeRandom.Next(MinWordCount, MaxWordCount) : _wordCount; 115 | return new Sentence(wordCount); 116 | } 117 | } 118 | 119 | object IEnumerator.Current => Current; 120 | 121 | public void Dispose() 122 | { 123 | // No-op 124 | } 125 | } 126 | 127 | public IEnumerator GetEnumerator() 128 | { 129 | return new Enumerator(_sentenceCount, _wordCount); 130 | } 131 | 132 | IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); 133 | } 134 | 135 | private sealed class Sentence 136 | { 137 | public int WordCount { get; } 138 | 139 | public Sentence(int wordCount) 140 | { 141 | WordCount = wordCount; 142 | } 143 | } 144 | } 145 | } -------------------------------------------------------------------------------- /cmake/common_setup.cmake: -------------------------------------------------------------------------------- 1 | function(common_setup) 2 | set(_build_version "unknown") 3 | set(_litecore_build_version "unknown") 4 | find_package(Git) 5 | if(GIT_FOUND) 6 | execute_process( 7 | COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD 8 | WORKING_DIRECTORY "${local_dir}" 9 | OUTPUT_VARIABLE _build_version 10 | ERROR_QUIET 11 | OUTPUT_STRIP_TRAILING_WHITESPACE 12 | ) 13 | 14 | execute_process( 15 | COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD 16 | WORKING_DIRECTORY "${LITECORE}" 17 | OUTPUT_VARIABLE _litecore_build_version 18 | ERROR_QUIET 19 | OUTPUT_STRIP_TRAILING_WHITESPACE 20 | ) 21 | endif() 22 | 23 | if((NOT "$ENV{VERSION}" STREQUAL "") AND (NOT "$ENV{BLD_NUM}" STREQUAL "")) 24 | set(TOOLS_VERSION "$ENV{VERSION}" CACHE INTERNAL "") 25 | set(TOOLS_BLD_NUM "$ENV{BLD_NUM}" CACHE INTERNAL "") 26 | endif() 27 | 28 | if((NOT "${TOOLS_VERSION}" STREQUAL "") AND (NOT "${TOOLS_BLD_NUM}" STREQUAL "")) 29 | message("Using provided values: VERSION: ${TOOLS_VERSION}, BLD_NUM: ${TOOLS_BLD_NUM}") 30 | string(REPLACE "." ";" VERSION_LIST ${TOOLS_VERSION}) 31 | list(GET VERSION_LIST 0 TOOLS_VERSION_MAJOR) 32 | list(GET VERSION_LIST 1 TOOLS_VERSION_MINOR) 33 | list(GET VERSION_LIST 2 TOOLS_VERSION_PATCH) 34 | set(TOOLS_VERSION_STRING "${TOOLS_VERSION} (${TOOLS_BLD_NUM}; ${_build_version}) with LiteCore ${_litecore_build_version}") 35 | else() 36 | message("No environment variables found, using git commit only") 37 | set(TOOLS_VERSION_MAJOR 0) 38 | set(TOOLS_VERSION_MINOR 0) 39 | set(TOOLS_VERSION_PATCH 0) 40 | set(TOOLS_VERSION_STRING "0.0.0 (${_build_version}) with LiteCore ${_litecore_build_version}") 41 | endif() 42 | 43 | configure_file( 44 | ${PROJECT_SOURCE_DIR}/../config.h.in 45 | ${PROJECT_BINARY_DIR}/generated_headers/config.h 46 | ) 47 | 48 | set(CMAKE_CXX_STANDARD_REQUIRED ON PARENT_SCOPE) 49 | set(CMAKE_CXX_STANDARD 11 PARENT_SCOPE) 50 | set(CMAKE_C_STANDARD_REQUIRED ON PARENT_SCOPE) 51 | set(CMAKE_C_STANDARD 11 PARENT_SCOPE) 52 | endfunction(common_setup) 53 | 54 | function(fix_cpp_runtime) 55 | if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") 56 | # Enable relative RPATHs for installed bits 57 | set (CMAKE_INSTALL_RPATH "\$ORIGIN/../lib") 58 | 59 | 60 | if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") 61 | if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 62 | message(FATAL_ERROR "${CMAKE_CXX_COMPILER_ID} is not supported for building!") 63 | endif() 64 | find_library(LIBCXX_LIB c++) 65 | if (NOT LIBCXX_LIB) 66 | message(FATAL_ERROR "libc++ not found") 67 | endif() 68 | message("Found libc++ at ${LIBCXX_LIB}") 69 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") 70 | 71 | find_library(LIBCXXABI_LIB c++abi) 72 | if (NOT LIBCXXABI_LIB) 73 | message(FATAL_ERROR "libc++abi not found") 74 | endif() 75 | message("Found libc++abi at ${LIBCXXABI_LIB}") 76 | find_path(LIBCXX_INCLUDE c++/v1/string 77 | HINTS "${CMAKE_BINARY_DIR}/tlm/deps/libcxx.exploded" 78 | PATH_SUFFIXES include) 79 | if (NOT LIBCXX_INCLUDE) 80 | message(FATAL_ERROR "libc++ header files not found") 81 | endif() 82 | message("Using libc++ header files in ${LIBCXX_INCLUDE}") 83 | include_directories("${LIBCXX_INCLUDE}/c++/v1") 84 | if(NOT EXISTS "/usr/include/xlocale.h") 85 | include_directories("${LIBCXX_INCLUDE}/c++/v1/support/xlocale") # this fixed path is here to avoid compilation on Ubuntu 17.10 where xlocale.h is searched by some header(s) in libc++ as but not found from search path without this modification. However, only do it if the original xlocale.h does not exist since this will get searched before /usr/include and override a valid file with an empty one. 86 | endif() 87 | include_directories("/usr/include/libcxxabi") # this fixed path is here to avoid Clang issue noted at http://lists.alioth.debian.org/pipermail/pkg-llvm-team/2015-September/005208.html 88 | endif() 89 | 90 | # libc++ is special - clang will introduce an implicit -lc++ when it is used. 91 | # That means we need to tell the linker the path to the directory containing 92 | # libc++.so rather than just linking the .so directly. This must be done 93 | # *before* the target declaration as it affects all subsequent targets. 94 | get_filename_component (LIBCXX_LIBDIR "${LIBCXX_LIB}" DIRECTORY) 95 | link_directories (${LIBCXX_LIBDIR}) 96 | endif() 97 | endfunction() 98 | 99 | function(get_platform_libs OUTPUT_VAR) 100 | if (APPLE) 101 | set(${OUTPUT_VAR} z 102 | "-framework CoreFoundation" "-framework Security" 103 | PARENT_SCOPE) 104 | elseif(MSVC) 105 | set(${OUTPUT_VAR} zlibstatic PARENT_SCOPE) 106 | else() 107 | set(${OUTPUT_VAR} pthread z dl ${LIBCXX_LIB} ${LIBCXXABI_LIB} PARENT_SCOPE) 108 | endif() 109 | endfunction(get_platform_libs OUTPUT_VAR) 110 | -------------------------------------------------------------------------------- /cblite/cbliteTool+revs.cc: -------------------------------------------------------------------------------- 1 | // 2 | // cbliteTool+revs.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "cbliteTool.hh" 20 | 21 | 22 | const Tool::FlagSpec CBLiteTool::kRevsFlags[] = { 23 | {"--remotes", (FlagHandler)&CBLiteTool::remotesFlag}, 24 | }; 25 | 26 | void CBLiteTool::revsUsage() { 27 | writeUsageCommand("revs", false, "DOCID"); 28 | cerr << 29 | " Shows a document's revision history\n" 30 | " --remotes : Shows which revisions are known current on remote databases\n" 31 | " -- : End of arguments (use if DOCID starts with '-')\n" 32 | " Revision flags are denoted by dashes or the letters:\n" 33 | " [D]eleted [X]Closed [C]onflict [A]ttachments [K]eep body [L]eaf\n" 34 | ; 35 | } 36 | 37 | 38 | void CBLiteTool::revsInfo() { 39 | // Read params: 40 | processFlags(kRevsFlags); 41 | if (_showHelp) { 42 | revsUsage(); 43 | return; 44 | } 45 | openDatabaseFromNextArg(); 46 | string docID = nextArg("document ID"); 47 | endOfArgs(); 48 | 49 | auto doc = readDoc(docID); 50 | if (!doc) 51 | return; 52 | 53 | cout << "Document \"" << ansiBold() << doc->docID << ansiReset() << "\""; 54 | if (doc->flags & kDocDeleted) 55 | cout << ", Deleted"; 56 | if (doc->flags & kDocConflicted) 57 | cout << ", Conflicted"; 58 | if (doc->flags & kDocHasAttachments) 59 | cout << ", Has Attachments"; 60 | cout << "\n"; 61 | 62 | // Collect remote status: 63 | RemoteMap remotes; 64 | if (_showRemotes) { 65 | for (C4RemoteID remoteID = 1; true; ++remoteID) { 66 | alloc_slice revID(c4doc_getRemoteAncestor(doc, remoteID)); 67 | if (!revID) 68 | break; 69 | remotes.emplace_back(revID); 70 | } 71 | } 72 | 73 | // Collect revision tree info: 74 | RevTree tree; 75 | alloc_slice root; // use empty slice as root of tree 76 | 77 | do { 78 | alloc_slice leafRevID(doc->selectedRev.revID); 79 | alloc_slice childID = leafRevID; 80 | while (c4doc_selectParentRevision(doc)) { 81 | alloc_slice parentID(doc->selectedRev.revID); 82 | tree[parentID].insert(childID); 83 | childID = parentID; 84 | } 85 | tree[root].insert(childID); 86 | c4doc_selectRevision(doc, leafRevID, false, nullptr); 87 | } while (c4doc_selectNextLeafRevision(doc, true, true, nullptr)); 88 | 89 | writeRevisionChildren(doc, tree, remotes, root, 1); 90 | 91 | for (C4RemoteID i = 1; i <= remotes.size(); ++i) { 92 | alloc_slice addr(c4db_getRemoteDBAddress(_db, i)); 93 | if (!addr) 94 | break; 95 | cout << "[REMOTE#" << i << "] = " << addr << "\n"; 96 | } 97 | } 98 | 99 | 100 | void CBLiteTool::writeRevisionTree(C4Document *doc, 101 | RevTree &tree, 102 | RemoteMap &remotes, 103 | alloc_slice root, 104 | int indent) 105 | { 106 | C4Error error; 107 | if (!c4doc_selectRevision(doc, root, true, &error)) 108 | fail("accessing revision", error); 109 | auto &rev = doc->selectedRev; 110 | cout << string(indent, ' '); 111 | cout << "* "; 112 | if ((rev.flags & kRevLeaf) && !(rev.flags & kRevClosed)) 113 | cout << ansiBold(); 114 | cout << rev.revID << ansiReset(); 115 | 116 | int pad = max(0, 50 - int(indent + 2 + rev.revID.size)); 117 | cout << string(pad, ' '); 118 | 119 | if (rev.flags & kRevClosed) 120 | cout << 'X'; 121 | else 122 | cout << ((rev.flags & kRevDeleted) ? 'D' : '-'); 123 | cout << ((rev.flags & kRevIsConflict) ? 'C' : '-'); 124 | cout << ((rev.flags & kRevClosed) ? 'X' : '-'); 125 | cout << ((rev.flags & kRevHasAttachments) ? 'A' : '-'); 126 | cout << ((rev.flags & kRevKeepBody) ? 'K' : '-'); 127 | cout << ((rev.flags & kRevLeaf) ? 'L' : '-'); 128 | 129 | cout << " #" << rev.sequence; 130 | if (rev.body.buf) { 131 | cout << ", "; 132 | writeSize(rev.body.size); 133 | } 134 | 135 | if (root == slice(doc->revID)) 136 | cout << ansiBold() << " [CURRENT]" << ansiReset(); 137 | 138 | C4RemoteID i = 1; 139 | for (alloc_slice &remote : remotes) { 140 | if (remote == root) 141 | cout << " [REMOTE#" << i << "]"; 142 | ++i; 143 | } 144 | 145 | cout << "\n"; 146 | writeRevisionChildren(doc, tree, remotes, root, indent+2); 147 | } 148 | 149 | void CBLiteTool::writeRevisionChildren(C4Document *doc, 150 | RevTree &tree, 151 | RemoteMap &remotes, 152 | alloc_slice root, 153 | int indent) 154 | { 155 | auto &children = tree[root]; 156 | for (auto i = children.rbegin(); i != children.rend(); ++i) { 157 | writeRevisionTree(doc, tree, remotes, *i, indent); 158 | } 159 | } 160 | 161 | 162 | -------------------------------------------------------------------------------- /Tool.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Tool.cc 3 | // 4 | // Copyright (c) 2017 Couchbase, Inc All rights reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "Tool.hh" 20 | #include "Logging.hh" 21 | #include "linenoise.h" 22 | #include 23 | #include 24 | 25 | #if !defined(_MSC_VER) 26 | #include 27 | #include 28 | #else 29 | #ifndef WIN32_LEAN_AND_MEAN 30 | #define WIN32_LEAN_AND_MEAN 31 | #endif 32 | #include 33 | #include 34 | #define isatty _isatty 35 | #define STDIN_FILENO _fileno(stdin) 36 | #define STDOUT_FILENO _fileno(stdout) 37 | #endif 38 | 39 | 40 | static constexpr int kDefaultLineWidth = 100; 41 | 42 | 43 | Tool* Tool::instance; 44 | 45 | Tool::Tool(const char* name) 46 | :_name(name) 47 | { 48 | if(!instance) { 49 | instance = this; 50 | } 51 | 52 | linenoiseHistorySetMaxLen(100); 53 | } 54 | 55 | Tool::~Tool() { 56 | linenoiseHistoryFree(); 57 | if (this == instance) 58 | instance = nullptr; 59 | } 60 | 61 | 62 | static bool inputIsTerminal() { 63 | return isatty(STDIN_FILENO) && getenv("TERM") != nullptr; 64 | } 65 | 66 | #ifdef _MSC_VER 67 | typedef LONG NTSTATUS, *PNTSTATUS; 68 | #define STATUS_SUCCESS (0x00000000) 69 | 70 | typedef NTSTATUS (WINAPI* RtlGetVersionPtr)(PRTL_OSVERSIONINFOW); 71 | 72 | RTL_OSVERSIONINFOW GetRealOSVersion() { 73 | HMODULE hMod = ::GetModuleHandleW(L"ntdll.dll"); 74 | if (hMod) { 75 | RtlGetVersionPtr fxPtr = (RtlGetVersionPtr)::GetProcAddress(hMod, "RtlGetVersion"); 76 | if (fxPtr != nullptr) { 77 | RTL_OSVERSIONINFOW rovi = { 0 }; 78 | rovi.dwOSVersionInfoSize = sizeof(rovi); 79 | if ( STATUS_SUCCESS == fxPtr(&rovi) ) { 80 | return rovi; 81 | } 82 | } 83 | } 84 | RTL_OSVERSIONINFOW rovi = { 0 }; 85 | return rovi; 86 | } 87 | #endif 88 | 89 | static bool sOutputIsColor = false; 90 | 91 | void Tool::enableColor() { 92 | const char *term = getenv("TERM"); 93 | auto tty = isatty(STDOUT_FILENO); 94 | if(!tty) {return;} 95 | if(term != nullptr 96 | && (strstr(term,"ANSI") || strstr(term,"ansi") || strstr(term,"color"))) { 97 | sOutputIsColor = true; 98 | return; 99 | } 100 | 101 | #ifdef _MSC_VER 102 | #ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING 103 | // Sick of this being missing for whatever reason 104 | #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 105 | #endif 106 | 107 | sOutputIsColor = GetRealOSVersion().dwMajorVersion >= 10; 108 | if(sOutputIsColor) { 109 | HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE); 110 | DWORD consoleMode; 111 | if(GetConsoleMode(hConsole, &consoleMode)) { 112 | SetConsoleMode(hConsole, consoleMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); 113 | } 114 | } 115 | #endif 116 | 117 | } 118 | 119 | string Tool::ansi(const char *command) { 120 | if (sOutputIsColor) 121 | return format("\033[%sm", command); 122 | else 123 | return ""; 124 | } 125 | 126 | 127 | int Tool::terminalWidth() { 128 | #if __APPLE__ 129 | struct ttysize ts; 130 | if (ioctl(0, TIOCGSIZE, &ts) == 0 && ts.ts_cols > 0) 131 | return ts.ts_cols; 132 | #endif 133 | return kDefaultLineWidth; 134 | } 135 | 136 | bool Tool::readLine(const char *cPrompt) { 137 | if (!inputIsTerminal()) 138 | return dumbReadLine(cPrompt); 139 | 140 | string prompt = cPrompt; 141 | if (sOutputIsColor) 142 | prompt = ansiBold() + prompt + ansiReset(); 143 | 144 | while (true) { 145 | char* line = linenoise(prompt.c_str()); 146 | // Returned line and lineLength include the trailing newline, unless user typed ^D. 147 | if (line != nullptr && strlen(line) > 0) { 148 | // Got a command! 149 | // Add line to history so user can recall it later: 150 | linenoiseHistoryAdd(line); 151 | _argTokenizer.reset(line); 152 | return true; 153 | } else if(linenoiseKeyType() == 2) { 154 | cout << endl; 155 | return false; 156 | } 157 | 158 | // No command was entered, so go round again: 159 | cout << "Please type a command, or Ctrl-D to exit.\n"; 160 | } 161 | } 162 | 163 | 164 | bool Tool::dumbReadLine(const char *prompt) { 165 | char inputBuffer[5000]; 166 | while (true) { 167 | cout << ansiBold() << prompt << ansiReset(); 168 | char* line = fgets(inputBuffer, sizeof(inputBuffer), stdin); 169 | if (!line) { 170 | cout << '\n'; 171 | return false; 172 | } 173 | if (strlen(line) > 0) { 174 | _argTokenizer.reset(line); 175 | return true; 176 | } 177 | cout << "Please type a command, or Ctrl-D to exit.\n"; 178 | } 179 | } 180 | 181 | 182 | string Tool::readPassword(const char *prompt) { 183 | #if defined(_MSC_VER) 184 | fail("Sorry, password input is unimplemented on Windows"); //FIX //TODO 185 | #else 186 | char *cpass = getpass(prompt); 187 | string password = cpass; 188 | memset(cpass, '*', strlen(cpass)); // overwrite password at known static location 189 | return password; 190 | #endif 191 | } 192 | --------------------------------------------------------------------------------