├── .gitmodules
├── .gitignore
├── jsonhelper.hh
├── demo.cc
├── CMakeLists.txt
├── pqdemo.cc
├── jsonhelper.cc
├── meson.build
├── LICENSE
├── .github
    └── workflows
    │   └── cmake.yml
├── jsontests.cc
├── minipsql.hh
├── psqlwriter.hh
├── sqlwriter.hh
├── minipsql.cc
├── README.md
├── psqlwriter.cc
├── testrunner.cc
└── sqlwriter.cc


/.gitmodules:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | *.o
3 | demo
4 | CMakeCache.txt
5 | CMakeFiles/
6 | CTestTestfile.cmake
7 | Makefile
8 | 
9 | 


--------------------------------------------------------------------------------
/jsonhelper.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "sqlwriter.hh"
 4 | #include "nlohmann/json.hpp"
 5 | 
 6 | // turn results from SQLiteWriter into JSON
 7 | nlohmann::json packResultsJson(const std::vector<std::unordered_map<std::string, MiniSQLite::outvar_t>>& result, bool fillnull=true);
 8 | 
 9 | // helper that makes a JSON string for you
10 | std::string packResultsJsonStr(const std::vector<std::unordered_map<std::string, MiniSQLite::outvar_t>>& result, bool fillnull=true);
11 | 
12 | 


--------------------------------------------------------------------------------
/demo.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <stdexcept>
 3 | #include <vector>
 4 | #include <algorithm>
 5 | #include <unistd.h>
 6 | #include "sqlwriter.hh"
 7 | 
 8 | using namespace std;
 9 | 
10 | 
11 | int main()
12 | try
13 | {
14 |   SQLiteWriter sqw("example.sqlite3");
15 | 
16 |   for(int n = 0 ; n < 1000000; ++n) {
17 |     sqw.addValue({{"pief", n}, {"poef", 1.1234567890123*n}, {"paf", "bert"}});
18 |   }
19 | 
20 |   sqw.addValue({{"timestamp", 1234567890}});
21 | 
22 |   for(int n = 0 ; n < 1; ++n) {
23 |     sqw.addValue({{"pief", n}, {"poef", 1.1234567890123*n}, {"paf", "bert"}});
24 |   }
25 | 
26 | }
27 | catch (std::exception& e)
28 | {
29 |     std::cout << "exception: " << e.what() << std::endl;
30 | }
31 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.1)
 2 | 
 3 | project(powmon VERSION 1.0
 4 |                   DESCRIPTION "csv-like storage to sqlite"
 5 |                   LANGUAGES CXX)
 6 | 
 7 | 
 8 | set(CMAKE_CXX_STANDARD 17 CACHE STRING "The C++ standard to use")
 9 | set(CMAKE_CXX_STANDARD_REQUIRED ON)
10 | set(CMAKE_CXX_EXTENSIONS ON)
11 | 
12 | set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
13 | set(THREADS_PREFER_PTHREAD_FLAG TRUE)
14 | find_package(Threads REQUIRED)
15 | 
16 | add_executable(demo demo.cc sqlwriter.cc)
17 | target_link_libraries(demo  sqlite3  pthread  dl  )
18 | 
19 | add_executable(pqdemo pqdemo.cc psqlwriter.cc minipsql.cc)
20 | target_link_libraries(pqdemo pthread  dl pq )
21 | 
22 | 
23 | add_executable(testrunner testrunner.cc sqlwriter.cc jsontests.cc jsonhelper.cc)
24 | target_link_libraries(testrunner sqlite3 pthread dl)
25 | 
26 | enable_testing()
27 | add_test(testname testrunner)
28 | 


--------------------------------------------------------------------------------
/pqdemo.cc:
--------------------------------------------------------------------------------
 1 | #include "psqlwriter.hh"
 2 | #include <iostream>
 3 | 
 4 | using namespace std;
 5 | 
 6 | int main(int argc, char *argv[])
 7 | try
 8 | {
 9 |   PSQLWriter sqw("");
10 |   for(unsigned n=0; n < 50000; ++n) {
11 |     sqw.addValue({{"test", 1},
12 |                   {"value", 2},
13 |                   {"name", "bert"},
14 |                   {"temp", 12.0}, {"n", n}, {"cool", (n%2) ? false : true}});
15 |   }
16 |   for(unsigned n=0; n < 5000; ++n) {
17 |     sqw.addValue({{"joh", 1.1*n}, {"wuh", to_string(n)+"_uhuh"}}, "vals");
18 |   }
19 |   for(unsigned n=0; n < 50000; ++n) {
20 |     sqw.addValue({{"pief", 1},
21 |                   {"poef", 2},
22 |                   {"paf", "bert"},
23 |                   {"temp", 12.0}, {"n", n}, {"cool", (n%2) ? false : true}});
24 |   }
25 | 
26 |   cout<<"Done with adding"<<endl;
27 | 
28 | }
29 | catch (const std::exception &e)
30 | {
31 |   std::cerr << e.what() << std::endl;
32 |   return 1;
33 | }
34 | 


--------------------------------------------------------------------------------
/jsonhelper.cc:
--------------------------------------------------------------------------------
 1 | #include "sqlwriter.hh"
 2 | #include "nlohmann/json.hpp"
 3 | 
 4 | using namespace std;
 5 | 
 6 | nlohmann::json packResultsJson(const vector<unordered_map<string, MiniSQLite::outvar_t>>& result, bool fillnull)
 7 | {
 8 |   nlohmann::json arr = nlohmann::json::array();
 9 |   
10 |   for(const auto& row : result) {
11 |     nlohmann::json j;
12 |     for(auto& col : row) {
13 |       std::visit([&j, &col, &fillnull](auto&& arg) {
14 |         using T = std::decay_t<decltype(arg)>;
15 |         if constexpr (std::is_same_v<T, nullptr_t>) {
16 |           if(fillnull)
17 | 	    j[col.first]="";
18 | 	  else
19 | 	    return;
20 |         }
21 |         else  {
22 |           j[col.first] = arg;
23 |         }
24 |       }, col.second);
25 |     }
26 |     arr += j;
27 |   }
28 |   return arr;
29 | }
30 | 
31 | string packResultsJsonStr(const vector<unordered_map<string, MiniSQLite::outvar_t>>& result, bool fillnull)
32 | {
33 |   return packResultsJson(result, fillnull).dump();
34 | }
35 | 


--------------------------------------------------------------------------------
/meson.build:
--------------------------------------------------------------------------------
 1 | project('sqlitewriter', 'cpp', default_options : ['cpp_std=c++17'])
 2 | 
 3 | sqlitedep = dependency('sqlite3', version : '>3')
 4 | 
 5 | threaddep = dependency('threads')
 6 | 
 7 | jsondep = dependency('nlohmann_json')
 8 | 
 9 | #add_executable(demo demo.cc sqlwriter.cc)
10 | #target_link_libraries(demo  sqlite3  pthread  dl  )
11 | 
12 | sqlitewriter_lib = library(
13 |   'sqlitewriter',
14 |   'sqlwriter.cc',
15 |   'jsonhelper.cc',
16 |   install: false,
17 |   include_directories: '',
18 |   dependencies: [sqlitedep, threaddep, jsondep]
19 | )
20 | 
21 | sqlitewriter_dep = declare_dependency(
22 |   link_with: sqlitewriter_lib,
23 |   include_directories: '',
24 | )
25 | 
26 | if meson.version().version_compare('>=0.54.0')
27 |   meson.override_dependency('sqlitewriter', sqlitewriter_dep)
28 | endif
29 | 
30 | 
31 | executable('demo', 'demo.cc', 'sqlwriter.cc',  'jsonhelper.cc',
32 | 	dependencies: [sqlitedep, threaddep, jsondep])
33 | 
34 | executable('testrunner', 'testrunner.cc', 'sqlwriter.cc',  'jsonhelper.cc',
35 | 	dependencies: [sqlitedep, threaddep, jsondep])
36 | 
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 bert hubert
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/.github/workflows/cmake.yml:
--------------------------------------------------------------------------------
 1 | name: CMake
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ main ]
 6 |   pull_request:
 7 |     branches: [ main ]
 8 | 
 9 | env:
10 |   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
11 |   BUILD_TYPE: Release
12 | 
13 | 
14 | jobs:
15 |   build:
16 |     # The CMake configure and build commands are platform agnostic and should work equally
17 |     # well on Windows or Mac.  You can convert this to a matrix build if you need
18 |     # cross-platform coverage.
19 |     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
20 |     runs-on: ubuntu-latest
21 | 
22 |     steps:
23 |     - uses: actions/checkout@v3
24 | 
25 |     - name: Install sqlite
26 |       run: sudo apt-get install libsqlite3-dev libpq-dev nlohmann-json3-dev
27 | 
28 |     - name: Configure CMake
29 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
30 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
31 |       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
32 | 
33 |     - name: Build
34 |       # Build your program with the given configuration
35 |       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
36 | 
37 |     - name: Test
38 |       working-directory: ${{github.workspace}}/build
39 |       # Execute tests defined by the CMake configuration.  
40 |       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
41 |       run: ctest -C ${{env.BUILD_TYPE}}
42 |       
43 | 


--------------------------------------------------------------------------------
/jsontests.cc:
--------------------------------------------------------------------------------
 1 | #include "ext/doctest.h"
 2 | #include "sqlwriter.hh"
 3 | #include "jsonhelper.hh"
 4 | #include <unistd.h>
 5 | using namespace std;
 6 | 
 7 | TEST_CASE("basic json test") {
 8 |   unlink("testrunner-example.sqlite3");
 9 |   {
10 |     SQLiteWriter sqw("testrunner-example.sqlite3");
11 |     sqw.addValue({{"pief",1}});
12 |     sqw.addValue({{"paf",12.0}});
13 |     sqw.addValue({{"user", "ahu"}, {"paf", 14}});
14 |     sqw.addValue({{"user", "jhu"}, {"paf", 14.23}, {"pief", 99}});
15 |   }
16 |   // best way to guarantee that you can query the data you inserted is by closing the connection
17 |   SQLiteWriter sqw("testrunner-example.sqlite3");
18 |   auto res = sqw.queryT("select * from data");
19 | 
20 |   //  cout << packResultsJsonStr(res) << endl;
21 |   
22 |   nlohmann::json js = packResultsJson(res);
23 |   CHECK(js[0]["pief"] == 1);
24 |   CHECK(js[1]["paf"] == 12.0);
25 |   CHECK(js[2]["user"] == "ahu");
26 |   
27 |   CHECK(res.size() == 4);
28 | 
29 |   res = sqw.queryT("select 1.0*sum(pief) as p from data");
30 |   js = packResultsJson(res);
31 |   CHECK(js[0]["p"]==100.0);
32 |   CHECK(get<double>(res[0]["p"])==100.0);
33 |   unlink("testrunner-example.sqlite3");
34 | }
35 | 
36 | 
37 | TEST_CASE("json helper")
38 | {
39 |   SQLiteWriter sqw("");
40 |   sqw.addValue({{"id", 1}, {"user", "ahu"}});
41 |   sqw.addValue({{"id", 2}});
42 |   auto res = sqw.queryT("select * from data");
43 |   REQUIRE(res.size() == 2);
44 | 
45 |   auto j = packResultsJson(res, false);
46 |   REQUIRE(j.size() == 2);
47 |   CHECK(j[0].count("user") == 1);
48 |   CHECK(j[1].count("user") == 0);
49 | 
50 |   j = packResultsJson(res);
51 |   REQUIRE(j.size() == 2);
52 |   CHECK(j[0].count("user") == 1);
53 |   CHECK(j[1].count("user") == 1);
54 |   string user = j[1]["user"];
55 |   CHECK(user =="");
56 | }
57 | 


--------------------------------------------------------------------------------
/minipsql.hh:
--------------------------------------------------------------------------------
 1 | #include <postgresql/libpq-fe.h>
 2 | #include <vector>
 3 | #include <string>
 4 | #include <unordered_map>
 5 | 
 6 | class MiniPSQL
 7 | {
 8 | public:
 9 |   MiniPSQL(std::string_view fname);
10 |   ~MiniPSQL();
11 |   std::vector<std::pair<std::string, std::string>> getSchema(const std::string& table);
12 |   void addColumn(const std::string& table, std::string_view name, std::string_view type);
13 |   
14 |   //!< execute a random query, for example a PRAGMA
15 |   std::vector<std::vector<std::string>> exec(const std::string& query);
16 |   //  std::vector<std::vector<std::string>> exec(const std::string& query, const std::vector<std::string>& params);
17 |   std::vector<std::vector<std::string>> exec(const std::string& query, std::vector<const char*> params);
18 | 
19 |   //!< set the prepared statement for a table, question marks as placeholder
20 |   void prepare(const std::string& table, std::string_view str, unsigned int paramsize);
21 |   // offset from 1!!
22 |   template<typename T>
23 |   void bindPrep(const std::string& table, int idx, const T& value)
24 |   {
25 |     bindPrep(table, idx, std::to_string(value));
26 |   }
27 | 
28 |   void bindPrep(const std::string& table, int idx, const std::string& value)
29 |   {
30 |     int pos = idx-1;
31 |     if(d_params[table].size() <= pos)
32 |       d_params[table].resize(pos+1);
33 |     d_params[table][pos]=value;
34 |   }
35 |   
36 |   //!< execute the prepared & bound statement
37 |   void execPrep(const std::string& table);
38 |   
39 |   void begin();
40 |   void commit();
41 |   void cycle();
42 |   
43 |   //!< do we have a prepared statement for this table
44 |   bool isPrepared(const std::string& table) const
45 |   {
46 |     return d_stmts.find(table) != d_stmts.end();
47 |   }
48 | 
49 | private:
50 |   PGconn* d_conn;
51 | 
52 |   std::unordered_map<std::string, std::string> d_stmts; // keyed on table name
53 |   std::unordered_map<std::string, std::vector<std::string>> d_params; // keyed on table name
54 | 
55 |   bool d_intransaction{false};
56 |   bool haveTable(const std::string& table);
57 | };
58 | 


--------------------------------------------------------------------------------
/psqlwriter.hh:
--------------------------------------------------------------------------------
 1 | #include <string>
 2 | #include <variant>
 3 | #include <vector>
 4 | #include <thread>
 5 | #include <unordered_map>
 6 | #include <unistd.h>
 7 | 
 8 | /* ok for a remote database, two things are very different:
 9 |    1) We need to stream or batch our inserts, otherwise we get killed by latency
10 |    2) We need isolation from database server failures
11 | 
12 |    The way to do this is to send all the inserts to a worker thread, which takes
13 |    care of the batching, sending, reconnecting etc.
14 | 
15 |    There can be multiple tables, which means the batching needs to happen per
16 |    table. There could be multiple insert signatures, which makes batching harder.
17 |    It may be possibe to use NULLs to create unified signatures though.
18 | 
19 | */
20 | 
21 | 
22 | class PSQLWriter
23 | {
24 | 
25 | public:
26 |   explicit PSQLWriter(std::string_view fname) 
27 |   {
28 |     pipe2(d_pipe, 0); //O_NONBLOCK);
29 |     d_thread = std::thread(&PSQLWriter::commitThread, this);
30 |   }
31 |   typedef std::variant<double, int32_t, uint32_t, int64_t, std::string> var_t;
32 |   void addValue(const std::initializer_list<std::pair<const char*, var_t>>& values, const std::string& table="data")
33 |   {
34 |     addValueGeneric(table, values);
35 |   }
36 |   
37 |   void addValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table="data")
38 |   {
39 |     addValueGeneric(table, values);
40 |   }
41 |   
42 |   template<typename T>
43 |   void addValueGeneric(const std::string& table, const T& values);
44 |   ~PSQLWriter()
45 |   {
46 |     //    std::cerr<<"Destructor called"<<std::endl;
47 |     close(d_pipe[1]); // this is the pleasequit signal
48 |     d_thread.join();
49 |   }
50 | 
51 | private:
52 |   void commitThread();
53 |   bool d_pleasequit{false};
54 |   std::thread d_thread;
55 | 
56 |   int d_pipe[2]; // [0] = read, [1] = write
57 |   
58 |   std::unordered_map<std::string, std::vector<std::pair<std::string, std::string>>> d_columns;
59 |   std::unordered_map<std::string, std::vector<std::string>> d_lastsig;
60 |   bool haveColumn(const std::string& table, std::string_view name);
61 | 
62 |   struct Message
63 |   {
64 |     std::string table;
65 |     std::unordered_map<std::string, var_t> values;
66 |   };
67 | };
68 | 
69 | 
70 | template<typename T>
71 | void PSQLWriter::addValueGeneric(const std::string& table, const T& values)
72 | {
73 |   auto msg = new Message({table});
74 |   for(const auto& v : values) {
75 |     msg->values[v.first] = v.second;
76 |   }
77 |   write(d_pipe[1], &msg, sizeof(msg));
78 | }
79 | 


--------------------------------------------------------------------------------
/sqlwriter.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <string>
  3 | #include <vector>
  4 | #include <unordered_map>
  5 | #include <variant>
  6 | #include <mutex>
  7 | #include <thread>
  8 | #include <iostream>
  9 | #include <map>
 10 | #include <atomic>
 11 | #include <optional>
 12 | struct sqlite3;
 13 | struct sqlite3_stmt;
 14 | 
 15 | enum class SQLWFlag
 16 | {
 17 |   NoFlag, ReadOnly, NoTransactions
 18 | };
 19 | 
 20 | 
 21 | class MiniSQLite
 22 | {
 23 | public:
 24 |   MiniSQLite(std::string_view fname, SQLWFlag = SQLWFlag::NoFlag);
 25 |   ~MiniSQLite();
 26 |   std::vector<std::pair<std::string, std::string>> getSchema(const std::string& table);
 27 |   void addColumn(const std::string& table, std::string_view name, std::string_view type, const std::string& meta=std::string());
 28 |   std::vector<std::vector<std::string>> exec(std::string_view query);
 29 |   void prepare(const std::string& table, std::string_view str);
 30 |   void bindPrep(const std::string& table, int idx, bool value);
 31 |   void bindPrep(const std::string& table, int idx, int value);
 32 |   void bindPrep(const std::string& table, int idx, uint32_t value);
 33 |   void bindPrep(const std::string& table, int idx, long value);
 34 |   void bindPrep(const std::string& table, int idx, unsigned long value);
 35 |   void bindPrep(const std::string& table, int idx, long long value); 
 36 |   void bindPrep(const std::string& table, int idx, unsigned long long value);
 37 |   void bindPrep(const std::string& table, int idx, double value);
 38 |   void bindPrep(const std::string& table, int idx, const std::string& value);
 39 |   void bindPrep(const std::string& table, int idx, const std::vector<uint8_t>& value);
 40 | 
 41 |   typedef std::variant<double, int64_t, std::string, std::vector<uint8_t>, std::nullptr_t> outvar_t; 
 42 |   void execPrep(const std::string& table, std::vector<std::unordered_map<std::string, outvar_t>>* rows=0, unsigned int msec=0); 
 43 |   void begin();
 44 |   void commit();
 45 |   void cycle();
 46 |   bool isPrepared(const std::string& table) const
 47 |   {
 48 |     if(auto iter = d_stmts.find(table); iter == d_stmts.end())
 49 |       return false;
 50 |     else
 51 |       return iter->second != nullptr;
 52 |   }
 53 |   static std::atomic<uint64_t> s_execs, s_sorts, s_fullscans, s_autoindexes;
 54 |   
 55 | private:
 56 |   sqlite3* d_sqlite;
 57 |   std::unordered_map<std::string, sqlite3_stmt*> d_stmts;
 58 |   std::vector<std::vector<std::string>> d_rows; // for exec()
 59 |   static int helperFunc(void* ptr, int cols, char** colvals, char** colnames);
 60 |   bool d_intransaction{false};
 61 |   bool haveTable(const std::string& table);
 62 | };
 63 | 
 64 | class SQLiteWriter
 65 | {
 66 | public:
 67 |   explicit SQLiteWriter(std::string_view fname,
 68 | 			const std::map<std::string,
 69 | 			 std::map<std::string,std::string>>& meta =
 70 | 			std::map<std::string,std::map<std::string,std::string>>(),
 71 | 			SQLWFlag flag = SQLWFlag::NoFlag) : d_db(fname, flag), d_flag(flag)
 72 |   {
 73 |     d_db.exec("PRAGMA journal_mode='wal'");
 74 |     if(flag != SQLWFlag::ReadOnly && flag != SQLWFlag::NoTransactions) {
 75 |       d_db.begin(); // open the transaction
 76 |       d_thread = std::thread(&SQLiteWriter::commitThread, this);
 77 |     }
 78 |     d_meta = meta;
 79 |   }
 80 | 
 81 |   explicit SQLiteWriter(std::string_view fname, const std::map<std::string,std::string>& meta) : SQLiteWriter(fname, {{"data", meta}})
 82 |   {}
 83 | 
 84 |   explicit SQLiteWriter(std::string_view fname, SQLWFlag flag) : SQLiteWriter(fname, {{{}}}, flag)
 85 |   {}
 86 | 
 87 |   
 88 |   typedef std::variant<double, int32_t, uint32_t, int64_t, std::string, std::vector<uint8_t>> var_t;
 89 |   void addValue(const std::initializer_list<std::pair<const char*, var_t>>& values, const std::string& table="data");
 90 |   void addValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table="data");
 91 | 
 92 |   void addOrReplaceValue(const std::initializer_list<std::pair<const char*, var_t>>& values, const std::string& table="data");
 93 |   void addOrReplaceValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table="data");
 94 | 
 95 |   template<typename T>
 96 |   void addValueGeneric(const std::string& table, const T& values, bool replace=false);
 97 | 
 98 |   ~SQLiteWriter()
 99 |   {
100 |     d_pleasequit=true;
101 |     if(d_thread) 
102 |       d_thread->join();
103 |   }
104 | 
105 |   // This is an odd function for a writer - it allows you to do simple queries & get back result as a vector of maps
106 |   // note that this function may very well NOT be coherent with addValue
107 |   // this function is useful for getting values of counters before logging for example
108 |   std::vector<std::unordered_map<std::string,std::string>> query(const std::string& q, const std::initializer_list<var_t>& values = std::initializer_list<var_t>());
109 | 
110 |   // same, but typed
111 |   std::vector<std::unordered_map<std::string,MiniSQLite::outvar_t>> queryT(const std::string& q, const std::initializer_list<var_t>& values = std::initializer_list<var_t>(), unsigned int msec=0);
112 |   
113 | private:
114 |   void commitThread();
115 |   bool d_pleasequit{false};
116 |   std::optional<std::thread> d_thread;
117 |   std::mutex d_mutex;  
118 |   MiniSQLite d_db;
119 |   SQLWFlag d_flag{SQLWFlag::NoFlag};
120 |   std::unordered_map<std::string, std::vector<std::pair<std::string, std::string>>> d_columns;
121 |   std::unordered_map<std::string, std::vector<std::string>> d_lastsig;
122 |   std::unordered_map<std::string, bool> d_lastreplace;
123 |   std::map<std::string, std::map<std::string, std::string>> d_meta;
124 | 
125 |   bool haveColumn(const std::string& table, std::string_view name);
126 |   template<typename T>
127 |   std::vector<std::unordered_map<std::string, MiniSQLite::outvar_t>> queryGen(const std::string& q, const T& values, unsigned int mesec=0);
128 | };
129 | 


--------------------------------------------------------------------------------
/minipsql.cc:
--------------------------------------------------------------------------------
  1 | #include "minipsql.hh"
  2 | #include "sqlwriter.hh"
  3 | #include <algorithm>
  4 | using namespace std;
  5 | 
  6 | 
  7 | MiniPSQL::MiniPSQL(std::string_view fname)
  8 | {
  9 |   d_conn = PQconnectdb(&fname[0]);
 10 |   if (PQstatus(d_conn) != CONNECTION_OK) {
 11 |     throw std::runtime_error("Error connecting to postgresql: "+ string(PQerrorMessage(d_conn)));
 12 |   }  
 13 | }
 14 | 
 15 | MiniPSQL::~MiniPSQL()
 16 | {
 17 |   if(d_intransaction)
 18 |     commit();
 19 | 
 20 |   PQfinish(d_conn);
 21 | }
 22 | 
 23 | 
 24 | struct QueryResult
 25 | {
 26 |   explicit QueryResult(PGconn* conn, const std::string& query)
 27 |   {
 28 |     d_res = PQexec(conn, query.c_str());
 29 |     if(PQresultStatus(d_res) == PGRES_COMMAND_OK) {
 30 |       d_ntuples = d_nfields = 0;
 31 |     }
 32 |     else if (PQresultStatus(d_res) != PGRES_TUPLES_OK) {
 33 |       PQclear(d_res);
 34 |       throw std::runtime_error(string("query error: ") + PQerrorMessage(conn));
 35 |     }
 36 |     d_ntuples = PQntuples(d_res);
 37 |     d_nfields = PQnfields(d_res);
 38 |   }
 39 | 
 40 |   explicit QueryResult(PGconn* conn, const std::string& table, const std::string& query, int paramsize)
 41 |   {
 42 |     d_res = PQprepare(conn, ("procedure_"+table).c_str(), query.c_str(), paramsize, NULL);
 43 | 
 44 |     if(PQresultStatus(d_res) != PGRES_COMMAND_OK) {
 45 |       PQclear(d_res);
 46 |       throw std::runtime_error(string("prepare error: ") + PQerrorMessage(conn));
 47 |     }
 48 |     PQclear(d_res);
 49 |     d_res=0;
 50 |   }
 51 | 
 52 |   explicit QueryResult(PGconn* conn, const std::string& query, const std::vector<const char*>& params)
 53 |   {
 54 |     d_res = PQexecParams(conn, query.c_str(), params.size(), NULL, &params[0], NULL, NULL, 0);
 55 |     
 56 |     if (PQresultStatus(d_res) == PGRES_COMMAND_OK) {
 57 |       d_ntuples = d_nfields = 0;
 58 |     }
 59 |     else if (PQresultStatus(d_res) != PGRES_TUPLES_OK) {
 60 |       PQclear(d_res);
 61 |       throw std::runtime_error(string("parameter query error: ") + PQerrorMessage(conn));
 62 |     }
 63 |     d_ntuples = PQntuples(d_res);
 64 |     d_nfields = PQnfields(d_res);
 65 |   }
 66 |   
 67 |   explicit QueryResult(PGconn* conn, const std::string& table, const std::vector<string>& params)
 68 |   {
 69 |     vector<const char*> pms;
 70 |     for(const auto& p : params) {
 71 |       //      cout<<"Adding param: '"<<p<<"'\n";
 72 |       pms.push_back(p.c_str());
 73 |     }
 74 |     
 75 |     d_res = PQexecPrepared(conn, ("procedure_"+table).c_str(), params.size(), &pms[0], NULL, NULL, 0);
 76 |     
 77 |     if (PQresultStatus(d_res) == PGRES_COMMAND_OK) {
 78 |       d_ntuples = d_nfields = 0;
 79 |     }
 80 |     else if (PQresultStatus(d_res) != PGRES_TUPLES_OK) {
 81 |       PQclear(d_res);
 82 |       throw std::runtime_error(string("prepared query error: ") + PQerrorMessage(conn));
 83 |     }
 84 |     d_ntuples = PQntuples(d_res);
 85 |     d_nfields = PQnfields(d_res);
 86 |   }
 87 | 
 88 |   
 89 |   vector<string> getRow()
 90 |   {
 91 |     vector<string> ret;
 92 | 
 93 |     if(d_row  < d_ntuples) {
 94 |       for (unsigned int j = 0; j < d_nfields; j++)
 95 |         ret.push_back(PQgetvalue(d_res, d_row, j));
 96 |     }
 97 |     d_row++;
 98 |     return ret;
 99 |   }
100 |   
101 |   ~QueryResult()
102 |   {
103 |     if(d_res)
104 |       PQclear(d_res);
105 |   }
106 |   PGresult* d_res=0;
107 |   unsigned int d_row=0;
108 |   unsigned int d_ntuples;
109 |   unsigned int d_nfields;
110 | };
111 | 
112 | std::vector<std::vector<std::string>> MiniPSQL::exec(const std::string& query)
113 | {
114 |   std::vector<std::vector<std::string>> ret;
115 | 
116 |   QueryResult qr(d_conn, query);
117 |   for(;;) {
118 |     auto row = qr.getRow();
119 |     if(row.empty())
120 |       break;;
121 |     ret.push_back(row);
122 |   }
123 | 
124 |   return ret;
125 | }
126 | 
127 | std::vector<std::vector<std::string>> MiniPSQL::exec(const std::string& query, vector<const char*> params)
128 | {
129 |   std::vector<std::vector<std::string>> ret;
130 | 
131 |   QueryResult qr(d_conn, query, params);
132 |   for(;;) {
133 |     auto row = qr.getRow();
134 |     if(row.empty())
135 |       break;;
136 |     ret.push_back(row);
137 |   }
138 | 
139 |   return ret;
140 | }
141 | 
142 | 
143 | void MiniPSQL::execPrep(const std::string& table)
144 | {
145 |   QueryResult qr(d_conn, table, d_params[table]);
146 |   d_params[table].clear();
147 | }
148 | 
149 | void MiniPSQL::addColumn(const std::string& table, std::string_view name, std::string_view type)
150 | {
151 |   // SECURITY PROBLEM - somehow we can't do prepared statements here
152 |   
153 |   if(!haveTable(table)) {
154 |     exec("create table if not exists "+table+" ( "+(string)name+" "+(string)type+" )");
155 |   } else {
156 |     //    cout<<"Adding column "<<name<<" to table "<<table<<endl;
157 |     exec("ALTER table \""+table+"\" add column \""+string(name)+ "\" "+string(type));
158 |   }
159 | 
160 | }
161 | 
162 | //! Get field names and types from a table
163 | vector<pair<string,string> > MiniPSQL::getSchema(const std::string& table)
164 | {
165 |   vector<pair<string,string>> ret;
166 |   
167 |   auto rows = exec("SELECT column_name, udt_name FROM information_schema.columns where table_name='"+table+"'");
168 | 
169 |   for(const auto& r : rows) {
170 |     ret.push_back({r[0], r[1]});
171 |   }
172 |   sort(ret.begin(), ret.end(), [](const auto& a, const auto& b) {
173 |     return a.first < b.first;
174 |   });
175 | 
176 |   //  cout<<"returning "<<ret.size()<<" rows for table "<<table<<"\n";
177 |   return ret;
178 | }
179 | 
180 | void MiniPSQL::prepare(const std::string& table, std::string_view str, unsigned int paramsize)
181 | {
182 |   cout<<"prep"<<endl;
183 |   if(!d_stmts[table].empty()) {
184 |     cout<<"dealloc!"<<endl;
185 |     exec("deallocate procedure_"+table);
186 |   }
187 | 
188 |   d_stmts[table]=str;
189 |   d_params[table].clear();
190 |   QueryResult qr(d_conn, (string)table, (string)str, paramsize);
191 | }
192 | 
193 | void MiniPSQL::begin()
194 | {
195 |   d_intransaction=true;
196 |   exec("begin");
197 | }
198 | void MiniPSQL::commit()
199 | {
200 |   d_intransaction=false;
201 |   exec("commit");
202 | }
203 | 
204 | void MiniPSQL::cycle()
205 | {
206 |   exec("commit;begin");
207 | }
208 | 
209 | bool MiniPSQL::haveTable(const string& table)
210 | {
211 |   return !getSchema(table).empty();
212 | }
213 | 
214 | 
215 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # sqlitewriter
  2 | csv-like storage to sqlite and PostgreSQL. Easy to use, type safe:
  3 | 
  4 | ```C++
  5 | SQLiteWriter sqw("example.sqlite3");
  6 | 
  7 | for(int n = 0 ; n < 1000000; ++n) {
  8 |   sqw.addValue({{"pief", n}, {"poef", 1.1*n}, {"paf", "bert"}});
  9 | }
 10 | 
 11 | sqw.addValue({{"timestamp", 1234567890}});
 12 | ```
 13 | 
 14 | This will create `example.sqlite3` if needed. Also, if needed it will create
 15 | a table called `data`. The sqlite3 file is set to WAL mode, the table is set
 16 | to STRICT mode:
 17 | 
 18 | ```
 19 | sqlite> PRAGMA journal_mode;
 20 | wal
 21 | 
 22 | sqlite> .schema
 23 | CREATE TABLE data ( pief INT , "poef" REAL, "paf" TEXT, "timestamp" INT) STRICT;
 24 | ```
 25 | 
 26 | The `addValue()` method can be used to pass an arbitrary number of fields.
 27 | Columns will automatically be generated in the table on their first use. 
 28 | 
 29 | Note that strings are stored as TEXT which means they benefit from utf-8 processing. If you don't want that, store `vector<uint8_t>`, which will end up as a blob.
 30 | 
 31 | You can also add data to other tables than `data` like this:
 32 | 
 33 | ```C++
 34 | // put data in table logins
 35 | sqw.addValue({{"timestamp", 1234567890}}, "logins");
 36 | ```
 37 | 
 38 | SQLiteWriter batches all writes in transactions. The transaction is cycled
 39 | every second. Because WAL mode is used, readers can be active without
 40 | interrupting writes to the file.
 41 | 
 42 | In case you are trying to read from the file while sqlwriter is trying to
 43 | write to it, a busy handler is implemented. This will not block your main
 44 | process, since writes are performed from a writing thread.
 45 | 
 46 | Prepared statements are automatically created and reused as long as the same
 47 | fields are passed to `addValue()`. If different fields are passed, a new
 48 | prepared statement is made.
 49 | 
 50 | The code above does around 550k inserts/second. This could likely be
 51 | improved further, but it likely suffices for most usecases.
 52 | 
 53 | # Compiling
 54 | Make sure you have a C++ compiler installed, plus CMake and of course the SQLite3 development files. In addition, the tests also exercise the optional JSON helpers. These are part of nlohmann-json3-dev or find the single include file needed [here](https://github.com/nlohmann/json/releases).
 55 | 
 56 | ```
 57 | git clone https://github.com/berthubert/sqlitewrite.git 
 58 | cd sqlitewrite
 59 | cmake .
 60 | make
 61 | ./testrunner
 62 | ```
 63 | 
 64 | # Using in your project
 65 | Simply drop `sqlitewriter.cc` and `sqlitewriter.hh` in your project, and
 66 | link in libsqlite3-dev (or if needed, the amalgamated sqlite3 source). If you want to benefit from the JSON helpers below, also compile in `jsonhelper.cc`, and include `jsonhelper.hh`.
 67 | 
 68 | # PostgreSQL version
 69 | Slightly less tested, but you can get the same API if you add psqlwriter.{cc,hh} and minipsql.{cc,hh} to your project. The class is called `PSQLWriter`, and instead of a filename you pass a connection string.
 70 | 
 71 | Despite valiant efforts, the PostgreSQL version is 5 times slower than the SQLite version (at least). However, if you want your logged data to be available remotely, this is still a very good option.
 72 | 
 73 | The reason for the relative slowness is mostly down to the interprocess/network latency. The code attempts to batch a lot and use efficient APIs, but simply serializing all the data eats up a lot of CPU.
 74 | 
 75 | # The (sqlite) writer can also read
 76 | Although it is a bit of anomaly, you can also use sqlitewriter to perform queries:
 77 | 
 78 | ```
 79 | SQLiteWriter sqw("example.sqlite3");
 80 | auto res = sqw.query("select * from data where pief = ?", {123});
 81 | cout << res.at(0)["pief"] << endl; // should print 123
 82 | ```
 83 | 
 84 | This returns a vector of rows, each represented by an `unordered_map<string,string>`. Alternatively you can use `queryT` which returns `std::variant<int64_t, double, string, nullptr_t, vector<uint8_t>>` values, which means you can benefit from the typesafety. Note that all integer numbers end  up as signed int64_t in this variant union.
 85 | 
 86 | Also note that these functions could very well not be coherent with `addValue()` because of buffering. This is by design.  The `query` function is very useful for getting the value of counters for example, so you can use these for subsequent logging. But don't count on a value you just wrote with `addValue()` to appear in an a `query()` immediately. 
 87 | 
 88 | To be on the safe side, don't interleave calls to `query()` with calls to `addValue()`.
 89 | 
 90 | ## Readonly mode
 91 | When using SQLiteWriter to only perform reads, it is possible to set read only mode. This instructs sqlite3 to not permit changes, and it als makes sqlitewriter not create a writing thread.
 92 | 
 93 | From the unit tests:
 94 | ```C++
 95 | ...
 96 |   SQLiteWriter ro(fname, SQLWFlag::ReadOnly);
 97 |   auto res = ro.queryT("select * from data where some='stuff'");
 98 |   REQUIRE(res.size() == 1);
 99 | 
100 |   REQUIRE_THROWS_AS(ro.addValue({{"some", "more"}}), std::exception);
101 |   res = ro.queryT("select count(1) c from data");
102 |   REQUIRE(res.size() == 1);
103 |   CHECK(get<int64_t>(res[0]["c"]) == 1);
104 | ```
105 | 
106 | In readonly mode, you can also pass an extra field to queryT, specifying how many milliseconds the query should be allowed to run. An exception is thrown if this time period is exceeded.
107 | 
108 | ## NoTransactions
109 | When SQLiteWriter is used for bulk data imports, the transaction mode works
110 | very well and presents a massive speedup.
111 | 
112 | When doing occasional reads and writes however the transactions can cause
113 | locking problems and SQLITE_BUSY errors because of sqlite's transaction
114 | model. 
115 | 
116 | By setting the SQLWFlag::NoTransactions, no transactions are opened and
117 | closed. This also means no thread is created for managing these
118 | transactions.
119 | 
120 | ## JSON helper
121 | It is often convenient to turn your query results into JSON. The helpers `packResultJson` and `packResultJsonStr` in `jsonhelper.cc` benefit from the typesafety provided by `queryT` to create JSON that knows that 1.0 is not "1.0":
122 | 
123 | ```C++
124 |   {
125 |     SQLiteWriter sqw("testrunner-example.sqlite3");
126 |     sqw.addValue({{"pief",1}});
127 |     sqw.addValue({{"paf",12.0}});
128 |     sqw.addValue({{"user", "ahu"}, {"paf", 14}});
129 |     sqw.addValue({{"user", "jhu"}, {"paf", 14.23}, {"pief", 99}});
130 |   }
131 |   // best way to guarantee that you can query the data you inserted 
132 |   // is by closing the connection
133 |   SQLiteWriter sqw("testrunner-example.sqlite3");
134 |   auto res = sqw.queryT("select * from data");
135 | 
136 |   cout << packResultsJsonString(res) << endl;
137 | ```
138 | 
139 | Slightly reformatted, this prints:
140 | ```
141 | [
142 |  {"pief":1},
143 |  {"paf":12.0},
144 |  {"paf":14.0, "user":"ahu"},
145 |  {"paf":14.23, "pief":99, "user":"jhu"}
146 | ]
147 | ```
148 | 
149 | # Advanced features
150 | ## Per-field metadata
151 | SQLiteWriter creates populates tables for you automatically, setting the correct type per field. But sometimes you want to add some additional instructions. For example, you might want to declare that a field is case-insensitive. To do so, you can add metadata per field, like this:
152 | 
153 | ```C++
154 | SQLiteWriter sqw("example.sqlite3", {{"name", "collate nocase"}});
155 | ```
156 | 
157 | This will lead SQLiteWriter to add 'collate nocase' when the 'name' field is created for the main 'data' table. To specify for specific tables, use this syntax:
158 | 
159 | ```C++
160 | SQLiteWriter sqw("example.sqlite.sqlite3", {{"second", 
161 |     {{"user", "UNIQUE"}}
162 |     }});
163 | ```
164 | 
165 | Note that this is also the only time when this metadata is used. If you have a pre-existing database that already has a 'name' field, it will not be changed to 'collate nocase'.
166 | 
167 | 
168 | # Status
169 | I use it as infrastructure in some of my projects. Seems to work well. 
170 | 


--------------------------------------------------------------------------------
/psqlwriter.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <stdexcept>
  3 | #include <poll.h>
  4 | #include "minipsql.hh"
  5 | #include <string.h>
  6 | #include <unistd.h>
  7 | #include <variant>
  8 | #include <thread>
  9 | #include <mutex>
 10 | #include <fcntl.h>
 11 | #include <unordered_set>
 12 | #include <unordered_map>
 13 | #include <optional>
 14 | #include <chrono>
 15 | #include <algorithm>
 16 | #include "psqlwriter.hh"
 17 | #include <map>
 18 | using namespace std;
 19 | 
 20 | struct DTime
 21 | {
 22 |   void start()
 23 |   {
 24 |     d_start =   std::chrono::steady_clock::now();
 25 |   }
 26 |   uint32_t lapUsec()
 27 |   {
 28 |     auto usec = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()- d_start).count();
 29 |     start();
 30 |     return usec;
 31 |   }
 32 | 
 33 |   std::chrono::time_point<std::chrono::steady_clock> d_start;
 34 | };
 35 | 
 36 | // seconds
 37 | static int waitForRWData(int fd, bool waitForRead, double* timeout, bool* error=0, bool* disconnected=0)
 38 | {
 39 |   int ret;
 40 | 
 41 |   struct pollfd pfd;
 42 |   memset(&pfd, 0, sizeof(pfd));
 43 |   pfd.fd = fd;
 44 | 
 45 |   if(waitForRead)
 46 |     pfd.events=POLLIN;
 47 |   else
 48 |     pfd.events=POLLOUT;
 49 | 
 50 |   ret = poll(&pfd, 1, timeout ? (*timeout * 1000) : -1);
 51 |   if ( ret == -1 ) {
 52 |     throw std::runtime_error("Waiting for data: "+std::string(strerror(errno)));
 53 |   }
 54 |   if(ret > 0) {
 55 |     if (error && (pfd.revents & POLLERR)) {
 56 |       *error = true;
 57 |     }
 58 |     if (disconnected && (pfd.revents & POLLHUP)) {
 59 |       *disconnected = true;
 60 |     }
 61 | 
 62 |   }
 63 |   return ret;
 64 | }
 65 | 
 66 | int waitForData(int fd, double timeout)
 67 | {
 68 |   return waitForRWData(fd, true, &timeout);
 69 | }
 70 | 
 71 | void SetNonBlocking(int sock, bool to)
 72 | {
 73 |   int flags=fcntl(sock,F_GETFL,0);
 74 |   if(flags<0)
 75 |     std::runtime_error(string("Retrieving socket flags: ")+ strerror(errno));
 76 | 
 77 |   // so we could optimize to not do it if nonblocking already set, but that would be.. semantics
 78 |   if(to) {
 79 |     flags |= O_NONBLOCK;
 80 |   }
 81 |   else 
 82 |     flags &= (~O_NONBLOCK);
 83 |       
 84 |   if(fcntl(sock, F_SETFL, flags) < 0)
 85 |     std::runtime_error(string("Setting socket flags: ")+ strerror(errno));
 86 | }
 87 | 
 88 | 
 89 | void PSQLWriter::commitThread()
 90 | {
 91 |   MiniPSQL mp("");
 92 |   mp.exec("begin");
 93 | 
 94 |   map<string,vector<pair<string,string>>> schemas;
 95 | 
 96 |   // so how does this work
 97 |   // we get a stream of messages, each aimed at a single table
 98 |   // we group the messages by table, and check if the table exists, and if all fields exist
 99 |   // if not, we create tables and fields to match
100 |   SetNonBlocking(d_pipe[0], true);
101 |   bool needWait = false;
102 |   time_t prevcommit=time(0);
103 |   for(;;) {
104 |     map<string, vector<Message*>> tabwork; // group by table
105 |     DTime dt;
106 |     dt.start();
107 |     int lim=0;
108 |     int sumparms = 0;
109 |     for(; lim < 10000 && sumparms < 60000; ++lim) {
110 |       Message* msg;
111 |       if(needWait) {
112 |         cout<<"Waiting for data.."<<endl;
113 |         waitForData(d_pipe[0], 1);
114 |       }
115 |       int rc = read(d_pipe[0], &msg, sizeof(msg));
116 | 
117 |       if(rc == 0 && !tabwork.empty())
118 |         break;
119 |       if(rc == 0 || (rc < 0 && errno != EAGAIN)) {
120 |         DTime dt2;
121 |         dt2.start();
122 |         mp.exec("commit");
123 | //        cout<<"Commit took "<<dt2.lapUsec()/1000.0<<" msec"<<endl;
124 |         return;
125 |       }
126 |       if(rc < 0 && errno==EAGAIN) {
127 |         needWait = true;
128 |         break;
129 |       }
130 |       sumparms += msg->values.size();
131 |       tabwork[msg->table].push_back(msg);
132 |       needWait=false;
133 |     }
134 |     cout<<"Received "<<lim<<" messages "<<"from "<<tabwork.size()<<" tables with work, took "<<dt.lapUsec()/1000.0<<"msec, fields:";
135 |     dt.start();
136 | 
137 |     for(auto& [table, work] : tabwork) {
138 |       if(!schemas.count(table)) // new table
139 |         schemas[table] = mp.getSchema(table);
140 | 
141 |       unordered_set<string> fields;
142 |       
143 |       for(const auto& m : work) {
144 |         for(const auto& f : m->values) {
145 |           if(auto iter = fields.find(f.first); iter == fields.end()) {
146 |             fields.insert(f.first);
147 |             pair<string, string> cmp{f.first, std::string()};
148 |             if(!binary_search(schemas[table].begin(), schemas[table].end(), cmp,
149 |                        [](const auto& a, const auto& b)
150 |                        {
151 |                          return a.first < b.first;
152 |                        })) {
153 |               cout<<"shit, we miss "<<f.first<<endl;
154 |               if(std::get_if<double>(&f.second)) {
155 |                 mp.addColumn(table, f.first, "REAL");
156 |                 schemas[table].push_back({f.first, "REAL"});
157 |               }
158 |               else if(std::get_if<string>(&f.second)) {
159 |                 mp.addColumn(table, f.first, "TEXT");
160 |                 schemas[table].push_back({f.first, "TEXT"});
161 |               } else  {
162 |                 mp.addColumn(table, f.first, "BIGINT");
163 |                 schemas[table].push_back({f.first, "BIGINT"});
164 |               }
165 |               
166 |               sort(schemas[table].begin(), schemas[table].end());
167 | 
168 |             }
169 |           }
170 |         }
171 |       }
172 |       
173 |       string query="insert into "+table+" (";
174 |     
175 |       bool first=true;
176 |       for(const auto& f : fields) {
177 |         if(!first)
178 |           query+=',';
179 |         first=false;
180 |         query+=f;
181 |         cout<<" "<<f;
182 |       }
183 |       query += ") values ";
184 |       int ctr=1;
185 |       
186 |       for(unsigned int n=0; n < work.size(); ++n) {
187 |         if(n)
188 |           query +=',';
189 |         query += '(';
190 |         first=true;              
191 |         for(const auto& f: fields ) {
192 |           if(!first)
193 |             query +=',';
194 |           first=false;
195 |           query += '$';
196 |           query += to_string(ctr);
197 |           ctr++;
198 |         }
199 |         
200 |         query += ')';
201 |       }
202 |       cout<<"building query: "<<dt.lapUsec()/1000.0<<"ms\n";
203 |       
204 |     //    cout<<query<<endl;
205 |       
206 |       vector<std::optional<string>> allstrings;
207 |       allstrings.reserve(work.size()*fields.size());
208 |       
209 |       for(const auto& m : work) {
210 |         for(const auto& f: fields) {
211 |           if(auto iter = m->values.find(f); iter!= m->values.end()) {
212 |             std::visit([&allstrings](auto&&arg) {
213 |               using T = std::decay_t<decltype(arg)>;
214 |               if constexpr (std::is_same_v<T, string>)
215 |                              allstrings.push_back(arg);
216 |               else 
217 |                 allstrings.push_back(to_string(arg));
218 |             }, iter->second);
219 |           }
220 |           else {
221 |             allstrings.push_back(std::optional<string>());
222 |           }
223 |         }
224 |       }
225 |       cout<<endl;
226 | //      cout<<"params 1 took "<<dt.lapUsec()/1000.0<<" msec\n";
227 |       // types: integers, values are variable length, the lengths are integers,
228 |       //                                      array of pointers
229 |       
230 |       vector<const char*> allptrs;
231 |       allptrs.reserve(allstrings.size());
232 |       for(const auto& p : allstrings) {
233 |         if(p)
234 |           allptrs.push_back(p->c_str());
235 |         else
236 |           allptrs.push_back(0);
237 |       }
238 |       
239 | //      cout<<"params2 took "<<dt.lapUsec()/1000.0<<" msec\n";
240 |       mp.exec(query, allptrs);
241 | //      cout<<"exec took "<<dt.lapUsec()/1000.0<<" msec\n";
242 |       for(const auto& m : work)
243 |         delete m;
244 | //      cout<<"cleanup took "<<dt.lapUsec()/1000.0<<" msec\n";
245 |       
246 |       if(prevcommit != time(0)) {
247 |         cout<<"commit"<<endl;
248 |         mp.exec("commit");
249 |         mp.exec("begin");
250 |         prevcommit = time(0);
251 |       }
252 |     }
253 |   }
254 | }
255 |   
256 | 
257 | 


--------------------------------------------------------------------------------
/testrunner.cc:
--------------------------------------------------------------------------------
  1 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
  2 | #include "ext/doctest.h"
  3 | #include "sqlwriter.hh"
  4 | using namespace std;
  5 | 
  6 | TEST_CASE("basic test") {
  7 |   unlink("testrunner-example.sqlite3");
  8 |   {
  9 |     SQLiteWriter sqw("testrunner-example.sqlite3");
 10 |     sqw.addValue({{"pief",1}});
 11 |   }
 12 |   MiniSQLite ms("testrunner-example.sqlite3");
 13 |   auto res=ms.exec("select pief from data");
 14 |   CHECK(res.size() == 1);
 15 |   CHECK((res[0][0])=="1");
 16 |   unlink("testrunner-example.sqlite3");
 17 | }
 18 | 
 19 | 
 20 | TEST_CASE("other table test") {
 21 |   unlink("testrunner-example.sqlite3");
 22 |   {
 23 |     SQLiteWriter sqw("testrunner-example.sqlite3");
 24 |     sqw.addValue({{"piefpaf", 2}}, "metadata");
 25 |   }
 26 |   MiniSQLite ms("testrunner-example.sqlite3");
 27 |   auto res=ms.exec("select piefpaf from metadata");
 28 |   CHECK(res.size() == 1);
 29 |   CHECK((res[0][0])=="2");
 30 |   unlink("testrunner-example.sqlite3");
 31 | }
 32 | 
 33 | TEST_CASE("test add columns") {
 34 |   unlink("testrunner-example.sqlite3");
 35 |   {
 36 |     SQLiteWriter sqw("testrunner-example.sqlite3");
 37 |     sqw.addValue({{"piefpaf", 2}});
 38 |     sqw.addValue({
 39 |         {"piefpaf", 3},
 40 |         {"poef", 1.02},
 41 |         {"poetput", "test"},
 42 |       });
 43 |     sqw.addValue({{"piefpaf", 1}});
 44 |   }
 45 |   MiniSQLite ms("testrunner-example.sqlite3");
 46 |   auto res=ms.exec("select sum(piefpaf), sum(poef) from data");
 47 |   CHECK(res.size() == 1);
 48 |   CHECK((res[0][0])=="6");
 49 |   CHECK((res[0][1])=="1.02");
 50 |   unlink("testrunner-example.sqlite3");
 51 | }
 52 | 
 53 | TEST_CASE("test multiple tables") {
 54 |   unlink("testrunner-example.sqlite3");
 55 |   {
 56 |     SQLiteWriter sqw("testrunner-example.sqlite3");
 57 |     sqw.addValue({{"piefpaf", 2}});
 58 |     sqw.addValue({{"piefpaf", 3}}, "metadata");
 59 |     sqw.addValue({{"poef", 21}});
 60 |     sqw.addValue({{"poef", 32}}, "metadata");    
 61 |   }
 62 |   MiniSQLite ms("testrunner-example.sqlite3");
 63 |   auto res=ms.exec("select sum(piefpaf), sum(poef) from data");
 64 |   CHECK(res.size() == 1);
 65 |   CHECK((res[0][0])=="2");
 66 |   CHECK((res[0][1])=="21");
 67 | 
 68 |   res=ms.exec("select sum(piefpaf), sum(poef) from metadata");
 69 |   CHECK(res.size() == 1);
 70 |   CHECK((res[0][0])=="3");
 71 |   CHECK((res[0][1])=="32");
 72 | 
 73 |   
 74 |   unlink("testrunner-example.sqlite3");
 75 | }
 76 | 
 77 | TEST_CASE("test queries") {
 78 |   unlink("testrunner-example.sqlite3");
 79 |   {
 80 |     SQLiteWriter sqw("testrunner-example.sqlite3");
 81 |     sqw.addValue({{"piefpaf", 2}});
 82 |     sqw.addValue({{"piefpaf", 7}});
 83 |     sqw.addValue({{"piefpaf", 3}}, "metadata");
 84 |     sqw.addValue({{"poef", 21}});
 85 |     sqw.addValue({{"poef", 32}}, "metadata");
 86 |     sqw.addValue({{"poef", -1}}, "metadata");    
 87 |   }
 88 | 
 89 |   SQLiteWriter sqw("testrunner-example.sqlite3");
 90 |   auto res = sqw.query("select * from data where piefpaf = ?", {2});
 91 |   CHECK(res.size() == 1);
 92 |   CHECK(res[0]["piefpaf"]=="2");
 93 | 
 94 |   res = sqw.query("select sum(piefpaf) as s from data");
 95 |   CHECK(res.size() == 1);
 96 |   CHECK(res[0]["s"]=="9");
 97 | 
 98 |   res = sqw.query("select piefpaf from data where piefpaf NOT NULL order by piefpaf");
 99 |   CHECK(res.size() == 2);
100 |   CHECK(res[0]["piefpaf"]=="2");
101 |   CHECK(res[1]["piefpaf"]=="7");
102 | 
103 |   res = sqw.query("select sum(poef) as s from metadata where poef > ?", {-2});
104 |   CHECK(res.size() == 1);
105 |   CHECK(res[0]["s"]=="31");
106 | 
107 |   res = sqw.query("select sum(poef) as s from metadata where poef > ?", {-1});
108 |   CHECK(res.size() == 1);
109 |   CHECK(res[0]["s"]=="32");
110 |   
111 |   unlink("testrunner-example.sqlite3");
112 | }
113 | 
114 | TEST_CASE("test queries typed") {
115 |   unlink("testrunner-example.sqlite3");
116 |   {
117 |     SQLiteWriter sqw("testrunner-example.sqlite3");
118 |     sqw.addValue({{"piefpaf", 2}});
119 |     sqw.addValue({{"piefpaf", 7}});
120 |     sqw.addValue({{"poef", 21}});
121 |     sqw.addValue({{"user", "ahu"}, {"piefpaf", 42}});
122 |     sqw.addValue({{"temperature", 18.2}, {"piefpaf", 3}});
123 | 
124 |     sqw.addValue({{"poef", 8}}, "metadata");
125 |     sqw.addValue({{"poef", 23}}, "metadata");
126 |     sqw.addValue({{"poef", -1}}, "metadata");
127 |     sqw.addValue({{"poef", -2}}, "metadata");
128 |   }
129 | 
130 |   SQLiteWriter sqw("testrunner-example.sqlite3");
131 |   auto res = sqw.queryT("select * from data where piefpaf = ?", {3});
132 |   CHECK(res.size() == 1);
133 |   CHECK(get<double>(res[0]["temperature"])==18.2);
134 |   CHECK(get<int64_t>(res[0]["piefpaf"])==3);
135 | 
136 |   res = sqw.queryT("select sum(temperature) as s from data");
137 |   CHECK(get<double>(res[0]["s"])==18.2);
138 | 
139 |   res = sqw.queryT("select user from data where piefpaf = ?", {42});
140 |   CHECK(get<string>(res[0]["user"])=="ahu");
141 | 
142 |   res = sqw.queryT("select * from data order by piefpaf", {});
143 |   CHECK(get<nullptr_t>(res[0]["temperature"])==nullptr);
144 | 
145 |   res = sqw.queryT("select piefpaf from data where piefpaf NOT NULL order by piefpaf");
146 |   CHECK(res.size() == 4);
147 |   CHECK(get<int64_t>(res[0]["piefpaf"])==2);
148 |   CHECK(get<int64_t>(res[2]["piefpaf"])==7);
149 | 
150 |   res = sqw.queryT("select sum(poef) as s from metadata where poef > ?", {-2});
151 |   CHECK(res.size() == 1);
152 |   CHECK(get<int64_t>(res[0]["s"])==30);
153 | 
154 |   res = sqw.queryT("select sum(poef) as s from metadata where poef > ?", {-1});
155 |   CHECK(res.size() == 1);
156 |   CHECK(get<int64_t>(res[0]["s"])==31);
157 | 
158 |   unlink("testrunner-example.sqlite3");
159 | }
160 | 
161 | 
162 | TEST_CASE("test meta") {
163 |   unlink("testrunner-example.sqlite3");
164 |   {
165 |     SQLiteWriter sqw("testrunner-example.sqlite3", {{"piefpaf", "collate nocase"}});
166 |     sqw.addValue({{"piefpaf", "Guus"}, {"poef", "guus"}});
167 |   }
168 | 
169 |   SQLiteWriter sqw("testrunner-example.sqlite3");
170 |   auto res = sqw.query("select count(1) as c from data where piefpaf = ?", {"GUUS"});
171 |   CHECK(res.size() == 1);
172 |   CHECK(res[0]["c"]=="1");
173 | 
174 |   res = sqw.query("select count(1) as c from data where poef = ?", {"GUUS"});
175 |   CHECK(res.size() == 1);
176 |   CHECK(res[0]["c"]=="0");
177 |   
178 |   unlink("testrunner-example.sqlite3");
179 | }
180 | 
181 | TEST_CASE("test meta non-default table") {
182 |   unlink("testrunner-example.sqlite3");
183 |   {
184 |     SQLiteWriter sqw("testrunner-example.sqlite3", {{"second", {{"piefpaf", "collate nocase"}}}});
185 |     sqw.addValue({{"piefpaf", "Guus"}, {"poef", "guus"}}, "second");
186 |     sqw.addValue({{"town", "Nootdorp"}, {"city", "Pijnacker-Nootdorp"}});
187 |   }
188 | 
189 |   SQLiteWriter sqw("testrunner-example.sqlite3");
190 |   auto res = sqw.query("select count(1) as c from second where piefpaf = ?", {"GUUS"});
191 |   CHECK(res.size() == 1);
192 |   CHECK(res[0]["c"]=="1");
193 | 
194 |   res = sqw.query("select count(1) as c from second where poef = ?", {"GUUS"});
195 |   CHECK(res.size() == 1);
196 |   CHECK(res[0]["c"]=="0");
197 | 
198 |   auto res2 = sqw.queryT("select count(1) as c from data where city = ?", {"pijnacker-nootdorp"});
199 |   REQUIRE(res2.size() == 1);
200 |   CHECK(get<int64_t>(res2[0]["c"]) == 0);
201 | 
202 |   res2 = sqw.queryT("select count(1) as c from data where city = ?", {"Pijnacker-Nootdorp"});
203 |   REQUIRE(res2.size() == 1);
204 |   CHECK(get<int64_t>(res2[0]["c"]) == 1);
205 |   
206 |   unlink("testrunner-example.sqlite3");
207 | }
208 | 
209 | TEST_CASE("8 bit safe string") {
210 |   unlink("testrunner-example.sqlite3");
211 |   string bit8;
212 |   for(int n=255; n; --n) {
213 |     bit8.append(1, (char)n);
214 |   }
215 |   {
216 |     SQLiteWriter sqw("testrunner-example.sqlite3");
217 |     sqw.addValue({{"binarystring", bit8}});
218 |   }
219 |   SQLiteWriter sqw("testrunner-example.sqlite3");
220 |   auto res =sqw.queryT("select cast(binarystring as BLOB) as bstr from data");
221 |   auto t = get<vector<uint8_t>>(res[0]["bstr"]);
222 |   CHECK(string((char*)&t.at(0), t.size())==bit8);
223 | 
224 |   res =sqw.queryT("select binarystring as bstr from data");
225 |   CHECK(get<string>(res.at(0).at("bstr")) == bit8);
226 |   unlink("testrunner-example.sqlite3");
227 | }
228 | 
229 | TEST_CASE("blob test") {
230 |   unlink("testrunner-example.sqlite3");
231 |   vector<uint8_t> bit8;
232 |   for(int n=128; n; --n) {
233 |     bit8.push_back(n);
234 |   }
235 |   for(int n=255; n > 128; --n) {
236 |     bit8.push_back(n);
237 |   }
238 |   
239 |   {
240 |     SQLiteWriter sqw("testrunner-example.sqlite3");
241 |     sqw.addValue({{"binblob", bit8}});
242 |   }
243 |   SQLiteWriter sqw("testrunner-example.sqlite3");
244 |   auto res =sqw.queryT("select binblob from data");
245 |   CHECK(get<vector<uint8_t>>(res[0]["binblob"])==bit8);
246 |   unlink("testrunner-example.sqlite3");
247 | }
248 | 
249 | TEST_CASE("empty blob and string") {
250 |   unlink("testrunner-example.sqlite3");
251 |   vector<uint8_t> bit8;
252 |   string leer;
253 |   
254 |   {
255 |     SQLiteWriter sqw("testrunner-example.sqlite3");
256 |     sqw.addValue({{"vec", bit8}, {"str", leer}});
257 |   }
258 |   SQLiteWriter sqw("testrunner-example.sqlite3");
259 |   auto res =sqw.queryT("select * from data");
260 |   CHECK(get<string>(res[0]["str"])==leer);
261 |   CHECK(get<vector<uint8_t>>(res[0]["vec"])==bit8);
262 | 
263 |   unlink("testrunner-example.sqlite3");
264 | }
265 | 
266 | TEST_CASE("deal with constraint error") {
267 |   unlink("testrunner-example.sqlite3");
268 |   SQLiteWriter sqw("testrunner-example.sqlite3", {{"user", "UNIQUE"}});
269 |   sqw.addValue({{"user", "ahu"}, {"test", 1}});
270 |   
271 |   REQUIRE_THROWS_AS(sqw.addValue({{"user", "ahu"}, {"test", 2}}), std::exception);
272 |   // we previously had a bug where throwing such an exception would
273 |   // leave a prepared statement in a bad state, which would lead
274 |   // to subsequent bogus violations of constraints
275 |   sqw.addValue({{"user", "ahu2"}, {"test", 1}});
276 |   sqw.addValue({{"user", "ahu3"}, {"test", 1}});
277 |   unlink("testrunner-example.sqlite3");
278 | }
279 | 
280 | 
281 | TEST_CASE("test foreign keys") {
282 |   unlink("testrunner-example.sqlite3");
283 |   SQLiteWriter sqw("testrunner-example.sqlite3",
284 |                    {
285 |                      {"posts", {{"id", "PRIMARY KEY NOT NULL"}} },
286 |                      {"images", {{"postid", "NOT NULL REFERENCES posts(id) ON DELETE CASCADE"}}},
287 |                    });;
288 |   auto res = sqw.query("PRAGMA foreign_keys");
289 |   CHECK(res.size()==1);
290 |   CHECK(res[0]["foreign_keys"]=="1");
291 |   
292 |   
293 | 
294 |   sqw.addValue({{"id", "main"}, {"name", "main album"}}, "posts");
295 |   sqw.addValue({{"id", "first"}, {"postid", "main"}}, "images");
296 | 
297 |   REQUIRE_THROWS_AS(sqw.addValue({{"id", "second"}}, "images"), std::exception);
298 |   REQUIRE_THROWS_AS(sqw.addValue({{"id", "second"}, {"postid", "nosuchpost"}}, "images"), std::exception);
299 | 
300 |   sqw.query("delete from posts where id=?", {"main"});
301 |   auto res2 = sqw.queryT("select count(1) as c from images");
302 |   REQUIRE(res2.size() == 1);
303 |   CHECK(get<int64_t>(res2[0]["c"])==0);
304 |   
305 | }
306 | 
307 | TEST_CASE("insert or replace") {
308 |   unlink("insertorreplace.sqlite3");
309 |   SQLiteWriter sqw("insertorreplace.sqlite3", {
310 |       {
311 | 	"data", {{"id", "PRIMARY KEY"}}}
312 |     });
313 |   sqw.addValue({{"id", 1}, {"user", "ahu"}});
314 |   REQUIRE_THROWS_AS(  sqw.addValue({{"id", 1}, {"user", "jhu"}}), std::exception);
315 |   auto res = sqw.queryT("select user from data where id=?", {1});
316 |   CHECK(res.size() == 1);
317 |   
318 |   sqw.addOrReplaceValue({{"id", 1}, {"user", "harry"}});
319 | 
320 |   res = sqw.queryT("select user from data where id=?", {1});
321 |   REQUIRE(res.size() == 1);
322 |   CHECK(get<string>(res[0]["user"]) == "harry");
323 | 
324 |   REQUIRE_THROWS_AS(  sqw.addValue({{"id", 1}, {"user", "jhu"}}), std::exception);
325 |   unlink("insertorreplace.sqlite3");
326 | }
327 | 
328 | TEST_CASE("readonly test") {
329 |   string fname="readonly-test.sqlite3";
330 |   unlink(fname.c_str());
331 | 
332 |   {
333 |     SQLiteWriter sqw(fname);
334 |     sqw.addValue({{"some", "stuff"}, {"val", 1234.0}});
335 |   }
336 | 
337 |   {
338 |     SQLiteWriter ro(fname, SQLWFlag::ReadOnly);
339 |     auto res = ro.queryT("select * from data where some='stuff'");
340 |     REQUIRE(res.size() == 1);
341 |     
342 |     REQUIRE_THROWS_AS(ro.addValue({{"some", "more"}}), std::exception);
343 |     res = ro.queryT("select count(1) c from data");
344 |     REQUIRE(res.size() == 1);
345 |     CHECK(get<int64_t>(res[0]["c"]) == 1);
346 |     
347 |     REQUIRE_THROWS_AS(ro.query("insert into data (some, val) values('blah', 123)"), std::exception);
348 |   }
349 |   unlink(fname.c_str());
350 | }
351 | 
352 | TEST_CASE("test scale") {
353 |   unlink("testrunner-example.sqlite3");
354 |   {
355 |     SQLiteWriter sqw("testrunner-example.sqlite3");
356 |     for(int n=0; n < 100000; ++n) {
357 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}});
358 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}}, "metadata");      
359 |     }
360 |     sqw.addValue({{"poef", 21}});
361 |     sqw.addValue({{"poef", 32}}, "metadata");    
362 |   }
363 |   MiniSQLite ms("testrunner-example.sqlite3");
364 |   auto res=ms.exec("select count(1) from data");
365 |   CHECK(res.size() == 1);
366 |   CHECK((res[0][0])=="100001");
367 | 
368 |   res=ms.exec("select count(1) from metadata");
369 |   CHECK(res.size() == 1);
370 |   CHECK((res[0][0])=="100001");
371 | 
372 |   res=ms.exec("select count(poef) from metadata");
373 |   CHECK(res.size() == 1);
374 |   CHECK((res[0][0])=="1");
375 | 
376 |   res=ms.exec("select count(wah) from metadata");
377 |   CHECK(res.size() == 1);
378 |   CHECK((res[0][0])=="100000");
379 | 
380 |   unlink("testrunner-example.sqlite3");
381 | }
382 | 
383 | TEST_CASE("test scale NoTransactions") {
384 |   unlink("testrunner-example.sqlite3");
385 |   {
386 |     SQLiteWriter sqw("testrunner-example.sqlite3", SQLWFlag::NoTransactions);
387 |     //    sqw.query("pragma synchronous=off");
388 |     for(int n=0; n < 1000; ++n) {
389 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}});
390 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}}, "metadata");      
391 |     }
392 |     sqw.addValue({{"poef", 21}});
393 |     sqw.addValue({{"poef", 32}}, "metadata");    
394 |   }
395 |   MiniSQLite ms("testrunner-example.sqlite3");
396 |   auto res=ms.exec("select count(1) from data");
397 |   CHECK(res.size() == 1);
398 |   CHECK((res[0][0])=="1001");
399 | 
400 |   res=ms.exec("select count(1) from metadata");
401 |   CHECK(res.size() == 1);
402 |   CHECK((res[0][0])=="1001");
403 | 
404 |   res=ms.exec("select count(poef) from metadata");
405 |   CHECK(res.size() == 1);
406 |   CHECK((res[0][0])=="1");
407 | 
408 |   res=ms.exec("select count(wah) from metadata");
409 |   CHECK(res.size() == 1);
410 |   CHECK((res[0][0])=="1000");
411 | 
412 |   unlink("testrunner-example.sqlite3");
413 | }
414 | 
415 | 
416 | TEST_CASE("timeout test") {
417 |   unlink("testrunner-example.sqlite3");
418 |   {
419 |     SQLiteWriter sqw("testrunner-example.sqlite3");
420 |     for(int n=0; n < 1000; ++n) {
421 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}}, "one");
422 |       sqw.addValue({{"piefpaf", n}, {"wuh", 1.0*n}, {"wah", std::to_string(n)}}, "two");
423 |     }
424 |   }
425 |   SQLiteWriter sqw("testrunner-example.sqlite3", SQLWFlag::ReadOnly);
426 |   auto res = sqw.queryT("select * from one", {}, 1000);
427 |   CHECK(res.size() == 1000);
428 | 
429 |   REQUIRE_THROWS_AS(sqw.queryT("select * from one,two", {}, 1234), std::runtime_error);
430 | }
431 | 
432 | TEST_CASE("memory database") {
433 |   unlink("testrunner-example.sqlite3");
434 |   SQLiteWriter sqw("testrunner-example.sqlite3");
435 | 
436 |   sqw.addValue({{"id", 234}}, "testtabel");
437 |   auto rows = sqw.queryT("select * from testtabel");
438 | 
439 |   CHECK(rows.size() == 1);
440 |   auto out = get<int64_t>(rows[0]["id"]); 
441 |   CHECK(out == 234);
442 |   
443 |   sqw.queryT("ATTACH DATABASE ':memory:' AS aux1");
444 | 
445 |   sqw.addValue({{"id", 1}}, "aux1.testtabel");
446 |   rows = sqw.queryT("select * from aux1.testtabel");
447 | 
448 |   CHECK(rows.size() == 1);
449 |   out = get<int64_t>(rows[0]["id"]); 
450 |   CHECK(out == 1);
451 | }
452 | 


--------------------------------------------------------------------------------
/sqlwriter.cc:
--------------------------------------------------------------------------------
  1 | #include "sqlwriter.hh"
  2 | #include <algorithm>
  3 | #include <unistd.h>
  4 | #include "sqlite3.h"
  5 | using namespace std;
  6 | 
  7 | std::atomic<uint64_t> MiniSQLite::s_execs, MiniSQLite::s_sorts, MiniSQLite::s_fullscans, MiniSQLite::s_autoindexes;
  8 | 
  9 | static string quoteTableName(const std::string& table)
 10 | {
 11 |   auto pos = table.find('.');
 12 |   if(pos == string::npos) 
 13 |     return "\""+ table +"\"";
 14 |   else
 15 |     return "\"" + table.substr(0, pos) + "\".\"" + table.substr(pos + 1)+"\"";
 16 | }
 17 | 
 18 | MiniSQLite::MiniSQLite(std::string_view fname, SQLWFlag flag)
 19 | {
 20 |   int flags;
 21 |   if(flag == SQLWFlag::ReadOnly)
 22 |     flags = SQLITE_OPEN_READONLY;
 23 |   else
 24 |     flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
 25 |   
 26 |   if ( sqlite3_open_v2(&fname[0], &d_sqlite, flags, 0)!=SQLITE_OK) {
 27 |     throw runtime_error("Unable to open "+(string)fname+" for sqlite");
 28 |   }
 29 |   sqlite3_extended_result_codes(d_sqlite, 1);
 30 |   exec("PRAGMA journal_mode='wal'");
 31 |   exec("PRAGMA foreign_keys=ON");
 32 |   // keeps us honest about "table" and 'string literals'
 33 |   sqlite3_db_config(d_sqlite, SQLITE_DBCONFIG_DQS_DDL, 0, (void*)0);
 34 |   sqlite3_db_config(d_sqlite, SQLITE_DBCONFIG_DQS_DML, 0, (void*)0);
 35 | 
 36 |   
 37 |   sqlite3_busy_timeout(d_sqlite, 60000);
 38 | }
 39 | 
 40 | //! Get field names and types from a table
 41 | vector<pair<string,string> > MiniSQLite::getSchema(const std::string& table)
 42 | {
 43 |   vector<pair<string,string>> ret;
 44 | 
 45 |   string schema = "main";
 46 |   string tablename;
 47 |   auto pos = table.find('.');
 48 |   if(pos == string::npos) {
 49 |     tablename = table;
 50 |   }
 51 |   else {
 52 |     schema = table.substr(0, pos); 
 53 |     tablename = table.substr(pos +1);
 54 |   }
 55 | 
 56 |   auto rows = exec("SELECT cid,name,type FROM pragma_table_xinfo('"+tablename+"', '"+schema+"')");
 57 | 
 58 |   for(const auto& r : rows) {
 59 |     ret.push_back({r[1], r[2]});
 60 |   }
 61 |   sort(ret.begin(), ret.end(), [](const auto& a, const auto& b) {
 62 |     return a.first < b.first;
 63 |   });
 64 | 
 65 |   //  cout<<"returning "<<ret.size()<<" rows for table "<<tablename<<" in schema "<<schema<<"\n";
 66 |   return ret;
 67 | }
 68 | 
 69 | int MiniSQLite::helperFunc(void* ptr, int cols, char** colvals, char** colnames [[maybe_unused]])
 70 | {
 71 |   vector<string> row;
 72 |   row.reserve(cols);
 73 |   for(int n=0; n < cols ; ++n)
 74 |     row.push_back(colvals[n]);
 75 |   ((MiniSQLite*)ptr)->d_rows.push_back(row);
 76 |   return 0;
 77 | }
 78 | 
 79 | vector<vector<string>> MiniSQLite::exec(std::string_view str)
 80 | {
 81 |   char *errmsg;
 82 |   std::string errstr;
 83 |   //  int (*callback)(void*,int,char**,char**)
 84 |   d_rows.clear();
 85 |   int rc = sqlite3_exec(d_sqlite, &str[0], helperFunc, this, &errmsg);
 86 |   if (rc != SQLITE_OK) {
 87 |     errstr = errmsg;
 88 |     sqlite3_free(errmsg);
 89 |     throw std::runtime_error("Error executing sqlite3 query '"+(string)str+"': "+errstr);
 90 |   }
 91 |   return d_rows; 
 92 | }
 93 | 
 94 | static void checkBind(int rc)
 95 | {
 96 |   if(rc) {
 97 |     throw std::runtime_error("Error binding value to prepared statement: " + string(sqlite3_errstr(rc)));
 98 |   }
 99 | }
100 | 
101 | void MiniSQLite::bindPrep(const std::string& table, int idx, bool value) {   checkBind(sqlite3_bind_int(d_stmts[table], idx, value ? 1 : 0));   }
102 | void MiniSQLite::bindPrep(const std::string& table, int idx, int value) {   checkBind(sqlite3_bind_int(d_stmts[table], idx, value));   }
103 | void MiniSQLite::bindPrep(const std::string& table, int idx, uint32_t value) {   checkBind(sqlite3_bind_int64(d_stmts[table], idx, value));   }
104 | void MiniSQLite::bindPrep(const std::string& table, int idx, long value) {   checkBind(sqlite3_bind_int64(d_stmts[table], idx, value));   }
105 | void MiniSQLite::bindPrep(const std::string& table, int idx, unsigned long value) {   checkBind(sqlite3_bind_int64(d_stmts[table], idx, value));   }
106 | void MiniSQLite::bindPrep(const std::string& table, int idx, long long value) {   checkBind(sqlite3_bind_int64(d_stmts[table], idx, value));   }
107 | void MiniSQLite::bindPrep(const std::string& table, int idx, unsigned long long value) {   checkBind(sqlite3_bind_int64(d_stmts[table], idx, value));   }
108 | void MiniSQLite::bindPrep(const std::string& table, int idx, double value) {   checkBind(sqlite3_bind_double(d_stmts[table], idx, value));   }
109 | void MiniSQLite::bindPrep(const std::string& table, int idx, const std::string& value) {   checkBind(sqlite3_bind_text(d_stmts[table], idx, value.c_str(), value.size(), SQLITE_TRANSIENT));   }
110 | void MiniSQLite::bindPrep(const std::string& table, int idx, const std::vector<uint8_t>& value) {
111 |   if(value.empty())
112 |     checkBind(sqlite3_bind_zeroblob(d_stmts[table], idx, 0));
113 |   else
114 |     checkBind(sqlite3_bind_blob(d_stmts[table], idx, &value.at(0), value.size(), SQLITE_TRANSIENT));
115 | }
116 | 
117 | 
118 | void MiniSQLite::prepare(const std::string& table, string_view str)
119 | {
120 |   if(d_stmts[table]) {
121 |     sqlite3_finalize(d_stmts[table]);
122 |     d_stmts[table] = 0;
123 |   }
124 |   const char* pTail;
125 |   
126 |   if (sqlite3_prepare_v2(d_sqlite, &str[0], -1, &d_stmts[table], &pTail ) != SQLITE_OK) {
127 |     throw runtime_error("Unable to prepare query "+(string)str + ": "+sqlite3_errmsg(d_sqlite));
128 |   }
129 | }
130 | 
131 | struct DeadlineCatcher
132 | {
133 |   DeadlineCatcher(sqlite3* db, unsigned int msec) : d_sqlite(db), d_msec(msec)
134 |   {
135 |     if(!msec)
136 |       return;
137 |     clock_gettime(CLOCK_MONOTONIC, &d_ttd);
138 |     auto r = div(msec, 1000); // seconds
139 |     d_ttd.tv_sec += r.quot;
140 |     d_ttd.tv_nsec += 1000000 * r.rem;
141 |     
142 |     if(d_ttd.tv_nsec > 1000000000) {
143 |       d_ttd.tv_sec++;
144 |       d_ttd.tv_nsec -= 1000000000;
145 |     }
146 |     sqlite3_progress_handler(d_sqlite, 100, [](void *ptr) -> int {
147 |       DeadlineCatcher* us = (DeadlineCatcher*) ptr;
148 |       us->called++;
149 |       struct timespec now;
150 |       clock_gettime(CLOCK_MONOTONIC, &now);
151 |       return std::tie(us->d_ttd.tv_sec, us->d_ttd.tv_nsec) <
152 | 	std::tie(now.tv_sec, now.tv_nsec);
153 |     }, this);
154 |   }
155 | 
156 |   DeadlineCatcher(const DeadlineCatcher& rhs) = delete;
157 |   
158 |   ~DeadlineCatcher()
159 |   {
160 |     if(d_msec) {
161 |       sqlite3_progress_handler(d_sqlite, 0, 0, 0); // remove
162 |       //      cout<<"Was called "<<called<<" times\n";
163 |     }
164 |   }
165 |   sqlite3* d_sqlite;
166 |   const unsigned int d_msec;
167 |   struct timespec d_ttd;
168 |   unsigned int called = 0;
169 | };
170 | 
171 | void MiniSQLite::execPrep(const std::string& table, std::vector<std::unordered_map<std::string, outvar_t>>* rows, unsigned int msec)
172 | {
173 |   int rc;
174 |   if(rows)
175 |     rows->clear();
176 | 
177 |   DeadlineCatcher dc(d_sqlite, msec); // noop if msec = 0
178 |   
179 |   std::unordered_map<string, outvar_t> row;
180 |   for(;;) {
181 |     rc = sqlite3_step(d_stmts[table]); 
182 |     if(rc == SQLITE_DONE)
183 |       break;
184 |     else if(rows && rc == SQLITE_ROW) {
185 |       row.clear();
186 |       for(int n = 0 ; n < sqlite3_column_count(d_stmts[table]);++n) {
187 |         int type = sqlite3_column_type(d_stmts[table], n);
188 |         
189 |         if(type == SQLITE_TEXT) {
190 |           const char* p = (const char*)sqlite3_column_text(d_stmts[table], n);
191 |           if(!p) {
192 |             row[sqlite3_column_name(d_stmts[table], n)] = string();
193 |           }
194 |           else
195 |             row[sqlite3_column_name(d_stmts[table], n)] = string(p, sqlite3_column_bytes(d_stmts[table], n));
196 |         }
197 |         else if(type == SQLITE_BLOB) {
198 |           const uint8_t* p = (const uint8_t*)sqlite3_column_blob(d_stmts[table], n);
199 |           if(!p) {
200 |             row[sqlite3_column_name(d_stmts[table], n)]= vector<uint8_t>();
201 |           }
202 |           else
203 |             row[sqlite3_column_name(d_stmts[table], n)]=vector<uint8_t>(p, p+sqlite3_column_bytes(d_stmts[table], n));
204 |         }
205 |         else if(type == SQLITE_FLOAT) {
206 |           row[sqlite3_column_name(d_stmts[table], n)]= sqlite3_column_double(d_stmts[table], n);
207 |         }
208 |         else if(type == SQLITE_INTEGER) {
209 |           row[sqlite3_column_name(d_stmts[table], n)]= sqlite3_column_int64(d_stmts[table], n);
210 |         }
211 |         else if(type == SQLITE_NULL) {
212 |           row[sqlite3_column_name(d_stmts[table], n)]= nullptr;
213 |         }
214 |         
215 |       }
216 |       rows->push_back(row);
217 |     }
218 |     else {
219 |       sqlite3_reset(d_stmts[table]);
220 |       sqlite3_clear_bindings(d_stmts[table]);
221 |       throw runtime_error("Sqlite error "+std::to_string(rc)+": "+sqlite3_errstr(rc));
222 |     }
223 |   }
224 |   s_fullscans += sqlite3_stmt_status(d_stmts[table], SQLITE_STMTSTATUS_FULLSCAN_STEP, true);
225 |   s_sorts += sqlite3_stmt_status(d_stmts[table], SQLITE_STMTSTATUS_SORT, true);
226 |   s_autoindexes += sqlite3_stmt_status(d_stmts[table], SQLITE_STMTSTATUS_AUTOINDEX, true);
227 |   s_execs++;
228 |   
229 |   rc= sqlite3_reset(d_stmts[table]);
230 |   if(rc != SQLITE_OK)
231 |     throw runtime_error("Sqlite error "+std::to_string(rc)+": "+sqlite3_errstr(rc));
232 |   sqlite3_clear_bindings(d_stmts[table]);
233 | }
234 | 
235 | void MiniSQLite::begin()
236 | {
237 |   d_intransaction=true;
238 |   exec("begin");
239 | }
240 | void MiniSQLite::commit()
241 | {
242 |   d_intransaction=false;
243 |   exec("commit");
244 | }
245 | 
246 | void MiniSQLite::cycle()
247 | {
248 |   exec("commit;begin");
249 | }
250 | 
251 | bool MiniSQLite::haveTable(const string& table)
252 | {
253 |   return !getSchema(table).empty();
254 | }
255 | 
256 | 
257 | //! Add a column to a table with a certain type
258 | void MiniSQLite::addColumn(const string& table, string_view name, string_view type, const std::string& meta)
259 | {
260 |   // SECURITY PROBLEM - somehow we can't do prepared statements here
261 |   
262 |   if(!haveTable(table)) {
263 | #if SQLITE_VERSION_NUMBER >= 3037001
264 |     exec("create table if not exists "+quoteTableName(table)+" ( '"+(string)name+"' "+(string)type+" "+meta+") STRICT");
265 | #else
266 |     exec("create table if not exists "+quoteTableName(table)+" ( '"+(string)name+"' "+(string)type+" "+ meta+")");
267 | #endif
268 |   } else {
269 |     exec("ALTER table "+quoteTableName(table)+" add column \""+string(name)+ "\" "+string(type)+ " "+meta);
270 |   }
271 | }
272 | 
273 | 
274 | 
275 | void SQLiteWriter::commitThread()
276 | {
277 |   int n=0;
278 |   while(!d_pleasequit) {
279 |     usleep(50000);
280 |     if(!(n%20)) {
281 |       std::lock_guard<std::mutex> lock(d_mutex);
282 |       d_db.cycle();
283 |     }
284 |     n++;
285 |   }
286 |   //  cerr<<"Thread exiting"<<endl;
287 | }
288 | 
289 | bool SQLiteWriter::haveColumn(const std::string& table, std::string_view name)
290 | {
291 |   if(d_columns[table].empty()) {
292 |     d_columns[table] = d_db.getSchema(table);
293 |   }
294 |   //  cout<<"Do we have column "<<name<<" in table "<<table<<endl;
295 |   // this could be more efficient somehow
296 |   pair<string, string> cmp{name, std::string()};
297 |   return binary_search(d_columns[table].begin(), d_columns[table].end(), cmp,
298 |                        [](const auto& a, const auto& b)
299 |                        {
300 |                          return a.first < b.first;
301 |                        });
302 | 
303 | }
304 | 
305 | 
306 | void SQLiteWriter::addValue(const initializer_list<std::pair<const char*, var_t>>& values, const std::string& table)
307 | {
308 |   addValueGeneric(table, values);
309 | }
310 | 
311 | void SQLiteWriter::addValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table)
312 | {
313 |   addValueGeneric(table, values);
314 | }
315 | 
316 | void SQLiteWriter::addOrReplaceValue(const initializer_list<std::pair<const char*, var_t>>& values, const std::string& table)
317 | {
318 |   addValueGeneric(table, values, true);
319 | }
320 | 
321 | void SQLiteWriter::addOrReplaceValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table)
322 | {
323 |   addValueGeneric(table, values, true);
324 | }
325 | 
326 | 
327 | 
328 | template<typename T>
329 | void SQLiteWriter::addValueGeneric(const std::string& table, const T& values, bool replace)
330 | {
331 |   if(d_flag == SQLWFlag::ReadOnly)
332 |     throw std::runtime_error("Attempting to write to a read-only database instance");
333 |   
334 |   std::lock_guard<std::mutex> lock(d_mutex);
335 |   if(!d_db.isPrepared(table) || d_lastreplace[table] != replace || !equal(values.begin(), values.end(),
336 |                                        d_lastsig[table].cbegin(), d_lastsig[table].cend(),
337 |                             [](const auto& a, const auto& b)
338 |   {
339 |     return a.first == b;
340 |   })) {
341 |     //    cout<<"Starting a new prepared statement"<<endl;
342 |     string q = string("insert ") + (replace ? "or replace " : "") + "into "+ quoteTableName(table)+" (";
343 |     string qmarks;
344 |     bool first=true;
345 |     for(const auto& p : values) {
346 |       if(!haveColumn(table, p.first)) {
347 |         if(std::get_if<double>(&p.second)) {
348 |           d_db.addColumn(table, p.first, "REAL", d_meta[table][p.first]);
349 |           d_columns[table].push_back({p.first, "REAL"});
350 |         }
351 |         else if(std::get_if<string>(&p.second)) {
352 |           d_db.addColumn(table, p.first, "TEXT", d_meta[table][p.first]);
353 |           d_columns[table].push_back({p.first, "TEXT"});
354 |         }
355 |         else if(std::get_if<vector<uint8_t>>(&p.second)) {
356 |           d_db.addColumn(table, p.first, "BLOB", d_meta[table][p.first]);
357 |           d_columns[table].push_back({p.first, "BLOB"});
358 |         }
359 |         else  {
360 |           d_db.addColumn(table, p.first, "INT", d_meta[table][p.first]);
361 |           d_columns[table].push_back({p.first, "INT"});
362 |         }
363 | 
364 |         sort(d_columns[table].begin(), d_columns[table].end());
365 |       }
366 |       if(!first) {
367 |         q+=", ";
368 |         qmarks += ", ";
369 |       }
370 |       first=false;
371 |       q+="'"+string(p.first)+"'";
372 |       qmarks +="?";
373 |     }
374 |     q+= ") values ("+qmarks+")";
375 | 
376 |     d_db.prepare(table, q);
377 |     
378 |     d_lastsig[table].clear();
379 |     for(const auto& p : values)
380 |       d_lastsig[table].push_back(p.first);
381 |     d_lastreplace[table]=replace;
382 |   }
383 |   
384 |   int n = 1;
385 |   for(const auto& p : values) {
386 |     std::visit([this, &n, &table](auto&& arg) {
387 |       d_db.bindPrep(table, n, arg);
388 |     }, p.second);
389 |     n++;
390 |   }
391 |   d_db.execPrep(table);
392 | }
393 | 
394 | std::vector<std::unordered_map<std::string, std::string>> SQLiteWriter::query(const std::string& q, const initializer_list<var_t>& values)
395 | {
396 |   auto res = queryGen(q, values);
397 |   std::vector<std::unordered_map<std::string, std::string>> ret;
398 |   for(const auto& rowin : res) {
399 |     std::unordered_map<std::string, std::string> rowout;
400 |     for(const auto& f : rowin) {
401 |       string str;
402 |       std::visit([&str](auto&& arg) {
403 |         using T = std::decay_t<decltype(arg)>;
404 |         if constexpr (std::is_same_v<T, string>)
405 |                        str=arg;
406 |         else if constexpr (std::is_same_v<T, nullptr_t>)
407 |                        str="";
408 |         else if constexpr (std::is_same_v<T, vector<uint8_t>>)
409 |           str = "<blob>";
410 |         else str = to_string(arg);
411 |       }, f.second);
412 |       rowout[f.first] = str;
413 |     }
414 |     ret.push_back(rowout);
415 |   }
416 |   return ret;
417 | }
418 | 
419 | std::vector<std::unordered_map<std::string, MiniSQLite::outvar_t>> SQLiteWriter::queryT(const std::string& q, const initializer_list<var_t>& values, unsigned int msec)
420 | {
421 |   return queryGen(q, values, msec);
422 | }
423 | 
424 | template<typename T>
425 | vector<std::unordered_map<string, MiniSQLite::outvar_t>> SQLiteWriter::queryGen(const std::string& q, const T& values, unsigned int msec)
426 | {
427 |   if(msec && d_flag != SQLWFlag::ReadOnly)
428 |     throw std::runtime_error("Timeout only possible for read-only connections");
429 | 
430 |   std::lock_guard<std::mutex> lock(d_mutex);
431 |   d_db.prepare("", q); // we use an empty table name so as not to collide with other things
432 |   int n = 1;
433 |   for(const auto& p : values) {
434 |     std::visit([this, &n](auto&& arg) {
435 |       d_db.bindPrep("", n, arg);
436 |     }, p);
437 |     n++;
438 |   }
439 |   vector<unordered_map<string, MiniSQLite::outvar_t>> ret;
440 |   d_db.execPrep("", &ret, msec);
441 | 
442 |   return ret;
443 | }
444 | 
445 | 
446 | MiniSQLite::~MiniSQLite()
447 | {
448 |   // needs to close down d_sqlite3
449 |   if(d_intransaction)
450 |     commit();
451 | 
452 |   for(auto& stmt: d_stmts)
453 |     if(stmt.second)
454 |       sqlite3_finalize(stmt.second);
455 |   
456 |   sqlite3_close(d_sqlite); // same
457 | }
458 | 


--------------------------------------------------------------------------------