├── .clang-format
├── .gitignore
├── Common.h
├── Config.mk
├── ImageSearch.cc
├── LICENSE
├── Makefile
├── Messages.bond
├── Messages_reflection.h
├── Messages_types.cpp
├── Messages_types.h
├── README.md
├── RecursiveSearch.cc
├── Search.cc
├── Service.cc
├── Service.h
├── VideoSearch.cc
└── WebSearch.cc


/.clang-format:
--------------------------------------------------------------------------------
 1 | BasedOnStyle:  Mozilla
 2 | AlwaysBreakTemplateDeclarations: true
 3 | BreakBeforeTernaryOperators: false
 4 | ColumnLimit:     120
 5 | MaxEmptyLinesToKeep: 2
 6 | NamespaceIndentation: Inner
 7 | Cpp11BracedListStyle: true
 8 | Standard:        Cpp11
 9 | BreakBeforeBraces: Attach
10 | DerivePointerBinding: false
11 | PointerBindsToType: true
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | Search
 2 | WebSearch
 3 | ImageSearch
 4 | VideoSearch
 5 | RecursiveSearch
 6 | 
 7 | *.o
 8 | .*.swp
 9 | 
10 | env
11 | 
12 | perf.data
13 | perf.data.old
14 | oprofile_data
15 | cachegrind.out*
16 | callgrind.out*
17 | massif.out*
18 | 


--------------------------------------------------------------------------------
/Common.h:
--------------------------------------------------------------------------------
 1 | #ifndef SEARCH_COMMON_H
 2 | #define SEARCH_COMMON_H
 3 | 
 4 | #include <ios>
 5 | #include <locale>
 6 | #include <vector>
 7 | #include <string>
 8 | #include <set>
 9 | #include <functional>
10 | 
11 | namespace Common {
12 | 
13 | using Request = std::string;
14 | using Response = std::set<std::string>;
15 | using RequestHandler = std::function<Response(Request)>;
16 | 
17 | inline void init() noexcept {
18 |   std::locale::global(std::locale(""));
19 |   std::ios_base::sync_with_stdio(false);
20 | }
21 | 
22 | inline std::vector<std::string> args(int argc, char** argv) {
23 |   return {argv, argv + argc};
24 | }
25 | 
26 | }
27 | 
28 | #endif
29 | 


--------------------------------------------------------------------------------
/Config.mk:
--------------------------------------------------------------------------------
1 | CXX = g++-4.9
2 | CC = gcc-4.9
3 | CXXFLAGS += -pthread -fdiagnostics-color=auto -fmax-errors=1 -std=c++14 -O2 -g -Wall -Wextra -pedantic -Wuninitialized -Wstrict-overflow=3 -Wshadow -fno-omit-frame-pointer -fno-inline
4 | CXXFLAGS += -DBOND_COMPACT_BINARY_PROTOCOL
5 | LDLIBS += -lstdc++ -lpthread -lnanomsg -lanl -ltbb -L/usr/local/lib/bond -Wl,-Bstatic -lbond -Wl,-Bdynamic
6 | LDFLAGS += -Wl,-O1 -Wl,--hash-style=gnu -Wl,--sort-common -Wl,--demangle -Wl,--build-id
7 | 


--------------------------------------------------------------------------------
/ImageSearch.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | 
 3 | #include "Common.h"
 4 | #include "Service.h"
 5 | 
 6 | int main(int argc, char** argv) try {
 7 |   Common::init();
 8 | 
 9 |   const auto args = Common::args(argc, argv);
10 |   const auto endpoint = args.size() == 2 ? args[1] : "tcp://localhost:9995";
11 | 
12 |   Service::Responder self{endpoint};
13 | 
14 |   // dummy request handler
15 |   const auto requestHandler = [&self](const Common::Request& req) -> Common::Response {
16 |     (void)req; // unused
17 |     return {"First Image Result", "Second Image Result"};
18 |   };
19 | 
20 |   self.onRequest(requestHandler);
21 | 
22 | } catch (const std::exception& e) {
23 |   std::cerr << e.what() << std::endl;
24 | }
25 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2015 Daniel J. Hofmann
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | include Config.mk
 2 | 
 3 | all: Search WebSearch ImageSearch VideoSearch RecursiveSearch
 4 | 
 5 | bond: Messages.bond
 6 | 	gbc c++ Messages.bond
 7 | 
 8 | Search: Search.o Service.o Messages_types.o
 9 | WebSearch: WebSearch.o Service.o Messages_types.o
10 | ImageSearch: ImageSearch.o Service.o Messages_types.o
11 | VideoSearch: VideoSearch.o Service.o Messages_types.o
12 | RecursiveSearch: RecursiveSearch.o Service.o Messages_types.o
13 | 
14 | clean:
15 | 	$(RM) *.o Search WebSearch ImageSearch VideoSearch RecursiveSearch
16 | 
17 | .PHONY: all bond clean
18 | 


--------------------------------------------------------------------------------
/Messages.bond:
--------------------------------------------------------------------------------
 1 | namespace Message
 2 | 
 3 | struct Request {
 4 |   // Search request for a specific keyword
 5 | 
 6 |   0: string keyword;
 7 | }
 8 | 
 9 | struct Response {
10 |   // Search response with matches -- may be empty
11 | 
12 |   0: set<string> matches;
13 | }
14 | 


--------------------------------------------------------------------------------
/Messages_reflection.h:
--------------------------------------------------------------------------------
  1 | 
  2 | //------------------------------------------------------------------------------
  3 | // This code was generated by a tool.
  4 | //
  5 | //   Tool : Bond Compiler 3.02
  6 | //   File : Messages_reflection.h
  7 | //
  8 | // Changes to this file may cause incorrect behavior and will be lost when
  9 | // the code is regenerated.
 10 | // <auto-generated />
 11 | //------------------------------------------------------------------------------
 12 | 
 13 | #pragma once
 14 | 
 15 | #include "Messages_types.h"
 16 | #include <bond/core/reflection.h>
 17 | 
 18 | namespace Message
 19 | {
 20 |     //
 21 |     // Request
 22 |     //
 23 |     struct Request::Schema
 24 |     {
 25 |         typedef bond::no_base base;
 26 | 
 27 |         static const bond::Metadata metadata;
 28 |         
 29 |         private: static const bond::Metadata s_keyword_metadata;
 30 | 
 31 |         public: struct var
 32 |         {
 33 |             // keyword
 34 |             typedef bond::reflection::FieldTemplate<
 35 |                 0,
 36 |                 bond::reflection::optional_field_modifier,
 37 |                 Request,
 38 |                 std::string,
 39 |                 &Request::keyword,
 40 |                 &s_keyword_metadata
 41 |             > keyword;
 42 |         };
 43 | 
 44 |         private: typedef boost::mpl::list<> fields0;
 45 |         private: typedef boost::mpl::push_front<fields0, var::keyword>::type fields1;
 46 | 
 47 |         public: typedef fields1::type fields;
 48 |         
 49 |         
 50 |         static bond::Metadata GetMetadata()
 51 |         {
 52 |             return bond::reflection::MetadataInit("Request", "Message.Request",
 53 |                 bond::reflection::Attributes()
 54 |             );
 55 |         }
 56 |     };
 57 |     
 58 | 
 59 |     //
 60 |     // Response
 61 |     //
 62 |     struct Response::Schema
 63 |     {
 64 |         typedef bond::no_base base;
 65 | 
 66 |         static const bond::Metadata metadata;
 67 |         
 68 |         private: static const bond::Metadata s_matches_metadata;
 69 | 
 70 |         public: struct var
 71 |         {
 72 |             // matches
 73 |             typedef bond::reflection::FieldTemplate<
 74 |                 0,
 75 |                 bond::reflection::optional_field_modifier,
 76 |                 Response,
 77 |                 std::set<std::string>,
 78 |                 &Response::matches,
 79 |                 &s_matches_metadata
 80 |             > matches;
 81 |         };
 82 | 
 83 |         private: typedef boost::mpl::list<> fields0;
 84 |         private: typedef boost::mpl::push_front<fields0, var::matches>::type fields1;
 85 | 
 86 |         public: typedef fields1::type fields;
 87 |         
 88 |         
 89 |         static bond::Metadata GetMetadata()
 90 |         {
 91 |             return bond::reflection::MetadataInit("Response", "Message.Response",
 92 |                 bond::reflection::Attributes()
 93 |             );
 94 |         }
 95 |     };
 96 |     
 97 | 
 98 |     
 99 | } // namespace Message
100 | 


--------------------------------------------------------------------------------
/Messages_types.cpp:
--------------------------------------------------------------------------------
 1 | 
 2 | //------------------------------------------------------------------------------
 3 | // This code was generated by a tool.
 4 | //
 5 | //   Tool : Bond Compiler 3.02
 6 | //   File : Messages_types.cpp
 7 | //
 8 | // Changes to this file may cause incorrect behavior and will be lost when
 9 | // the code is regenerated.
10 | // <auto-generated />
11 | //------------------------------------------------------------------------------
12 | 
13 | #include "Messages_reflection.h"
14 | #include <bond/core/exception.h>
15 | 
16 | namespace Message
17 | {
18 |     
19 |     const bond::Metadata Request::Schema::metadata
20 |         = Request::Schema::GetMetadata();
21 |     
22 |     const bond::Metadata Request::Schema::s_keyword_metadata
23 |         = bond::reflection::MetadataInit("keyword");
24 | 
25 |     
26 |     const bond::Metadata Response::Schema::metadata
27 |         = Response::Schema::GetMetadata();
28 |     
29 |     const bond::Metadata Response::Schema::s_matches_metadata
30 |         = bond::reflection::MetadataInit("matches");
31 | 
32 |     
33 | } // namespace Message
34 | 


--------------------------------------------------------------------------------
/Messages_types.h:
--------------------------------------------------------------------------------
  1 | 
  2 | //------------------------------------------------------------------------------
  3 | // This code was generated by a tool.
  4 | //
  5 | //   Tool : Bond Compiler 3.02
  6 | //   File : Messages_types.h
  7 | //
  8 | // Changes to this file may cause incorrect behavior and will be lost when
  9 | // the code is regenerated.
 10 | // <auto-generated />
 11 | //------------------------------------------------------------------------------
 12 | 
 13 | #pragma once
 14 | 
 15 | #include <bond/core/bond_version.h>
 16 | 
 17 | #if BOND_VERSION < 0x302
 18 | #error This file was generated by a newer version of Bond compiler
 19 | #error and is incompatible with your version Bond library.
 20 | #endif
 21 | 
 22 | #if BOND_MIN_CODEGEN_VERSION > 0x302
 23 | #error This file was generated by an older version of Bond compiler
 24 | #error and is incompatible with your version Bond library.
 25 | #endif
 26 | 
 27 | #include <bond/core/config.h>
 28 | #include <bond/core/containers.h>
 29 | 
 30 | 
 31 | 
 32 | namespace Message
 33 | {
 34 |     
 35 |     struct Request
 36 |     {
 37 |         std::string keyword;
 38 |         
 39 |         Request()
 40 |         {
 41 |         }
 42 | 
 43 |         
 44 | #ifndef BOND_NO_CXX11_DEFAULTED_FUNCTIONS
 45 |         // Compiler generated copy ctor OK
 46 |         Request(const Request& other) = default;
 47 | #endif
 48 |         
 49 | #ifndef BOND_NO_CXX11_RVALUE_REFERENCES
 50 |         Request(Request&& other)
 51 |           : keyword(std::move(other.keyword))
 52 |         {
 53 |         }
 54 | #endif
 55 |         
 56 |         
 57 | #ifndef BOND_NO_CXX11_DEFAULTED_FUNCTIONS
 58 |         // Compiler generated operator= OK
 59 |         Request& operator=(const Request& other) = default;
 60 | #endif
 61 | 
 62 |         bool operator==(const Request& other) const
 63 |         {
 64 |             return true
 65 |                 && (keyword == other.keyword);
 66 |         }
 67 | 
 68 |         bool operator!=(const Request& other) const
 69 |         {
 70 |             return !(*this == other);
 71 |         }
 72 | 
 73 |         void swap(Request& other)
 74 |         {
 75 |             using std::swap;
 76 |             swap(keyword, other.keyword);
 77 |         }
 78 | 
 79 |         struct Schema;
 80 | 
 81 |     protected:
 82 |         void InitMetadata(const char*, const char*)
 83 |         {
 84 |         }
 85 |     };
 86 | 
 87 |     inline void swap(Request& left, Request& right)
 88 |     {
 89 |         left.swap(right);
 90 |     }
 91 | 
 92 |     
 93 |     struct Response
 94 |     {
 95 |         std::set<std::string> matches;
 96 |         
 97 |         Response()
 98 |         {
 99 |         }
100 | 
101 |         
102 | #ifndef BOND_NO_CXX11_DEFAULTED_FUNCTIONS
103 |         // Compiler generated copy ctor OK
104 |         Response(const Response& other) = default;
105 | #endif
106 |         
107 | #ifndef BOND_NO_CXX11_RVALUE_REFERENCES
108 |         Response(Response&& other)
109 |           : matches(std::move(other.matches))
110 |         {
111 |         }
112 | #endif
113 |         
114 |         
115 | #ifndef BOND_NO_CXX11_DEFAULTED_FUNCTIONS
116 |         // Compiler generated operator= OK
117 |         Response& operator=(const Response& other) = default;
118 | #endif
119 | 
120 |         bool operator==(const Response& other) const
121 |         {
122 |             return true
123 |                 && (matches == other.matches);
124 |         }
125 | 
126 |         bool operator!=(const Response& other) const
127 |         {
128 |             return !(*this == other);
129 |         }
130 | 
131 |         void swap(Response& other)
132 |         {
133 |             using std::swap;
134 |             swap(matches, other.matches);
135 |         }
136 | 
137 |         struct Schema;
138 | 
139 |     protected:
140 |         void InitMetadata(const char*, const char*)
141 |         {
142 |         }
143 |     };
144 | 
145 |     inline void swap(Response& left, Response& right)
146 |     {
147 |         left.swap(right);
148 |     }
149 | } // namespace Message
150 | 
151 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Distributed Search Engine in C++14
 2 | 
 3 | Technical walkthrough: https://daniel-j-h.github.io/post/distributed-search-nanomsg-bond/
 4 | 
 5 | This project mimics Rob Pike's search example introducing Go's channels:
 6 | 
 7 | * Query different services (Web Search, Image Search, Video Search, ...)
 8 | * Gather and merge results
 9 | * Timeouts, replication, reconnection, and more!
10 | 
11 | Instead of using Go and its channels, this project makes use of nanomsg for communication and bond for serialization.
12 | 
13 | See:
14 | 
15 | * https://talks.golang.org/2012/concurrency.slide#42 (slides 42-52)
16 | * https://www.youtube.com/watch?v=f6kdp27TYZs&t=1721 (starting from 28:40)
17 | 
18 | If you're interested in nanomsg and bond (or similar technologies, such as ZeroMQ and Cap'n Proto) this is probably the perfect project for you to get started :)
19 | 
20 | 
21 | ## Requirements:
22 | 
23 | * nanomsg for communication: https://github.com/nanomsg/nanomsg
24 | * bond for serialization: https://github.com/Microsoft/bond
25 | 
26 | Note: RapidJSON is required as of now, see: https://github.com/Microsoft/bond/issues/44
27 | As a quickfix build it from bond's thirdparty directory then copy the rapidjson/include/rapidjson directory to /usr/local/include/.
28 | 
29 | Message stubs were generated with bond's gbc using:
30 | 
31 |     gbc c++ Messages.bond
32 | 
33 | See also: https://microsoft.github.io/bond/manual/bond_cpp.html
34 | 
35 | 
36 | ## Usage:
37 | 
38 | To start the user facing Search service, you can issue queries against:
39 | 
40 |     ./Search
41 | 
42 | Now the Search service will show you no results yet, since there is no service for answering queries. Let's start some:
43 | 
44 |     ./WebSearch
45 | 
46 | Now you'll get WebSearch results. There is also a ImageSearch and a VideoSearch service. You can start and stop them at any time. Replication is also possible, but note that the results are unique. For example when two WebSearch services are started, the Search service receives the web results from both of them, but only shows them once.
47 | 
48 | By default the services communicate over TCP port 9995. You can change this if you want:
49 | 
50 |     ./Search "tcp://*:9555"
51 |     ./WebSearch "tcp://localhost:9555"
52 | 
53 | 
54 | Or using IPC:
55 | 
56 |     ./Search "ipc:///tmp/search.ipc"
57 |     ./WebSearch "ipc:///tmp/search.ipc"
58 | 
59 | 
60 | There is a RecursiveSearch service available, showing an example of how to build a service tree:
61 | 
62 |     ./Search "tcp://*:9995"
63 |     ./RecursiveSearch "tcp://*:9996" "tcp://localhost:9995"
64 |     ./WebSearch "tcp://localhost:9996"
65 | 
66 | With this setup, ./Search is the tree's root, with ./RecursiveSearch attached to it and ./WebSearch attached to the ./RecursiveSearch leaf. Attaching more services is possible on each layer of the tree. Just attach them to the subtree's specific root-service.
67 | 
68 | Note: if you try the recursive example on a single machine, you have to change the port for each layer, otherwise there would be no way to distinguish the root from internal tree nodes.
69 | 
70 | 
71 | ## License
72 | 
73 | Copyright © 2015 Daniel J. Hofmann
74 | 
75 | Distributed under the MIT License (MIT).
76 | 


--------------------------------------------------------------------------------
/RecursiveSearch.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | 
 3 | #include "Common.h"
 4 | #include "Service.h"
 5 | 
 6 | int main(int argc, char** argv) try {
 7 |   Common::init();
 8 | 
 9 |   const auto args = Common::args(argc, argv);
10 |   const auto bind = args.size() >= 2 ? args[1] : "tcp://*:9996";
11 |   const auto conn = args.size() == 3 ? args[2] : "tcp://localhost:9995";
12 | 
13 |   // requester represents attaching services
14 |   Service::Requester requester{bind, 500};  // lower timeout, since we have to respond, too
15 | 
16 |   // responder represents service we send the results to
17 |   Service::Responder responder{conn};
18 | 
19 |   // recursively query attached services; pass request on
20 |   const auto recursiveRequestHandler = [&requester](const Common::Request& req) -> Common::Response {
21 |     return requester.query(req);
22 |   };
23 | 
24 |   responder.onRequest(recursiveRequestHandler);
25 | 
26 | 
27 | } catch (const std::exception& e) {
28 |   std::cerr << e.what() << std::endl;
29 | }
30 | 


--------------------------------------------------------------------------------
/Search.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <string>
 3 | 
 4 | #include "Common.h"
 5 | #include "Service.h"
 6 | 
 7 | 
 8 | static void onInteraction(Common::RequestHandler);
 9 | 
10 | 
11 | int main(int argc, char** argv) try {
12 |   Common::init();
13 | 
14 |   const auto args = Common::args(argc, argv);
15 |   const auto endpoint = args.size() == 2 ? args[1] : "tcp://*:9995";
16 | 
17 |   Service::Requester self{endpoint};
18 | 
19 |   const auto handler = [&self](const Common::Request& req) -> Common::Response {
20 |     // query all attached services on interaction
21 |     return self.query(req);
22 |   };
23 | 
24 |   onInteraction(handler);
25 | 
26 | } catch (const std::exception& e) {
27 |   std::cerr << e.what() << std::endl;
28 | }
29 | 
30 | 
31 | static void onInteraction(Common::RequestHandler handler) {
32 |   std::string request;
33 |   do {
34 |     if (!request.empty()) {
35 |       const auto responses = handler(request);
36 | 
37 |       std::cout << "\nResults>\n";
38 |       for (auto&& resp : responses)
39 |         std::cout << " * " << resp << '\n';
40 |     }
41 |     std::cout << "\nSearch>" << std::endl;
42 |   } while (std::getline(std::cin, request));
43 | }
44 | 


--------------------------------------------------------------------------------
/Service.cc:
--------------------------------------------------------------------------------
  1 | #include <cassert>
  2 | #include <cstdint>
  3 | #include <cstddef>
  4 | #include <stdexcept>
  5 | #include <memory>
  6 | #include <iterator>
  7 | #include <utility>
  8 | 
  9 | #include <nanomsg/nn.h>
 10 | #include <nanomsg/survey.h>
 11 | 
 12 | #include "Messages_reflection.h"
 13 | #include <bond/core/bond.h>
 14 | #include <bond/stream/output_buffer.h>
 15 | 
 16 | #include <tbb/pipeline.h>
 17 | 
 18 | #include "Service.h"
 19 | 
 20 | #define nn_throw_on(...)                                                                                               \
 21 |   if (__VA_ARGS__)                                                                                                     \
 22 |     throw std::runtime_error{::nn_strerror(::nn_errno())};
 23 | 
 24 | namespace Service {
 25 | 
 26 | Requester::Requester(const std::string& address, int timeout) {
 27 |   sock_ = ::nn_socket(AF_SP, NN_SURVEYOR);
 28 |   nn_throw_on(sock_ < 0);
 29 | 
 30 |   int rv;
 31 | 
 32 |   rv = ::nn_setsockopt(sock_, NN_SURVEYOR, NN_SURVEYOR_DEADLINE, &timeout, sizeof(timeout));
 33 |   nn_throw_on(rv < 0);
 34 | 
 35 |   rv = ::nn_bind(sock_, address.c_str());
 36 |   nn_throw_on(rv < 0);
 37 | }
 38 | 
 39 | Requester::~Requester() {
 40 |   const int rv = ::nn_close(sock_);
 41 |   assert(rv == 0);
 42 | }
 43 | 
 44 | Common::Response Requester::query(const std::string& keyword) {
 45 |   // make request
 46 |   Message::Request req;
 47 |   req.keyword = keyword;
 48 | 
 49 |   bond::OutputBuffer output;
 50 |   bond::CompactBinaryWriter<bond::OutputBuffer> writer{output};
 51 |   Serialize(req, writer);
 52 | 
 53 |   const auto blob = output.GetBuffer();
 54 | 
 55 |   int rv = ::nn_send(sock_, blob.data(), blob.size(), 0);
 56 |   nn_throw_on(rv < 0);
 57 | 
 58 |   // merge responses into one, containing all matches
 59 |   Common::Response matches;
 60 | 
 61 | 
 62 |   const auto releaser = [](void* msg) { ::nn_freemsg(msg); };
 63 |   // debian jessie's tbb version (4.2) does not support moving data between stages (yet?)
 64 |   // i.e. moveable unique_ptr -- why? tbb fix this! shared_ptr with custom deleter does the job.
 65 |   using Blob = std::shared_ptr<char>;
 66 |   using SizedBlob = std::pair<Blob, std::size_t>;
 67 | 
 68 |   // received memory blobs from the network and passes them on
 69 |   const auto receiveFn = [this, releaser](tbb::flow_control& flow) -> SizedBlob {
 70 |     int recvRv;
 71 |     char* recvBuffer{};
 72 | 
 73 |     if ((recvRv = ::nn_recv(sock_, &recvBuffer, NN_MSG, 0)) >= 0) {
 74 |       assert(recvBuffer);
 75 | 
 76 |       Blob memBlob{recvBuffer, releaser};
 77 |       SizedBlob sBlob{std::move(memBlob), static_cast<std::size_t>(recvRv)};
 78 |       return sBlob;
 79 |     } else {
 80 |       flow.stop();
 81 |       return SizedBlob{Blob{nullptr}, 0};
 82 |     }
 83 |   };
 84 | 
 85 |   // gets memory blobs from stage before, deserializes blobs as Responses and passes them on
 86 |   const auto deserializeFn = [](SizedBlob sBlob) -> Message::Response {
 87 |     bond::InputBuffer input{sBlob.first.get(), static_cast<std::uint32_t>(sBlob.second)};
 88 |     bond::CompactBinaryReader<bond::InputBuffer> reader{input};
 89 | 
 90 |     Message::Response resp;
 91 |     Deserialize(reader, resp);
 92 | 
 93 |     return resp;
 94 |     // blob's lifetime ends, releaser gets invoked, automatically cleaning up the message
 95 |   };
 96 | 
 97 |   // gets responses from stage before, merges them into single response object
 98 |   const auto mergeFn = [&](Message::Response resp) -> void {
 99 |     // this merges all responses, discarding duplicates (set property)
100 |     matches.insert(begin(resp.matches), end(resp.matches));
101 |   };
102 | 
103 | 
104 |   // upper bound of number of stages that will be run concurrently
105 |   const constexpr auto tokens = 256u;
106 | 
107 |   const constexpr auto par = tbb::filter::parallel;
108 |   const constexpr auto seq = tbb::filter::serial_out_of_order;
109 | 
110 |   const auto receiveStage = tbb::make_filter<void, SizedBlob>(seq, receiveFn);
111 |   const auto deserializeStage = tbb::make_filter<SizedBlob, Message::Response>(par, deserializeFn);
112 |   const auto mergeStage = tbb::make_filter<Message::Response, void>(seq, mergeFn);
113 | 
114 |   tbb::parallel_pipeline(tokens, receiveStage & deserializeStage & mergeStage);
115 | 
116 |   return matches;
117 | }
118 | 
119 | Responder::Responder(const std::string& endpoint) {
120 |   sock_ = ::nn_socket(AF_SP, NN_RESPONDENT);
121 |   nn_throw_on(sock_ < 0);
122 | 
123 |   const int rv = ::nn_connect(sock_, endpoint.c_str());
124 |   nn_throw_on(rv < 0);
125 | }
126 | 
127 | Responder::~Responder() {
128 |   const int rv = ::nn_close(sock_);
129 |   assert(rv == 0);
130 | }
131 | 
132 | void Responder::onRequest(Common::RequestHandler handler) {
133 |   int rv;
134 |   char* recv_buffer{};
135 |   const auto releaser = [](void* msg) { ::nn_freemsg(msg); };
136 | 
137 |   // eventloop
138 |   for (;;) {
139 |     // get keyword request
140 |     rv = ::nn_recv(sock_, &recv_buffer, NN_MSG, 0);
141 |     nn_throw_on(rv < 0);
142 | 
143 |     assert(recv_buffer);
144 | 
145 |     // make sure the receive buffer is cleaned up on all exit paths (e.g. bond may throw)
146 |     const std::unique_ptr<char[], decltype(releaser)> defer{recv_buffer, releaser};
147 | 
148 |     bond::InputBuffer input{recv_buffer, static_cast<std::uint32_t>(rv)};
149 |     bond::CompactBinaryReader<bond::InputBuffer> reader{input};
150 | 
151 |     Message::Request req;
152 |     Deserialize(reader, req);
153 | 
154 |     // respond with matches; call site provides handler
155 |     const Common::Response matches = handler(Common::Request{req.keyword});
156 | 
157 |     Message::Response resp;
158 |     resp.matches.insert(begin(matches), end(matches));
159 | 
160 |     bond::OutputBuffer output;
161 |     bond::CompactBinaryWriter<bond::OutputBuffer> writer{output};
162 |     Serialize(resp, writer);
163 | 
164 |     const auto blob = output.GetBuffer();
165 | 
166 |     rv = ::nn_send(sock_, blob.data(), blob.size(), 0);
167 |     nn_throw_on(rv < 0);
168 |   }
169 | }
170 | 
171 | #undef nn_throw_on
172 | }
173 | 


--------------------------------------------------------------------------------
/Service.h:
--------------------------------------------------------------------------------
 1 | #ifndef SEARCH_SERVICE_H
 2 | #define SEARCH_SERVICE_H
 3 | 
 4 | #include <string>
 5 | 
 6 | #include "Common.h"
 7 | 
 8 | namespace Service {
 9 | 
10 | class Requester final {
11 | public:
12 |   Requester(const std::string& address, int timeout = 1000);
13 |   ~Requester();
14 | 
15 |   // query attached services, return responses (unique, merged) for specific keyword
16 |   Common::Response query(const std::string& keyword);
17 | 
18 |   Requester(const Requester&) = delete;
19 |   Requester& operator=(const Requester&) = delete;
20 | 
21 | private:
22 |   int sock_;
23 | };
24 | 
25 | 
26 | class Responder final {
27 | public:
28 |   Responder(const std::string& endpoint);
29 |   ~Responder();
30 | 
31 |   // wait for requests, process them with user-provided handler
32 |   void onRequest(Common::RequestHandler handler);
33 | 
34 |   Responder(const Responder&) = delete;
35 |   Responder& operator=(const Responder&) = delete;
36 | 
37 | private:
38 |   int sock_;
39 | };
40 | 
41 | }
42 | 
43 | #endif
44 | 


--------------------------------------------------------------------------------
/VideoSearch.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | 
 3 | #include "Common.h"
 4 | #include "Service.h"
 5 | 
 6 | int main(int argc, char** argv) try {
 7 |   Common::init();
 8 | 
 9 |   const auto args = Common::args(argc, argv);
10 |   const auto endpoint = args.size() == 2 ? args[1] : "tcp://localhost:9995";
11 | 
12 |   Service::Responder self{endpoint};
13 | 
14 |   // dummy request handler
15 |   const auto requestHandler = [&self](const Common::Request& req) -> Common::Response {
16 |     (void)req; // unused
17 |     return {"First Video Result", "Second Video Result"};
18 |   };
19 | 
20 |   self.onRequest(requestHandler);
21 | 
22 | } catch (const std::exception& e) {
23 |   std::cerr << e.what() << std::endl;
24 | }
25 | 


--------------------------------------------------------------------------------
/WebSearch.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | 
 3 | #include "Common.h"
 4 | #include "Service.h"
 5 | 
 6 | int main(int argc, char** argv) try {
 7 |   Common::init();
 8 | 
 9 |   const auto args = Common::args(argc, argv);
10 |   const auto endpoint = args.size() == 2 ? args[1] : "tcp://localhost:9995";
11 | 
12 |   Service::Responder self{endpoint};
13 | 
14 |   // dummy request handler
15 |   const auto requestHandler = [&self](const Common::Request& req) -> Common::Response {
16 |     (void)req; // unused
17 |     return {"First Web Result", "Second Web Result"};
18 |   };
19 | 
20 |   self.onRequest(requestHandler);
21 | 
22 | } catch (const std::exception& e) {
23 |   std::cerr << e.what() << std::endl;
24 | }
25 | 


--------------------------------------------------------------------------------