├── .gitignore ├── CMakeLists.txt ├── CONTRIBUTING ├── LICENSE ├── README ├── README.md ├── analysis_options.proto ├── analyzers ├── examples │ ├── account_access_analyzer.cc │ ├── account_access_analyzer.h │ ├── account_access_analyzer_test.cc │ ├── account_access_defs.cc │ ├── account_access_defs.h │ ├── account_access_graph.cc │ ├── account_access_graph.h │ ├── account_access_graph_test.cc │ ├── curio_analyzer.cc │ ├── curio_analyzer.h │ ├── curio_analyzer_test.cc │ ├── curio_defs.cc │ ├── curio_defs.h │ ├── stream_dependency_graph.cc │ ├── stream_dependency_graph.h │ └── stream_dependency_graph_test.cc └── plaso │ ├── plaso_analyzer.cc │ ├── plaso_analyzer.h │ ├── plaso_analyzer_test.cc │ ├── plaso_defs.cc │ ├── plaso_defs.h │ ├── plaso_event.cc │ ├── plaso_event.h │ ├── plaso_event.proto │ ├── plaso_event_graph.cc │ ├── plaso_event_graph.h │ ├── plaso_event_graph_test.cc │ └── plaso_event_test.cc ├── base ├── string.h └── vector.h ├── build_test ├── README ├── account_access_analyzer_build_test.cc ├── account_access_graph_build_test.cc ├── analysis_options_build_test.cc ├── ast_build_test.cc ├── curio_analyzer_build_test.cc ├── dot_printer_build_test.cc ├── gflags_build_test.cc ├── graph_exporter_build_test.cc ├── graph_transformer_build_test.cc ├── jsoncpp_build_test.cc ├── labeled_graph_build_test.cc ├── logging_build_test.cc ├── morphism_build_test.cc ├── plaso_analyzer_build_test.cc ├── plaso_event_build_test.cc ├── plaso_event_graph_build_test.cc ├── stream_dependency_graph_build_test.cc ├── type_build_test.cc ├── type_checker_build_test.cc ├── value_build_test.cc └── value_checker_build_test.cc ├── frontend.cc ├── frontend.h ├── graph ├── ast.cc ├── ast.h ├── ast.proto ├── ast_test.cc ├── dot_printer.cc ├── dot_printer.h ├── dot_printer_test.cc ├── graph_analyzer.cc ├── graph_analyzer.h ├── graph_analyzer_test.cc ├── graph_explorer.proto ├── graph_exporter.cc ├── graph_exporter.h ├── graph_interface.h ├── graph_transformer.cc ├── graph_transformer.h ├── graph_transformer_test.cc ├── labeled_graph.cc ├── labeled_graph.h ├── labeled_graph_test.cc ├── morphism.cc ├── morphism.h ├── morphism_test.cc ├── test_graphs.cc ├── test_graphs.h ├── test_graphs_test.cc ├── type.cc ├── type.h ├── type_checker.cc ├── type_checker.h ├── type_checker_test.cc ├── type_test.cc ├── value.cc ├── value.h ├── value_checker.cc ├── value_checker.h ├── value_checker_test.cc └── value_test.cc ├── gtest.h ├── logle.cc ├── third_party ├── CMakeLists.txt ├── CMakeLists.txt.gflags ├── CMakeLists.txt.googletest └── CMakeLists.txt.jsoncpp └── util ├── CMakeLists.txt ├── csv.cc ├── csv.h ├── csv_test.cc ├── json_reader.cc ├── json_reader.h ├── logging.cc ├── logging.h ├── map_utils.h ├── map_utils_test.cc ├── status.cc ├── status.h ├── status_test.cc ├── string_utils.cc ├── string_utils.h ├── string_utils_test.cc ├── time_utils.cc ├── time_utils.h └── time_utils_test.cc /.gitignore: -------------------------------------------------------------------------------- 1 | <<<<<<< HEAD 2 | BUILD 3 | build/* 4 | google/* 5 | README.google 6 | google3/third_party/logle/BUILD 7 | ======= 8 | google/ 9 | BUILD 10 | logle.cc 11 | >>>>>>> 105867e... ported includes 12 | -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | This project is being developed purely internally at the moment. We are not 2 | accepting contributions. 3 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Logle is a tool for using graphs to analyze and summarize the content of 2 | formatted logs. The primary use case is to analyze supertimelines generated by 3 | Plaso, which is an open source, forensic analysis tool 4 | (https://github.com/log2timeline/plaso). Logle currently parses supertimelines 5 | and constructs labeled graphs from them. There is support for visualization 6 | using GraphViz. 7 | 8 | Logle is not an official Google product. 9 | 10 | Development is at an early stage. 11 | 12 | About the name: 13 | logle is a portmanteau of log and ogle. The intended pronunciation is as a 14 | rhyme of 'ogle'. Eg: "let's logle long logs." 15 | 16 | 17 | To install run the following commands: 18 | 19 | mkdir build 20 | cd build 21 | cmake .. 22 | 23 | The project uses the following libraries: 24 | - The Boost graph and string libraries. 25 | - The Boost regular expression library. 26 | - The Jsoncpp library for parsing JSON. 27 | - The Google Test libraries. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Morphie : Data Analysis with Graphs ## 3 | 4 | Morphie is a tool that uses graphs to analyze and summarize the content of 5 | formatted input data in JSON or CSV format. The primary use case is to analyze 6 | supertimelines generated by [Plaso](https://github.com/log2timeline/plaso), 7 | which is an open source, forensic analysis tool . Morphie currently parses 8 | supertimelines and constructs labeled graphs from them. There is support for 9 | visualization using [GraphViz](http://www.graphviz.org/). 10 | 11 | ### Disclaimers ### 12 | 13 | * Morphie is not an official Google product. 14 | 15 | * Development is at an early stage. 16 | 17 | ### About the name ### 18 | 19 | Morphie uses graph transformations to reduce the amount of structure and 20 | complexity in a graph constructed from log data. It morphs graphs and more 21 | precisely, the relationships between the graphs before and after a 22 | transformation are given by a kind of homomorphism. 23 | 24 | 25 | ## System Requirements ## 26 | 27 | In order to build, the following packages must be available in the system. 28 | * The [CMake](https://cmake.org/) build system, version 2.8.12 or higher. 29 | * The [Google Protocol Buffer compiler](https://github.com/google/protobuf/tree/v2.6.1), version 2.6.1 or higher. 30 | * Some Debian repositories are still at version 2.5.0, in which case, the 31 | compiler can be installed from GitHub. 32 | * The build currently requires a system-wide install. 33 | * If you system has multiple versions, check that `protoc` resolves to the 34 | 2.6.1 version. 35 | * The [Boost C++ libraries](http://www.boost.org/). 36 | * The [Boost Graph Library](http://www.boost.org/doc/libs/1_61_0/libs/graph/doc/table_of_contents.html). 37 | * The [Boost String Algorithms Library](http://www.boost.org/doc/libs/1_61_0/doc/html/string_algo.html). 38 | * [Boost Regex](http://www.boost.org/doc/libs/1_43_0/libs/regex/doc/html/index.html), version 1.43.0 or higher. 39 | 40 | Network access is required to build because the following packages will be 41 | downloaded. 42 | * The [Google Test libraries](https://github.com/google/googletest). 43 | * The [JsonCpp](https://github.com/open-source-parsers/jsoncpp) JSON parser. 44 | * The [gflags package](https://github.com/gflags/gflags). 45 | ([An explanation](https://github.com/gflags/gflags#25-january-2012) 46 | of how Google flags became gflags). 47 | 48 | ## Installation ## 49 | 50 | To install, run the following commands: 51 | 52 | ``` 53 | # Get the code from GitHub 54 | git clone git@github.com:google/morphie.git 55 | # Create a directory in which CMake can do its magic 56 | cd morphie 57 | mkdir build 58 | # Create a directory in which CMake can do its magic 59 | cd build 60 | cmake .. 61 | make 62 | ``` 63 | -------------------------------------------------------------------------------- /analysis_options.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Command-line options supported by the tool, represented as protocol buffers. 16 | syntax = "proto2"; 17 | 18 | package morphie; 19 | 20 | // Options available for analyzing Plaso input. 21 | message PlasoOptions { 22 | // If true, every source file of an event will be displayed in output graphs. 23 | optional bool show_all_sources = 1 [default = false]; 24 | } 25 | 26 | // An AnalysisOptions message specifies which analyzer should be run and the 27 | // input and output formats for that analyzer. 28 | message AnalysisOptions { 29 | // The name of the analyzer to run. 30 | optional string analyzer = 1; 31 | 32 | // The name of the file containing the data to analyze. This must be a CSV, 33 | // JSON object or JSON stream file. A JSON object file contains one single 34 | // JSON object and a JSON stream file contains a sequence of JSON objects. 35 | oneof input_file { 36 | string csv_file = 2; 37 | string json_file = 3; 38 | string json_stream_file = 4; 39 | } 40 | 41 | // Visual output can be written to as a GraphViz DOT or a proto accepted by 42 | // GraphExplorer. The proto is represented as a human readable string obtained 43 | // by calling DebugString() on a message. 44 | oneof output_file { 45 | string output_dot_file = 5; 46 | string output_pbtxt_file = 6; 47 | } 48 | 49 | optional PlasoOptions plaso_options = 7; 50 | } 51 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_analyzer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/account_access_analyzer.h" 16 | 17 | #include // NOLINT 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "analyzers/examples/account_access_defs.h" 24 | #include "base/vector.h" 25 | #include "util/logging.h" 26 | #include "util/string_utils.h" 27 | 28 | namespace { 29 | 30 | const int kMaxMalformedLines = 1000000; 31 | const char kNullAccessGraphErr[] = "The access graph is null."; 32 | 33 | } // namespace 34 | 35 | namespace morphie { 36 | // This function performs a series of checks, described by the error message 37 | // generated if the check fails. 38 | util::Status AccessAnalyzer::Initialize( 39 | std::unique_ptr parser) { 40 | csv_parser_ = std::move(parser); 41 | auto row_it = csv_parser_->begin(); 42 | if (row_it == csv_parser_->end()) { 43 | return util::Status(Code::INVALID_ARGUMENT, "The input is empty."); 44 | } 45 | util::Status status = InitializeFieldMap(); 46 | if (!status.ok()) { 47 | return status; 48 | } 49 | ++row_it; 50 | if (row_it == csv_parser_->end()) { 51 | return util::Status(Code::INVALID_ARGUMENT, "No data in the input."); 52 | } 53 | return util::Status::OK; 54 | } 55 | 56 | int AccessAnalyzer::NumGraphNodes() const { 57 | CHECK(access_graph_ != nullptr, kNullAccessGraphErr); 58 | return access_graph_->NumNodes(); 59 | } 60 | 61 | int AccessAnalyzer::NumGraphEdges() const { 62 | CHECK(access_graph_ != nullptr, kNullAccessGraphErr); 63 | return access_graph_->NumEdges(); 64 | } 65 | 66 | util::Status AccessAnalyzer::BuildAccessGraph() { 67 | if (csv_parser_ == nullptr) { 68 | return util::Status(Code::INVALID_ARGUMENT, 69 | "The CSV parser has not been initialized."); 70 | } 71 | if (access_graph_ != nullptr) { 72 | return util::Status(Code::INVALID_ARGUMENT, 73 | "The access graph has already been created."); 74 | } 75 | access_graph_.reset(new AccountAccessGraph); 76 | util::Status status = access_graph_->Initialize(); 77 | if (!status.ok()) { 78 | access_graph_.reset(nullptr); 79 | return status; 80 | } 81 | for (const util::Record& record : *csv_parser_) { 82 | ++num_lines_read_; 83 | if (record.fields().size() != field_to_index_.size()) { 84 | IncrementSkipCounter(); 85 | continue; 86 | } 87 | access_graph_->ProcessAccessData(field_to_index_, record.fields()); 88 | } 89 | return util::Status::OK; 90 | } 91 | 92 | string AccessAnalyzer::AccessGraphAsDot() const { 93 | return (access_graph_ == nullptr) ? "" : access_graph_->ToDot(); 94 | } 95 | 96 | void AccessAnalyzer::IncrementSkipCounter() { 97 | ++num_lines_skipped_; 98 | CHECK(num_lines_skipped_ < kMaxMalformedLines, 99 | util::StrCat("Over ", std::to_string(kMaxMalformedLines), 100 | " malformed lines in input. Aborting.")); 101 | } 102 | 103 | util::Status AccessAnalyzer::InitializeFieldMap() { 104 | const std::vector& field_names = csv_parser_->begin()->fields(); 105 | if (field_names.empty()) { 106 | return util::Status(Code::INVALID_ARGUMENT, "First line has no columns."); 107 | } 108 | int index = 0; 109 | std::set provided_fields; 110 | for (const string& field_name : field_names) { 111 | if (field_name.empty()) { 112 | return util::Status( 113 | Code::INVALID_ARGUMENT, 114 | util::StrCat("Column number ", std::to_string(index + 1), 115 | " has no name.")); 116 | } 117 | auto insert_status = field_to_index_.insert({field_name, index}); 118 | if (!insert_status.second) { 119 | return util::Status( 120 | Code::INVALID_ARGUMENT, 121 | util::StrCat("More than one column named:", field_name)); 122 | } 123 | provided_fields.insert(field_name); 124 | ++index; 125 | } 126 | const std::set required_fields = 127 | util::SplitToSet(access::kRequiredFields, ','); 128 | std::set missing_fields; 129 | std::set_difference(required_fields.begin(), required_fields.end(), 130 | provided_fields.begin(), provided_fields.end(), 131 | std::inserter(missing_fields, missing_fields.begin())); 132 | if (missing_fields.size() > 0) { 133 | string fields = util::SetJoin(missing_fields, ","); 134 | return util::Status( 135 | Code::INVALID_ARGUMENT, 136 | util::StrCat("The following required fields are missing: ", fields)); 137 | } 138 | return util::Status::OK; 139 | } 140 | 141 | } // namespace morphie 142 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_analyzer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // An analyzer for visualizing relationships between actors and accounts they 16 | // access. The input data to this analyzer provides details about an actor and 17 | // the accounts they access. The output of this analyzer is a visualization of 18 | // this data. 19 | #ifndef LOGLE_ACCOUNT_ACCESS_ANALYZER_H_ 20 | #define LOGLE_ACCOUNT_ACCESS_ANALYZER_H_ 21 | 22 | #include 23 | #include 24 | 25 | #include "analyzers/examples/account_access_graph.h" 26 | #include "base/string.h" 27 | #include "util/csv.h" 28 | #include "util/status.h" 29 | 30 | namespace morphie { 31 | 32 | // The AccessAnalyzer is initialized with a CSV parser. The fields that must be 33 | // present in the CSV input are defined in account_access_defs.h. 34 | class AccessAnalyzer { 35 | public: 36 | AccessAnalyzer() : num_lines_read_(0), num_lines_skipped_(0) {} 37 | 38 | // Initializes the analyzer using a CSV parser. Returns 39 | // * OK : if the following requirements are satisfied. 40 | // - The input contains the fields in access::kRequiredFields. 41 | // - There is at least one row of data after the CSV header. 42 | // * INVALID_ARGUMENT : otherwise with the error message containing the 43 | // reason why initialization failed. 44 | util::Status Initialize(std::unique_ptr parser); 45 | 46 | util::Status BuildAccessGraph(); 47 | 48 | // Utilities for accounting and error checking. 49 | int NumLinesRead() const { return num_lines_read_; } 50 | int NumLinesSkipped() const { return num_lines_skipped_; } 51 | int NumLinesProcessed() const { return num_lines_read_ - num_lines_skipped_; } 52 | 53 | // Statistics about the account access graph. These functions require that the 54 | // access graph has been created and crash otherwise. 55 | int NumGraphNodes() const; 56 | int NumGraphEdges() const; 57 | 58 | // Returns the account access graph in GraphViz DOT format. 59 | string AccessGraphAsDot() const; 60 | 61 | private: 62 | void IncrementSkipCounter(); 63 | // Initializes field_to_index_ using the first line of csv_parser_. 64 | util::Status InitializeFieldMap(); 65 | 66 | // A map from input field names to the input column with that data. 67 | unordered_map field_to_index_; 68 | std::unique_ptr access_graph_; 69 | 70 | int num_lines_read_; 71 | int num_lines_skipped_; 72 | std::unique_ptr csv_parser_; 73 | }; 74 | 75 | } // namespace morphie 76 | #endif // LOGLE_ACCOUNT_ACCESS_ANALYZER_H_ 77 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_analyzer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/account_access_analyzer.h" 16 | 17 | #include 18 | 19 | #include "gtest.h" 20 | #include "util/csv.h" 21 | #include "util/status.h" 22 | #include "util/string_utils.h" 23 | 24 | namespace morphie { 25 | namespace { 26 | 27 | void TestCSVInitialization(const char* kInput, Code code) { 28 | std::unique_ptr parser( 29 | new util::CSVParser(new std::stringstream(kInput))); 30 | AccessAnalyzer access_analyzer; 31 | util::Status s = access_analyzer.Initialize(std::move(parser)); 32 | EXPECT_EQ(code, s.code()); 33 | } 34 | 35 | void TestInvalidCSVInitialization(const char* kInput) { 36 | TestCSVInitialization(kInput, Code::INVALID_ARGUMENT); 37 | } 38 | 39 | void TestValidCSVInitialization(const char* kInput) { 40 | TestCSVInitialization(kInput, Code::OK); 41 | } 42 | 43 | // The next few tests below check that the analyzer cannot be initialized with 44 | // invalid input. 45 | TEST(AccessAnalyzerTest, RejectsEmptyCSVInput) { 46 | TestInvalidCSVInitialization(""); 47 | TestInvalidCSVInitialization("\n\n"); 48 | } 49 | 50 | TEST(AccessAnalyzerTest, RejectsNamelessCSVFields) { 51 | TestInvalidCSVInitialization(",fromx,tox\n"); 52 | TestInvalidCSVInitialization("fromx,tox,\n"); 53 | TestInvalidCSVInitialization("fromx,,tox\n"); 54 | } 55 | 56 | TEST(AccessAnalyzerTest, RejectsDuplicateCSVFieldNames) { 57 | TestInvalidCSVInitialization("fromx,fromx,tox\n"); 58 | TestInvalidCSVInitialization("fromx,tox,fromx\n"); 59 | } 60 | 61 | TEST(AccessAnalyzerTest, RejectsHeaderOnlyInput) { 62 | TestInvalidCSVInitialization("fromx,attr_actor_title,tox,attr_count"); 63 | } 64 | 65 | const char header[] = 66 | "fromx,tox,attr_actor_manager,attr_actor_cost_center," 67 | "attr_first_access," 68 | "sttr_last_access,attr_count,attr_actor_title"; 69 | 70 | TEST(AccessAnalyzerTest, AcceptsNonEmptyOnlyInput) { 71 | string content = "\nabc@xyz.tuv,def@tuv.xyz,Alpha,None,1,2,3,Engineer"; 72 | TestValidCSVInitialization(util::StrCat(header, content).c_str()); 73 | } 74 | 75 | // Test that running the analyzer on 'kInput' produces a graph with the expected 76 | // number of nodes and edges. 77 | void TestGraphConstruction(const char* kInput, int num_nodes, int num_edges) { 78 | std::unique_ptr parser( 79 | new util::CSVParser(new std::stringstream(kInput))); 80 | AccessAnalyzer access_analyzer; 81 | util::Status s = access_analyzer.Initialize(std::move(parser)); 82 | ASSERT_TRUE(s.ok()); 83 | s = access_analyzer.BuildAccessGraph(); 84 | EXPECT_TRUE(s.ok()) << "Graph construction failed."; 85 | EXPECT_EQ(num_nodes, access_analyzer.NumGraphNodes()) 86 | << "Error when processing :\n" 87 | << kInput; 88 | EXPECT_EQ(num_edges, access_analyzer.NumGraphEdges()) 89 | << "Error when processing :\n" 90 | << kInput; 91 | } 92 | 93 | TEST(AccessAnalyzerTest, TestGraphConstruction) { 94 | string content1 = "\nabc@xyz.tuv,def@tuv.xyz,Alpha,None,1,2,3,Engineer"; 95 | // One actor and one account and one edge between them. 96 | TestGraphConstruction(util::StrCat(header, content1).c_str(), 2, 1); 97 | // Duplicating the data will not increase the number of edges in the graph. 98 | TestGraphConstruction(util::StrCat(header, content1, content1).c_str(), 2, 1); 99 | string content2 = "\nabc@xyz.tuv,ghi@tuv.xyz,Alpha,None,1,2,3,Engineer"; 100 | TestGraphConstruction(util::StrCat(header, content1, content2).c_str(), 3, 2); 101 | string content3 = "\ndef@xperson,ghi@tuv.xyz,Alpha,None,1,2,3,Engineer"; 102 | TestGraphConstruction(util::StrCat(header, content1, content2).c_str(), 3, 2); 103 | } 104 | 105 | } // namespace 106 | } // namespace morphie 107 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_defs.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/account_access_defs.h" 16 | 17 | namespace morphie { 18 | namespace access { 19 | 20 | const char kRequiredFields[] = "fromx,tox,attr_count,attr_actor_title"; 21 | const char kActor[] = "fromx"; 22 | const char kActorManager[] = "attr_actor_manager"; 23 | const char kActorTitle[] = "attr_actor_title"; 24 | const char kNumAccesses[] = "attr_count"; 25 | const char kUser[] = "tox"; 26 | 27 | } // namespace access 28 | } // namespace morphie 29 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_defs.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains definitions of constants used for account access analysis. 16 | #ifndef LOGLE_ACCOUNT_ACCESS_DEFS_H_ 17 | #define LOGLE_ACCOUNT_ACCESS_DEFS_H_ 18 | 19 | namespace morphie { 20 | // The 'access' namespace contains definitions of fields used in access-oriented 21 | // analysis of access to email accounts. 22 | namespace access { 23 | 24 | // A comma-separated list of fields that must be present in the input. 25 | extern const char kRequiredFields[]; 26 | // The strings below are the names of the fields that provide information about 27 | // a single account access. 28 | // 29 | // The account used to perform an access. 30 | extern const char kActor[]; 31 | // The manager of the accessor. 32 | extern const char kActorManager[]; 33 | // The job title of the accessor (Software Engineer, Content Reviewer, etc.). 34 | extern const char kActorTitle[]; 35 | // The number of accesses. 36 | extern const char kNumAccesses[]; 37 | // The account that is accessed. 38 | extern const char kUser[]; 39 | 40 | } // namespace access 41 | } // namespace morphie 42 | #endif // LOGLE_ACCOUNT_ACCESS_DEFS_H_ 43 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_graph.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // An account access graph represents accesses to accounts by actors. There are 16 | // two types of nodes: actor nodes and user nodes. An actor node is labelled 17 | // with the actor's account, title and manager. A user node is labelled with a 18 | // user account. An access edge goes from an actor to a user and is labelled 19 | // with the number of accesses that were made. 20 | // 21 | // Status: The purpose of this graph is to demonstrate potential applications of 22 | // graph-based log analysis. There is currently only support for construction 23 | // and rudimentary visualization of this graph. 24 | #ifndef LOGLE_ACCOUNT_ACCESS_GRAPH_H_ 25 | #define LOGLE_ACCOUNT_ACCESS_GRAPH_H_ 26 | 27 | #include 28 | 29 | #include "base/string.h" 30 | #include "base/vector.h" 31 | #include "graph/graph_interface.h" 32 | #include "graph/labeled_graph.h" 33 | #include "util/status.h" 34 | 35 | namespace morphie { 36 | 37 | // The account access graph contains labels with the following names and types. 38 | // The labels are all unique, meaning there can be at most one node with each 39 | // label in the graph and at most one edge between two nodes with a given label. 40 | // * Actor : tuple(ActorId : string, Title : string, Manager : string) 41 | // * User : string 42 | // * Access Edge : Count : int 43 | // As currently implemented, if an account is an actor and a user, it will 44 | // appear as two nodes in the graph. 45 | // 46 | // The account access graph must be initialized explicitly by calling Initialize 47 | // before calling any other functions. Functions will crash if the graph is not 48 | // initialized. 49 | class AccountAccessGraph : public GraphInterface { 50 | public: 51 | AccountAccessGraph() : is_initialized_(false) {} 52 | 53 | // Initializes the graph. This function must be called before all other 54 | // functions in this class. Returns 55 | // - Status::OK - if a graph with the appropriate types was created. 56 | // - Status::INTERNAL - otherwise, with the reason accessible via the 57 | // Status::error_message() function of the Status object. 58 | util::Status Initialize(); 59 | 60 | // Functions for statistics about nodes and edges. 61 | // Statistics about nodes. 62 | int NumNodes() const; 63 | int NumLabeledNodes(const TaggedAST& label) const; 64 | // Statistics about edges. 65 | int NumEdges() const; 66 | int NumLabeledEdges(const TaggedAST& label) const; 67 | 68 | // Extract data from 'fields' and add nodes and edges to the graph for a 69 | // single access. The arguments are: 70 | // - 'fields' - a vector of fields derived from the input. 71 | // - 'field_index' - a map from field names to their position in 'fields'. 72 | // Requires that 73 | // * 'field_index' is not empty. 74 | // * 'fields' is not empty. 75 | // * for each pair (name, index) in 'field_index', 'index' is a valid index of 76 | // 'fields'. 77 | // This function does not perform data validation, meaning that every entry in 78 | // 'fields' is assumed to be valid. If, for example, an empty string is used 79 | // to represent an unknown user, there will be a user node in the graph 80 | // labelled with the empty string. 81 | void ProcessAccessData(const unordered_map& field_index, 82 | const std::vector& fields); 83 | 84 | // Return a representation of the graph in Graphviz DOT format. 85 | string ToDot() const; 86 | 87 | private: 88 | // The functions below create each of the three types of labels in the graph. 89 | TaggedAST MakeActorLabel(const unordered_map& field_index, 90 | const std::vector& fields); 91 | TaggedAST MakeUserLabel(const unordered_map& field_index, 92 | const std::vector& fields); 93 | TaggedAST MakeEdgeLabel(const unordered_map& field_index, 94 | const std::vector& fields); 95 | 96 | bool is_initialized_; 97 | LabeledGraph graph_; 98 | }; // class AccountAccessGraph 99 | 100 | } // namespace morphie 101 | 102 | #endif // LOGLE_ACCOUNT_ACCESS_GRAPH_H_ 103 | -------------------------------------------------------------------------------- /analyzers/examples/account_access_graph_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/account_access_graph.h" 16 | 17 | #include "analyzers/examples/account_access_defs.h" 18 | #include "gtest.h" 19 | 20 | namespace morphie { 21 | namespace { 22 | 23 | TEST(AccountAccessGraphDeathTest, UninitializedCall) { 24 | AccountAccessGraph graph; 25 | EXPECT_DEATH({ graph.ProcessAccessData(unordered_map(), {}); }, 26 | ".*"); 27 | } 28 | 29 | class AccountAccessGraphTest : public ::testing::Test { 30 | protected: 31 | void SetUp() override { 32 | util::Status s = graph_.Initialize(); 33 | EXPECT_TRUE(s.ok()); 34 | } 35 | 36 | AccountAccessGraph graph_; 37 | }; 38 | 39 | static std::vector fields1{"bad-person@logle", "manager-person@logle", 40 | "Bad Actor", "32", "user@fake-mail"}; 41 | 42 | static std::vector fields2{"good-person@logle", "manager-person@logle", 43 | "Good Actor", "32", "user@logle-mail"}; 44 | 45 | unordered_map GetIndex() { 46 | unordered_map index; 47 | index.insert({access::kActor, 0}); 48 | index.insert({access::kActorManager, 1}); 49 | index.insert({access::kActorTitle, 2}); 50 | index.insert({access::kNumAccesses, 3}); 51 | index.insert({access::kUser, 4}); 52 | return index; 53 | } 54 | 55 | TEST_F(AccountAccessGraphTest, ProcessSingleAccess) { 56 | EXPECT_EQ(0, graph_.NumNodes()); 57 | EXPECT_EQ(0, graph_.NumEdges()); 58 | graph_.ProcessAccessData(GetIndex(), fields1); 59 | EXPECT_EQ(2, graph_.NumNodes()); 60 | EXPECT_EQ(1, graph_.NumEdges()); 61 | // Processing the same data multiple times will not change the graph. 62 | graph_.ProcessAccessData(GetIndex(), fields1); 63 | EXPECT_EQ(2, graph_.NumNodes()); 64 | EXPECT_EQ(1, graph_.NumEdges()); 65 | // Changing the user but not the actor should add only one new node to the 66 | // graph. 67 | fields1[4] = "user2@fake-mail"; 68 | graph_.ProcessAccessData(GetIndex(), fields1); 69 | EXPECT_EQ(3, graph_.NumNodes()); 70 | EXPECT_EQ(2, graph_.NumEdges()); 71 | // Changing the actor but not the user should also only one new node to the 72 | // graph. 73 | fields1[0] = "another-actor@logle"; 74 | graph_.ProcessAccessData(GetIndex(), fields1); 75 | EXPECT_EQ(4, graph_.NumNodes()); 76 | EXPECT_EQ(3, graph_.NumEdges()); 77 | } 78 | 79 | } // namespace 80 | } // namespace morphie 81 | -------------------------------------------------------------------------------- /analyzers/examples/curio_analyzer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/curio_analyzer.h" 16 | 17 | #include 18 | 19 | #include "util/logging.h" 20 | #include "util/status.h" 21 | #include "util/string_utils.h" 22 | 23 | namespace { 24 | 25 | // The maximum number of malformed JSON objects allowed in the input. 26 | const int kMaxMalformedObjects = 1000000; 27 | 28 | // The clock is a special stream that is guaranteed to be present. 29 | const char kClockId[] = "[:clock]"; 30 | const char kNullDocErr[] = "The pointer to the document is null."; 31 | 32 | // Check if the dependency tree has the required JSON fields. 33 | bool HasRequiredFields(const Json::Value& json_tree) { 34 | return json_tree.isMember("Node") && json_tree["Node"].isMember("ID") && 35 | json_tree["Node"]["ID"].isMember("Name") && 36 | json_tree.isMember("Children"); 37 | } 38 | 39 | } // namespace 40 | 41 | namespace morphie { 42 | 43 | util::Status CurioAnalyzer::Initialize( 44 | std::unique_ptr<::Json::Value> json_doc) { 45 | if (json_doc == nullptr) { 46 | return util::Status(Code::INVALID_ARGUMENT, kNullDocErr); 47 | } 48 | json_doc_ = std::move(json_doc); 49 | if (json_doc_->empty()) { 50 | return util::Status(Code::INVALID_ARGUMENT, 51 | "The input document must not be empty."); 52 | } 53 | if (!json_doc_->isObject()) { 54 | return util::Status(Code::INVALID_ARGUMENT, 55 | "The document must be an object."); 56 | } 57 | num_streams_processed_ = 0; 58 | num_streams_skipped_ = 0; 59 | return util::Status::OK; 60 | } 61 | 62 | util::Status CurioAnalyzer::BuildDependencyGraph() { 63 | CHECK(json_doc_ != nullptr, kNullDocErr); 64 | dependency_graph_.reset(new StreamDependencyGraph); 65 | util::Status status = dependency_graph_->Initialize(); 66 | if (!status.ok()) { 67 | return status; 68 | } 69 | for (const auto& consumer_id : json_doc_->getMemberNames()) { 70 | // Every stream depends on the clock so the clock stream is not added to the 71 | // dependency graph to reduce structural and visual noise. 72 | if (consumer_id == kClockId) { 73 | continue; 74 | } 75 | if (!HasRequiredFields((*json_doc_)[consumer_id])) { 76 | status = IncrementSkipCounter(); 77 | if (!status.ok()) { 78 | return status; 79 | } 80 | continue; 81 | } 82 | status = AddDependencies(consumer_id, (*json_doc_)[consumer_id]); 83 | ++num_streams_processed_; 84 | if (!status.ok()) { 85 | return status; 86 | } 87 | } 88 | return util::Status::OK; 89 | } 90 | 91 | int CurioAnalyzer::NumGraphNodes() const { 92 | return dependency_graph_ == nullptr ? 0 : dependency_graph_->NumNodes(); 93 | } 94 | 95 | int CurioAnalyzer::NumGraphEdges() const { 96 | return dependency_graph_ == nullptr ? 0 : dependency_graph_->NumEdges(); 97 | } 98 | 99 | // Recursively traverse the dependency tree rooted at 'consumer_tree'. The 100 | // traversal skips a subtree below a node if that node is not well defined. 101 | util::Status CurioAnalyzer::AddDependencies(const string& consumer_id, 102 | const Json::Value& consumer_tree) { 103 | util::Status status = util::Status::OK; 104 | string consumer_name = consumer_tree["Node"]["ID"]["Name"].asString(); 105 | Json::Value producer_tree; 106 | for (const auto& producer_id : consumer_tree["Children"].getMemberNames()) { 107 | // Every stream depends on the clock the clock stream is not added to the 108 | // dependency graph to reduce structural and visual noise. 109 | if (producer_id == kClockId) { 110 | continue; 111 | } 112 | producer_tree = consumer_tree["Children"][producer_id]; 113 | if (!HasRequiredFields(producer_tree)) { 114 | status = IncrementSkipCounter(); 115 | if (!status.ok()) { 116 | return status; 117 | } 118 | continue; 119 | } 120 | string producer_name = producer_tree["Node"]["ID"]["Name"].asString(); 121 | dependency_graph_->AddDependency(consumer_id, consumer_name, producer_id, 122 | producer_name); 123 | status = AddDependencies(producer_id, producer_tree); 124 | ++num_streams_processed_; 125 | if (!status.ok()) { 126 | return status; 127 | } 128 | } 129 | return status; 130 | } 131 | 132 | string CurioAnalyzer::DependencyGraphAsDot() const { 133 | return dependency_graph_ == nullptr ? "" : dependency_graph_->ToDot(); 134 | } 135 | 136 | util::Status CurioAnalyzer::IncrementSkipCounter() { 137 | ++num_streams_skipped_; 138 | if (num_streams_skipped_ >= kMaxMalformedObjects) { 139 | return util::Status( 140 | Code::INVALID_ARGUMENT, 141 | util::StrCat("Over ", std::to_string(kMaxMalformedObjects), 142 | " malformed JSON objects in input. Aborting.")); 143 | } 144 | return util::Status::OK; 145 | } 146 | 147 | } // namespace morphie 148 | -------------------------------------------------------------------------------- /analyzers/examples/curio_analyzer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // An analyzer for Curio streams. The analyzer uses graphs to represent 16 | // relationships between Curio streams. These graphs can then be used for 17 | // analysis and visualization. 18 | #ifndef LOGLE_CURIO_ANALYZER_H_ 19 | #define LOGLE_CURIO_ANALYZER_H_ 20 | 21 | #include 22 | 23 | #include "analyzers/examples/stream_dependency_graph.h" 24 | #include "base/string.h" 25 | #include "json/json.h" 26 | #include "util/status.h" 27 | 28 | namespace morphie { 29 | 30 | // The CurioAnalyzer ingests stream definitions provided in JSON and calls the 31 | // appropriate APIs for graph construction and visualization. Initialize() must 32 | // be called before any other function is called or those calls will crash. 33 | class CurioAnalyzer { 34 | public: 35 | CurioAnalyzer() : num_streams_processed_(0), num_streams_skipped_(0) {} 36 | 37 | // Initializes the analyzer with a JSON document and resets the counters 38 | // that track the number of streams processed and skipped. 39 | // * Returns OK if 'json_doc' satisfies these conditions: 40 | // - It must not be null. 41 | // - It must not be an empty JSON document. 42 | // - It must be an object. 43 | // * Returns INVALID_ARGUMENT otherwise with an error message. 44 | util::Status Initialize(std::unique_ptr<::Json::Value> json_doc); 45 | 46 | // Constructs a StreamDependencyGraph object from the JSON document provided 47 | // to Initialize. 48 | util::Status BuildDependencyGraph(); 49 | 50 | int NumStreamsProcessed() const { return num_streams_processed_; } 51 | int NumStreamsSkipped() const { return num_streams_skipped_; } 52 | 53 | int NumGraphNodes() const; 54 | int NumGraphEdges() const; 55 | 56 | // Returns a GraphViz DOT representation of the dependency graph. 57 | string DependencyGraphAsDot() const; 58 | 59 | private: 60 | // Recursively adds nodes and edges to the dependency graph for each stream in 61 | // 'consumer_tree'. The 'consumer_tree' is assumed to contain only one JSON 62 | // object at the top level. Returns: 63 | // - INVALID_ARGUMENT : if the number of malformed stream definitions exceeds 64 | // a predefined threshold (kMaxMalformedObjects in curio_analyzer.cc). 65 | // - OK : otherwise. 66 | util::Status AddDependencies(const string& consumer_id, 67 | const Json::Value& consumer_tree); 68 | 69 | // Increments a global counter of the number of objects in the JSON input that 70 | // have been skipped. Returns 71 | // - INVALID_ARGUMENT if the result of the increment exceeds the threshold 72 | // defined by kMaxMalformedObjects in curio_analyzer.cc. 73 | // - OK otherwise. 74 | util::Status IncrementSkipCounter(); 75 | 76 | int num_streams_processed_; 77 | int num_streams_skipped_; 78 | 79 | std::unique_ptr json_doc_; 80 | std::unique_ptr dependency_graph_; 81 | }; 82 | 83 | } // namespace morphie 84 | 85 | #endif // LOGLE_CURIO_ANALYZER_H_ 86 | -------------------------------------------------------------------------------- /analyzers/examples/curio_defs.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/curio_defs.h" 16 | 17 | namespace morphie { 18 | namespace curio { 19 | 20 | const char kChildrenField[] = "Children"; 21 | 22 | const char kStreamTag[] = "Stream"; 23 | const char kStreamIdTag[] = "Stream Id"; 24 | const char kStreamNameTag[] = "Stream Name"; 25 | const char kDependentTag[] = "Dependent"; 26 | 27 | } // namespace curio 28 | } // namespace morphie 29 | -------------------------------------------------------------------------------- /analyzers/examples/curio_defs.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains constants used in the analysis of Curio streams. 16 | #ifndef LOGLE_CURIO_DEFS_H_ 17 | #define LOGLE_CURIO_DEFS_H_ 18 | 19 | namespace morphie { 20 | // The 'curio' namespace encloses definitions used to analyze Curio streams. 21 | namespace curio { 22 | 23 | // The field name for children of a stream. 24 | extern const char kChildrenField[]; 25 | 26 | // Tags used in graph labels. 27 | extern const char kStreamTag[]; 28 | extern const char kStreamIdTag[]; 29 | extern const char kStreamNameTag[]; 30 | extern const char kDependentTag[]; 31 | 32 | } // namespace curio 33 | } // namespace morphie 34 | 35 | #endif // LOGLE_CURIO_DEFS_H_ 36 | -------------------------------------------------------------------------------- /analyzers/examples/stream_dependency_graph.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/stream_dependency_graph.h" 16 | 17 | #include 18 | #include 19 | 20 | #include "analyzers/examples/curio_defs.h" 21 | #include "base/vector.h" 22 | #include "graph/dot_printer.h" 23 | #include "graph/type.h" 24 | #include "graph/type_checker.h" 25 | #include "graph/value.h" 26 | #include "ast.pb.h" 27 | #include "util/logging.h" 28 | #include "util/string_utils.h" 29 | 30 | namespace morphie { 31 | 32 | namespace type = ast::type; 33 | namespace value = ast::value; 34 | 35 | namespace { 36 | 37 | const char kInitializationErr[] = "The graph is not initialized."; 38 | 39 | // Returns a tagged abstract syntax tree (AST) in which the tag is 40 | // curio::kStreamTag, and the AST represents the tuple of strings 41 | // (node_id, node_name). 42 | // The AST is created using the factory function MakeNullTuple from value.h. The 43 | // fields of the tuple are set using SetField. The argument 'type' must be a 44 | // pair of strings because SetField guarantees that the field update respects 45 | // the type of the tuple. 46 | TaggedAST MakeNodeLabel(const AST& type, const string& node_id, 47 | const string& node_name) { 48 | TaggedAST label; 49 | label.set_tag(curio::kStreamTag); 50 | AST* ast = label.mutable_ast(); 51 | *ast = value::MakeNullTuple(2); 52 | value::SetField(type, 0, value::MakeString(node_id), ast); 53 | value::SetField(type, 1, value::MakeString(node_name), ast); 54 | return label; 55 | } 56 | 57 | } // namespace 58 | 59 | util::Status StreamDependencyGraph::Initialize() { 60 | // Create a unique node label of type tuple(string, string) for each stream. 61 | std::vector args; 62 | args.emplace_back( 63 | type::MakeString(curio::kStreamIdTag, false /*May not be null*/)); 64 | args.emplace_back( 65 | type::MakeString(curio::kStreamNameTag, true /*May be null*/)); 66 | type::Types node_types; 67 | node_types.emplace( 68 | curio::kStreamTag, 69 | type::MakeTuple(curio::kStreamTag, false /*May not be null*/, args)); 70 | std::set unique_nodes = {curio::kStreamTag}; 71 | type::Types edge_types; 72 | edge_types.emplace(curio::kDependentTag, 73 | type::MakeNull(curio::kDependentTag)); 74 | std::set unique_edges = {curio::kDependentTag}; 75 | // There is no graph-level label. 76 | AST graph_type = type::MakeNull(curio::kStreamTag); 77 | 78 | util::Status s = graph_.Initialize(node_types, unique_nodes, edge_types, 79 | unique_edges, graph_type); 80 | if (!s.ok()) { 81 | return util::Status(Code::INTERNAL, s.message()); 82 | } 83 | is_initialized_ = true; 84 | return s; 85 | } 86 | 87 | int StreamDependencyGraph::NumNodes() const { 88 | CHECK(is_initialized_, kInitializationErr); 89 | return graph_.NumNodes(); 90 | } 91 | 92 | int StreamDependencyGraph::NumEdges() const { 93 | CHECK(is_initialized_, kInitializationErr); 94 | return graph_.NumEdges(); 95 | } 96 | 97 | void StreamDependencyGraph::AddDependency(const string& consumer_id, 98 | const string& consumer_name, 99 | const string& producer_id, 100 | const string& producer_name) { 101 | CHECK(is_initialized_, kInitializationErr); 102 | std::pair type = graph_.GetNodeType(curio::kStreamTag); 103 | TaggedAST consumer = MakeNodeLabel(type.second, consumer_id, consumer_name); 104 | TaggedAST producer = MakeNodeLabel(type.second, producer_id, producer_name); 105 | NodeId consumer_node = graph_.FindOrAddNode(consumer); 106 | NodeId producer_node = graph_.FindOrAddNode(producer); 107 | TaggedAST edge_label; 108 | edge_label.set_tag(curio::kDependentTag); 109 | *edge_label.mutable_ast() = value::MakeNull(); 110 | graph_.FindOrAddEdge(consumer_node, producer_node, edge_label); 111 | } 112 | 113 | string StreamDependencyGraph::ToDot() const { 114 | CHECK(is_initialized_, kInitializationErr); 115 | AttributeFn node_attribute = [](const string& tag, const AST& ast) { 116 | return DotPrinter::NodeAttribute(tag, ast.c_ast().arg(1)); 117 | }; 118 | AttributeFn edge_attribute = DotPrinter::EdgeAttribute; 119 | 120 | DotPrinter dot_printer(node_attribute, edge_attribute); 121 | string dot_graph = dot_printer.AllNodesInDot(graph_); 122 | util::StrAppend(&dot_graph, dot_printer.AllEdgesInDot(graph_), "\n"); 123 | return util::StrCat("digraph stream_dependencies {\n", dot_graph, "}"); 124 | } 125 | 126 | } // namespace morphie 127 | -------------------------------------------------------------------------------- /analyzers/examples/stream_dependency_graph.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // A Curio stream definition determines how data will be processed to generate a 16 | // stream (which is a Dremel table). A stream 't' depends on a stream 's' if the 17 | // definition of 't' uses the stream 's'. Dependence is determined by stream 18 | // definitions and need not be transitive. For example, if 't' is defined in 19 | // terms of 's' and 's' is defined in terms of 'r', and the definition of 't' 20 | // does not explicitly involve 'r', then 't' does not depend on 'r'. 21 | // 22 | // A stream dependency graph represents dependencies between stream definitions. 23 | // If 't' depends on 's', then 's' is a producer and 't' is a consumer of data. 24 | #ifndef LOGLE_STREAM_DEPENDENCY_GRAPH_H_ 25 | #define LOGLE_STREAM_DEPENDENCY_GRAPH_H_ 26 | 27 | #include "base/string.h" 28 | #include "graph/graph_interface.h" 29 | #include "graph/labeled_graph.h" 30 | #include "util/status.h" 31 | 32 | namespace morphie { 33 | 34 | // The StreamDependencyGraph class implements a stream dependency graph. The 35 | // nodes in the graph are labels represented as abstract syntax trees (see 36 | // ast.proto). Each node label is a pair of strings with the first string 37 | // representing the stream identifier and the second string representing the 38 | // stream name. There are no edge labels. 39 | // 40 | // The function Initialize() must be called before other functions are called. 41 | // The functions will crash otherwise. 42 | class StreamDependencyGraph : public GraphInterface { 43 | public: 44 | StreamDependencyGraph() : is_initialized_(false) {} 45 | 46 | // Initialize the graph. Returns 47 | // - Status::OK - if a graph with the appropriate node and edge types has 48 | // been created. 49 | // - Status::INTERNAL - otherwise, with the reason accessible via the 50 | // Status::error_message() function of the Status object. 51 | util::Status Initialize(); 52 | 53 | int NumNodes() const; 54 | int NumEdges() const; 55 | 56 | // Adds an edge for a dependency of consumer with id 'consumer_id' and name 57 | // 'consumer_name' on producer with id 'producer_id' and name 'producer_name'. 58 | // Creates nodes for consumer and producer if they do not already exist. 59 | void AddDependency(const string& consumer_id, const string& consumer_name, 60 | const string& producer_id, const string& producer_name); 61 | 62 | // Return a representation of the graph in Graphviz DOT format. 63 | string ToDot() const; 64 | 65 | private: 66 | // This variable is set to false by the constructor and is set to 'true' if 67 | // Initialize() completes successfully. It is not modified thereafter. 68 | bool is_initialized_; 69 | 70 | LabeledGraph graph_; 71 | }; 72 | 73 | } // namespace morphie 74 | #endif // LOGLE_STREAM_DEPENDENCY_GRAPH_H_ 75 | -------------------------------------------------------------------------------- /analyzers/examples/stream_dependency_graph_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/examples/stream_dependency_graph.h" 16 | 17 | #include "base/vector.h" 18 | 19 | #include "base/string.h" 20 | #include "gtest.h" 21 | 22 | namespace morphie { 23 | namespace { 24 | 25 | TEST(PlasoEventGraphDeathTest, UninitializedCalls) { 26 | StreamDependencyGraph graph; 27 | const char kInitializationRegEx[] = ".*not initialized.*"; 28 | EXPECT_DEATH({ graph.NumNodes(); }, kInitializationRegEx); 29 | EXPECT_DEATH({ graph.NumEdges(); }, kInitializationRegEx); 30 | EXPECT_DEATH({ graph.AddDependency("", "", "", ""); }, kInitializationRegEx); 31 | EXPECT_DEATH({ graph.ToDot(); }, kInitializationRegEx); 32 | } 33 | 34 | static const std::vector> streams = { 35 | {"[/path/to/stream:stream1]", "stream1"}, 36 | {"[/path/to/stream:stream2]", "stream2"}, 37 | {"[/path/to/stream:stream3]", "stream3"}, 38 | }; 39 | 40 | // Construct the dependency graph below and check that the number of nodes and 41 | // edges is as expected at each stage of the construction. 42 | // 43 | // stream3 44 | // ^ ^ 45 | // | | 46 | // stream1 --> stream2 47 | TEST(PlasoEventGraphTest, AddDependency) { 48 | StreamDependencyGraph graph; 49 | EXPECT_TRUE(graph.Initialize().ok()); 50 | graph.AddDependency(streams[0].first, streams[0].second, streams[1].first, 51 | streams[1].second); 52 | EXPECT_EQ(2, graph.NumNodes()); 53 | EXPECT_EQ(1, graph.NumEdges()); 54 | graph.AddDependency(streams[0].first, streams[0].second, streams[2].first, 55 | streams[2].second); 56 | EXPECT_EQ(3, graph.NumNodes()); 57 | EXPECT_EQ(2, graph.NumEdges()); 58 | graph.AddDependency(streams[1].first, streams[1].second, streams[2].first, 59 | streams[2].second); 60 | EXPECT_EQ(3, graph.NumNodes()); 61 | EXPECT_EQ(3, graph.NumEdges()); 62 | // Test that there can be at most one dependency edge between two nodes. 63 | graph.AddDependency(streams[1].first, streams[1].second, streams[2].first, 64 | streams[2].second); 65 | EXPECT_EQ(3, graph.NumNodes()); 66 | EXPECT_EQ(3, graph.NumEdges()); 67 | } 68 | 69 | } // namespace 70 | } // namespace morphie 71 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_analyzer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/plaso/plaso_analyzer.h" 16 | 17 | #include // NOLINT 18 | 19 | #include 20 | #include 21 | 22 | #include "analyzers/plaso/plaso_defs.h" 23 | #include "analyzers/plaso/plaso_event.h" 24 | #include "base/vector.h" 25 | #include "util/json_reader.h" 26 | #include "util/logging.h" 27 | #include "util/status.h" 28 | #include "util/string_utils.h" 29 | 30 | namespace { 31 | 32 | const int kMaxMalformedLines = 1000000; 33 | 34 | } // namespace 35 | 36 | namespace morphie { 37 | 38 | util::Status PlasoAnalyzer::Initialize( 39 | JsonDocumentIterator* doc_iterator) { 40 | CHECK(doc_iterator != nullptr, "The pointer to the JSON document is null."); 41 | this-> doc_iterator_ = doc_iterator; 42 | return util::Status::OK; 43 | } 44 | 45 | void PlasoAnalyzer::BuildPlasoGraph() { 46 | plaso_graph_.reset(new PlasoEventGraph(show_all_sources_)); 47 | if (!plaso_graph_->Initialize().ok()) { 48 | plaso_graph_.reset(nullptr); 49 | return; 50 | } 51 | return BuildPlasoGraphFromJSON(); 52 | } 53 | 54 | string PlasoAnalyzer::PlasoGraphDot() const { 55 | return (plaso_graph_ == nullptr) ? "" : plaso_graph_->ToDot(); 56 | } 57 | 58 | string PlasoAnalyzer::PlasoGraphPbTxt() const { 59 | return (plaso_graph_ == nullptr) ? "" : plaso_graph_->ToPbTxt(); 60 | } 61 | 62 | string PlasoAnalyzer::PlasoGraphStats() const { 63 | if (plaso_graph_ == nullptr) { 64 | return "Graph has not been created!"; 65 | } 66 | return plaso_graph_->GetStats(); 67 | } 68 | 69 | void PlasoAnalyzer::IncrementSkipCounter() { 70 | ++num_lines_skipped_; 71 | CHECK(num_lines_skipped_ < kMaxMalformedLines, 72 | "Over a million malformed lines in input. Aborting."); 73 | } 74 | 75 | void PlasoAnalyzer::BuildPlasoGraphFromJSON() { 76 | const std::set required_fields = 77 | util::SplitToSet(plaso::kRequiredFields, ','); 78 | CHECK(!required_fields.empty(), "No required fields in input."); 79 | // List of all event names. 80 | // This variable will point to the data for a single event. 81 | const Json::Value* json_event; 82 | // This proto will contain fields extracted from '*json_event'. 83 | PlasoEvent event_data; 84 | bool has_all_fields; 85 | 86 | while (this->doc_iterator_->HasNext()) { 87 | json_event = this->doc_iterator_->Next(); 88 | CHECK(json_event != nullptr, "json_event is null!"); 89 | has_all_fields = std::all_of(required_fields.begin(), required_fields.end(), 90 | [json_event](const string& field) { 91 | return json_event->isMember(field); 92 | }); 93 | if (!has_all_fields) { 94 | IncrementSkipCounter(); 95 | continue; 96 | } 97 | event_data = plaso::ParseJSON(*json_event); 98 | plaso_graph_->ProcessEvent(event_data); 99 | } 100 | plaso_graph_->AddTemporalEdges(); 101 | } 102 | 103 | } // namespace morphie 104 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_analyzer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // An analyzer for Plaso logs. Plaso is a forensic tool that generates 16 | // semi-structured output called a super-timeline, which consists of a 17 | // timestamped sequence of events. This analyzer operates on a supertimeline 18 | // represented as a JSON document. The output of the analysis is either a 19 | // visualization of log data or the results of an analysis. 20 | #ifndef LOGLE_PLASO_ANALYZER_H_ 21 | #define LOGLE_PLASO_ANALYZER_H_ 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "analyzers/plaso/plaso_event_graph.h" 28 | #include "base/string.h" 29 | #include "json/json.h" 30 | #include "util/json_reader.h" 31 | #include "util/status.h" 32 | 33 | namespace morphie { 34 | 35 | // The PlasoAnalyzer uses graphs to extract information from log data. The 36 | // analyzer must be initialized with a JSON document (a Json::Value object from 37 | // JSONCpp). The document must satisfy constraints described in the comments 38 | // above the Initialize function. 39 | class PlasoAnalyzer { 40 | public: 41 | explicit PlasoAnalyzer(bool show_all_sources) 42 | : show_all_sources_(show_all_sources), 43 | num_lines_read_(0), 44 | num_lines_skipped_(0) {} 45 | 46 | // Initializes the log analyzer with a JSON document. 47 | // * Requires that 'json_doc' is not null. 48 | // * Returns OK if the JSON document is an object containing key-value pairs. 49 | // * Returns INVALID_ARGUMENT otherwise. 50 | // The document structure is defined by the data source not by logle. The keys 51 | // in the object specify an event identifier and the value is another object 52 | // containing event data. Additional error validation is done 53 | // during graph construction. 54 | util::Status Initialize(JsonDocumentIterator* json_doc); 55 | 56 | // Constructs a PlasoEventGraph (defined in plaso_event_graph.h) from the 57 | // input data. Requires that the analyzer has been initialized and that every 58 | // object in the JSON input contains the fields listed in the documentation of 59 | // the Initialize function above. 60 | void BuildPlasoGraph(); 61 | 62 | // Utilities for accounting and error checking. 63 | int NumLinesRead() { return num_lines_read_; } 64 | int NumLinesSkipped() { return num_lines_skipped_; } 65 | int NumLinesProcessed() { return num_lines_read_ - num_lines_skipped_; } 66 | 67 | int NumNodes() { 68 | return (plaso_graph_ == nullptr) ? 0 : plaso_graph_->NumNodes(); 69 | } 70 | 71 | int NumEdges() { 72 | return (plaso_graph_ == nullptr) ? 0 : plaso_graph_->NumEdges(); 73 | } 74 | 75 | string PlasoGraphStats() const; 76 | string PlasoGraphDot() const; 77 | string PlasoGraphPbTxt() const; 78 | 79 | private: 80 | // Constructs a Plaso graph using a JSON document. 81 | void BuildPlasoGraphFromJSON(); 82 | // The skip counter tracks the number of the serialized event objects in the 83 | // input that were skipped. 84 | void IncrementSkipCounter(); 85 | 86 | // Configuration options for the analyzer. 87 | bool show_all_sources_; 88 | 89 | // Data about analyzer state. 90 | std::unique_ptr plaso_graph_; 91 | int num_lines_read_; 92 | int num_lines_skipped_; 93 | JsonDocumentIterator* doc_iterator_; 94 | }; 95 | 96 | } // namespace morphie 97 | 98 | #endif // LOGLE_PLASO_ANALYZER_H_ 99 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_analyzer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/plaso/plaso_analyzer.h" 16 | 17 | #include 18 | #include "base/vector.h" 19 | 20 | #include "analyzers/plaso/plaso_defs.h" 21 | #include "base/string.h" 22 | #include "gtest.h" 23 | #include "util/json_reader.h" 24 | #include "util/string_utils.h" 25 | 26 | namespace morphie { 27 | namespace { 28 | namespace util = morphie::util; 29 | 30 | // Tests of JSON input validation and input processing. 31 | // NOLINTNEXTLINE 32 | string json_object = R"({"event1":{"data_type": "fs:stat", "display_name": "GZIP:/usr/share/info/bc.info.gz", "timestamp": 0, "timestamp_desc": "mtime" } 33 | , "event2":{"display_name": "GZIP:/usr/share/info/bc.info.gz", "data_type": "fs:stat", "timestamp": 0, "timestamp_desc": "mtime" } 34 | , "event3":{"display_name": "GZIP:/usr/share/info/coreutils.info.gz", "data_type": "fs:stat", "timestamp": 0, "timestamp_desc": "mtime"} 35 | })"; 36 | 37 | // NOLINTNEXTLINE 38 | string json_stream = R"({"data_type": "fs:stat", "display_name": "GZIP:/usr/share/info/bc.info.gz", "timestamp": 0, "timestamp_desc": "mtime" } 39 | {"display_name": "GZIP:/usr/share/info/bc.info.gz", "data_type": "fs:stat", "timestamp": 0, "timestamp_desc": "mtime" } 40 | {"display_name": "GZIP:/usr/share/info/coreutils.info.gz", "data_type": "fs:stat", "timestamp": 0, "timestamp_desc": "mtime"})"; 41 | 42 | void TestInitialization( 43 | const string& file_content, bool is_line_json) { 44 | PlasoAnalyzer analyzer(false); 45 | istringstream stream(file_content); 46 | util::Status status; 47 | 48 | if (is_line_json) { 49 | morphie::StreamJson jstream(&stream); 50 | status = analyzer.Initialize(&jstream); 51 | analyzer.BuildPlasoGraph(); 52 | } else { 53 | morphie::FullJson jstream(&stream); 54 | status = analyzer.Initialize(&jstream); 55 | analyzer.BuildPlasoGraph(); 56 | } 57 | analyzer.PlasoGraphDot(); 58 | } 59 | 60 | // Basic testing for correct JSON input files. 61 | TEST(PlasoAnalyzerTest, AcceptValidJSONInput) { 62 | TestInitialization(json_object, false); 63 | TestInitialization(json_stream, true); 64 | } 65 | 66 | // Basic testing for incorrect JSON input files. 67 | TEST(PlasoAnalyzerDeathTest, RequiresCorrectJSONDoc) { 68 | std::unique_ptr<::Json::Value> doc; 69 | PlasoAnalyzer analyzer(false); 70 | EXPECT_DEATH(TestInitialization(json_object, true), 71 | ".*Line is not in JSON format.*"); 72 | EXPECT_DEATH(TestInitialization(json_stream, false), 73 | ".*JSON*"); 74 | } 75 | } // namespace 76 | } // namespace morphie 77 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_defs.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/plaso/plaso_defs.h" 16 | 17 | namespace morphie { 18 | namespace plaso { 19 | 20 | const char kRequiredFields[] = 21 | "data_type,display_name,timestamp,timestamp_desc"; 22 | 23 | const char kDataTypeName[] = "data_type"; 24 | const char kDescriptionName[] = "timestamp_desc"; 25 | const char kMessageName[] = "message"; 26 | const char kSourceFileName[] = "display_name"; 27 | const char kTimestampName[] = "timestamp"; 28 | 29 | } // namespace plaso 30 | } // namespace morphie 31 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_defs.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains definitions of constants specific to Plaso output. 16 | #ifndef LOGLE_PLASO_DEFS_H_ 17 | #define LOGLE_PLASO_DEFS_H_ 18 | 19 | namespace morphie { 20 | namespace plaso { 21 | 22 | // A comma-separated list of fields that must be present in the input. 23 | extern const char kRequiredFields[]; 24 | 25 | // The strings below are names of fields associated with a specific event. 26 | // A Plaso-internal canonical descriptor of the format of the event data. 27 | extern const char kDataTypeName[]; 28 | // A human-readable description of the event. 29 | extern const char kDescriptionName[]; 30 | // Details about an event. 31 | extern const char kMessageName[]; 32 | // File from which the event was reconstructed by Plaso. 33 | extern const char kSourceFileName[]; 34 | // An RFC3339 timestamp representing the time at which the event occurred. 35 | extern const char kTimestampName[]; 36 | 37 | } // namespace plaso 38 | } // namespace morphie 39 | 40 | #endif // LOGLE_PLASO_DEFS_H_ 41 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_event.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // A PlasoEvent proto (defined in analyzers/plaso/plaso_event.proto) represents 16 | // data extracted from the output of Plaso (go/plaso). This file defines 17 | // utilities for parsing input and populating PlasoEvent protos. 18 | #ifndef LOGLE_PLASO_EVENT_H_ 19 | #define LOGLE_PLASO_EVENT_H_ 20 | 21 | #include "base/string.h" 22 | #include "json/json.h" 23 | #include "plaso_event.pb.h" 24 | #include "ast.pb.h" 25 | 26 | namespace morphie { 27 | // The 'plaso' namespace contains utilities for dealing with Plaso output. 28 | namespace plaso { 29 | 30 | // Converts a Unix-style Plaso filename into a File proto. This is not a generic 31 | // filename parsing utility. Plaso normalizes filenames so that even a Windows 32 | // filename is of the form C:/path/to/file.ext. This function also assumes that 33 | // - the file delimiter is '/'. 34 | // - a filename cannot be the empty string. 35 | // Examples. 36 | // - "" generates no filename and no directory. 37 | // - "/" generates no filename and the empty string as the directory. 38 | // - "filename.txt" generates a filename with no directory. 39 | // - "/usr/local/" generates the directory with path "usr", "local" but no 40 | // filename. 41 | // - "/foo/bar/baz.f" generates "foo", "bar" as 'directory().path()' and 42 | // "baz.f" as the filename. 43 | File ParseFilename(const string& filename); 44 | 45 | // Constructs an event proto from a JSON object generated by Timesketch. 46 | PlasoEvent ParseJSON(const ::Json::Value& json_event); 47 | 48 | // Return a PlasoEventGraph AST representing a file. 49 | AST ToAST(const File& file); 50 | 51 | // Return a string with the full path and filename of the file in 'file'. 52 | string ToString(const File& file); 53 | 54 | } // namespace plaso 55 | } // namespace morphie 56 | 57 | #endif // LOGLE_PLASO_EVENT_H_ 58 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_event.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | syntax = "proto2"; 16 | 17 | package morphie; 18 | 19 | // Plaso (go/plaso) is a forensic analysis tool that extracts timed events from 20 | // forensic artefacts. This file defines the PlasoEvent proto, which stores 21 | // attributes about events. This proto is the main data structure for 22 | // transforming data generated by Plaso into data used by logle for graph 23 | // analysis. The PlasoEvent proto is based on the EventObject message in 24 | // plaso/proto/plaso_storage.proto but reduces the amount of processing required 25 | // to extract data for graph analysis. 26 | 27 | // This enum defines a set of event types. The event types are not independent. 28 | // For example a FILE_CONTENT_MODIFIED event is also a FILE_METADATA_MODIFIED 29 | // event but metadata can be modified without content being modified. The event 30 | // types are designed such that there is exactly one type for each event 31 | // generated by Plaso. See plaso_event.cc for a description of when DEFAULT is 32 | // used. 33 | enum EventType { 34 | SKIP = 1; 35 | DEFAULT = 2; 36 | FILE_CREATED = 3; 37 | FILE_METADATA_MODIFIED = 4; 38 | FILE_CONTENT_MODIFIED = 5; 39 | FILE_ACCESSED = 6; 40 | PAGE_VISITED = 7; 41 | FILE_DOWNLOADED = 8; 42 | BROWSER_EXTENSION_INSTALLED = 9; 43 | APPLICATION_EXECUTED = 10; 44 | } 45 | 46 | // A directory is represented by the sequence of subdirectories on a path. 47 | // Examples 1: 48 | // - / is represented by the empty string. 49 | // - /mnt is represented by the string "mnt". 50 | // - /usr/local/bin is represented by the sequence "usr", "local", "bin". 51 | // A path of size zero represents that no directory is known. 52 | message Directory { 53 | repeated string path = 1; 54 | } 55 | 56 | // A file is represented by a filename and the directory containing the file. 57 | // If directory information is not available, the 'directory' field will be 58 | // absent. The same applies to the filename. Empty strings should not be used to 59 | // represent unknown filenames. 60 | // Examples 2: 61 | // - abc.txt has no directory and 'filename' as "abc.txt". 62 | // - /Users/MrWhite/blacklist will have 'filename' as "blacklist" and 63 | // 'directory' as the sequence "Users", "MrWhite". 64 | message File { 65 | optional Directory directory = 1; 66 | optional string filename = 2; 67 | } 68 | 69 | // A PlasoEvent message consists of data extracted from Plaso output and an 70 | // event type generated by logle. All fields except the event type are optional. 71 | message PlasoEvent { 72 | // Event timestamp if available. 73 | optional int64 timestamp = 1; 74 | 75 | required EventType type = 2 [default = DEFAULT]; 76 | 77 | // A natural language description that Plaso generates for the event. Example 78 | // descriptions are "Metadata Modification Time" or "Last Login Time". The set 79 | // of descriptions is defined in plaso/lib/eventdata.py and output to CSV in 80 | // the 'timestamp_desc' field. 81 | optional string desc = 3; 82 | 83 | // The file from which Plaso extracted this event. This file need not be 84 | // explicitly modified by an event. For example, Plaso uses the Apple System 85 | // Log to detect if a file was downloaded on a OS X system. In a file download 86 | // event, the Apple System Log is the event source file, but the result of the 87 | // download is a different file. In MAC time events, the event data file is 88 | // the file modified in the event. 89 | optional File event_source_file = 4; 90 | optional string event_id = 5; 91 | 92 | // URLs and files in an event can be sources or targets of data or actions. 93 | // For example, in a file download event, there is a source URL and a target 94 | // file, while a file upload event has a source file and a target URL. 95 | optional string source_url = 6; 96 | optional string target_url = 7; 97 | 98 | optional File source_file = 8; 99 | optional File target_file = 9; 100 | 101 | // Information about browser extensions. 102 | optional string extension_name = 10; 103 | optional string extension_id = 11; 104 | 105 | // The name and version of an application. The version is a string 106 | // to allow for different represenations of that information. 107 | optional string application_name = 12; 108 | optional string application_version = 13; 109 | } 110 | -------------------------------------------------------------------------------- /analyzers/plaso/plaso_event_graph_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "analyzers/plaso/plaso_event_graph.h" 16 | 17 | #include // for __alloc_traits<>::value_type 18 | 19 | #include "analyzers/plaso/plaso_event.h" 20 | #include "graph/value.h" 21 | #include "gtest.h" 22 | #include "plaso_event.pb.h" 23 | #include "util/status.h" 24 | #include "util/time_utils.h" 25 | 26 | namespace morphie { 27 | namespace { 28 | 29 | namespace value = ast::value; 30 | 31 | TEST(PlasoEventGraphDeathTest, UninitializedAPICalls) { 32 | PlasoEventGraph graph(false); 33 | EXPECT_DEATH({ graph.NumNodes(); }, ".*"); 34 | EXPECT_DEATH({ graph.NumEdges(); }, ".*"); 35 | PlasoEvent event_data; 36 | EXPECT_DEATH({ graph.ProcessEvent(event_data); }, ".*"); 37 | EXPECT_DEATH({ graph.AddTemporalEdges(); }, ".*"); 38 | } 39 | 40 | class PlasoEventGraphTest : public ::testing::Test { 41 | protected: 42 | PlasoEventGraphTest() : graph_(false) {} 43 | 44 | void SetUp() override { 45 | util::Status s = graph_.Initialize(); 46 | EXPECT_TRUE(s.ok()); 47 | } 48 | 49 | PlasoEventGraph graph_; 50 | }; 51 | 52 | // Returns a PlasoEvent proto with the timestamp and description fields set. 53 | PlasoEvent GetProto() { 54 | PlasoEvent event; 55 | int64_t unix_micros; 56 | if (util::RFC3339ToUnixMicros("2012-04-03T00:25:22+00:00", &unix_micros)) { 57 | event.set_timestamp(unix_micros); 58 | } 59 | event.set_desc("ctime"); 60 | return event; 61 | } 62 | 63 | // Test processing of an event with only a timestamp and description. Adding the 64 | // same event multiple times causes multiple nodes to be added to the graph. 65 | TEST_F(PlasoEventGraphTest, ProcessResourceslessEvents) { 66 | EXPECT_EQ(0, graph_.NumNodes()); 67 | EXPECT_EQ(0, graph_.NumEdges()); 68 | PlasoEvent event = GetProto(); 69 | graph_.ProcessEvent(event); 70 | EXPECT_EQ(1, graph_.NumNodes()); 71 | EXPECT_EQ(0, graph_.NumEdges()); 72 | graph_.ProcessEvent(event); 73 | EXPECT_EQ(2, graph_.NumNodes()); 74 | EXPECT_EQ(0, graph_.NumEdges()); 75 | graph_.ProcessEvent(event); 76 | EXPECT_EQ(3, graph_.NumNodes()); 77 | EXPECT_EQ(0, graph_.NumEdges()); 78 | } 79 | 80 | // There are no temporal edges between event with the same timestamp. 81 | TEST_F(PlasoEventGraphTest, NoEdgesBetweenConcurrentEvents) { 82 | PlasoEvent event = GetProto(); 83 | graph_.ProcessEvent(event); 84 | graph_.ProcessEvent(event); 85 | graph_.ProcessEvent(event); 86 | graph_.AddTemporalEdges(); 87 | EXPECT_EQ(3, graph_.NumNodes()); 88 | EXPECT_EQ(0, graph_.NumEdges()); 89 | } 90 | 91 | // There should be edges between temporally ordered, subsequent events. 92 | TEST_F(PlasoEventGraphTest, TemporalEdgesBetweenEvents) { 93 | PlasoEvent event = GetProto(); 94 | graph_.ProcessEvent(event); 95 | event.set_timestamp(event.timestamp() + (int64_t)5000000); 96 | graph_.ProcessEvent(event); 97 | event.set_timestamp(event.timestamp() + (int64_t)5000000); 98 | graph_.ProcessEvent(event); 99 | graph_.AddTemporalEdges(); 100 | EXPECT_EQ(3, graph_.NumNodes()); 101 | EXPECT_EQ(2, graph_.NumEdges()); 102 | } 103 | 104 | TEST_F(PlasoEventGraphTest, ProcessEventsWithFiles) { 105 | PlasoEvent event = GetProto(); 106 | File file = plaso::ParseFilename("example.txt"); 107 | *event.mutable_source_file() = file; 108 | *event.mutable_target_file() = file; 109 | graph_.ProcessEvent(event); 110 | // There should be a node for the event, one for the file, and two edges 111 | // between them as the file is a source and a target. 112 | EXPECT_EQ(2, graph_.NumNodes()); 113 | EXPECT_EQ(2, graph_.NumEdges()); 114 | // Adding the same event information twice should result in two events but 115 | // still only one file. Since there is no target, there should be three edges. 116 | event.clear_source_file(); 117 | graph_.ProcessEvent(event); 118 | EXPECT_EQ(3, graph_.NumNodes()); 119 | EXPECT_EQ(3, graph_.NumEdges()); 120 | } 121 | 122 | TEST_F(PlasoEventGraphTest, ProcessEventsWithURLs) { 123 | PlasoEvent event = GetProto(); 124 | event.set_source_url("www.google.com"); 125 | graph_.ProcessEvent(event); 126 | // There should be a node for the event, one for the URL, and an edge between 127 | // them. 128 | EXPECT_EQ(2, graph_.NumNodes()); 129 | EXPECT_EQ(1, graph_.NumEdges()); 130 | // Adding the same event information twice should result in two events but 131 | // still only one URL. 132 | graph_.ProcessEvent(event); 133 | EXPECT_EQ(3, graph_.NumNodes()); 134 | EXPECT_EQ(2, graph_.NumEdges()); 135 | } 136 | 137 | } // namespace 138 | } // namespace morphie 139 | -------------------------------------------------------------------------------- /base/string.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #ifndef LOGLE_BASE_STRING_H_ 16 | #define LOGLE_BASE_STRING_H_ 17 | 18 | #include 19 | 20 | namespace morphie { 21 | 22 | using std::string; 23 | 24 | } // namespace morphie 25 | #endif // LOGLE_BASE_STRING_H_ 26 | -------------------------------------------------------------------------------- /base/vector.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #ifndef LOGLE_BASE_VECTOR_H_ 16 | #define LOGLE_BASE_VECTOR_H_ 17 | 18 | #include 19 | 20 | namespace morphie { 21 | 22 | using std::vector; 23 | 24 | } // morphie 25 | #endif // LOGLE_BASE_VECTOR_H_ 26 | -------------------------------------------------------------------------------- /build_test/README: -------------------------------------------------------------------------------- 1 | This directory contains simple C++ .cc files that link to intermediate libraries 2 | in the project. These files are used to test and debug issues that may arise 3 | when compiling or linking with CMake. 4 | -------------------------------------------------------------------------------- /build_test/account_access_analyzer_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Initializes the account access analyzer and prints a message. 16 | #include 17 | #include 18 | 19 | #include "account_access_analyzer.h" 20 | 21 | int main(int argc, char **argv) { 22 | morphie::AccessAnalyzer analyzer; 23 | std::unique_ptr parser( 24 | new morphie::util::CSVParser(new std::stringstream(""))); 25 | analyzer.Initialize(std::move(parser)); 26 | std::cout << "Initialized account access analyzer." << std::endl; 27 | } 28 | -------------------------------------------------------------------------------- /build_test/account_access_graph_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Initializes the account access graph builds and prints the empty graph. 16 | #include 17 | #include "account_access_graph.h" 18 | 19 | int main(int argc, char **argv) { 20 | morphie::AccountAccessGraph graph; 21 | graph.Initialize(); 22 | std::cout << "Initialized account access graph." << std::endl 23 | << graph.ToDot() << std::endl; 24 | } 25 | -------------------------------------------------------------------------------- /build_test/analysis_options_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Tests that the AnalysisOptions proto can be built and linked to. 16 | #include 17 | 18 | #include "analysis_options.pb.h" 19 | 20 | int main(int argc, char **argv) { 21 | morphie::AnalysisOptions options; 22 | options.set_output_dot_file("/tmp/out.dot"); 23 | std::cout << options.DebugString() << std::endl; 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /build_test/ast_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Tests that the Abstract Syntax Tree proto and helper functions build and 16 | // prints a simple AST. 17 | #include 18 | 19 | #include "ast.pb.h" 20 | 21 | int main(int argc, char **argv) { 22 | morphie::AST ast; 23 | morphie::PrimitiveValue pval; 24 | pval.set_int_val(5); 25 | ast.mutable_p_ast()->set_type(morphie::INT); 26 | *(ast.mutable_p_ast()->mutable_val()) = pval; 27 | std::cout << ast.DebugString() << std::endl; 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /build_test/curio_analyzer_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Initializes the Curio analyzer and prints a message. 16 | #include 17 | 18 | #include "curio_analyzer.h" 19 | #include "json/json.h" 20 | 21 | int main(int argc, char **argv) { 22 | std::unique_ptr<::Json::Value> doc(new ::Json::Value); 23 | morphie::CurioAnalyzer analyzer; 24 | analyzer.Initialize(std::move(doc)); 25 | std::cout << "Initialized Curio analyzer." << std::endl; 26 | } 27 | -------------------------------------------------------------------------------- /build_test/dot_printer_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Creates and empty graph and uses the DOT printer to serialize it. 16 | #include 17 | 18 | #include "ast.pb.h" 19 | #include "dot_printer.h" 20 | #include "labeled_graph.h" 21 | #include "type.h" 22 | 23 | int main(int argc, char **argv) { 24 | morphie::LabeledGraph graph; 25 | morphie::AST ast = morphie::ast::type::MakeInt("int label", false); 26 | graph.Initialize({}, {}, {}, {}, ast); 27 | morphie::DotPrinter dot_printer; 28 | std::cout << dot_printer.DotGraph(graph) << std::endl; 29 | } 30 | -------------------------------------------------------------------------------- /build_test/gflags_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Tests that the build can link to the external GFlags library (formerly Google 16 | // Flags). The code below should generate an executable that does nothing but 17 | // must be invoked with the flag "--test_flag" with a non-empty argument. 18 | #include 19 | #include 20 | 21 | #include "gflags/gflags.h" 22 | 23 | DEFINE_string(test_flag, "", "Value of the input flag."); 24 | 25 | static bool CheckFlagValueNotEmpty(const char* flagname, 26 | const std::string& value) { 27 | if (value.empty()) { 28 | std::cerr << "The flat " << flagname << " must be non-empty."; 29 | return false; 30 | } 31 | return true; 32 | } 33 | 34 | static bool is_config_empty = 35 | google::RegisterFlagValidator(&FLAGS_test_flag, &CheckFlagValueNotEmpty); 36 | 37 | int main(int argc, char** argv) { 38 | gflags::ParseCommandLineFlags(&argc, &argv, true); 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /build_test/graph_exporter_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Create an exporter for the empty graph and print its output. 16 | #include "graph_exporter.h" 17 | 18 | #include 19 | 20 | #include "ast.pb.h" 21 | #include "labeled_graph.h" 22 | #include "type.h" 23 | 24 | int main(int argc, char **argv) { 25 | morphie::LabeledGraph graph; 26 | morphie::AST ast = morphie::ast::type::MakeInt("int label", false); 27 | graph.Initialize({}, {}, {}, {}, ast); 28 | morphie::viz::GraphExporter exporter(graph); 29 | std::cout << "Exported graph: " << exporter.GraphAsString() << std::endl; 30 | } 31 | -------------------------------------------------------------------------------- /build_test/graph_transformer_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Uses the graph transformer to delete nodes in a graph and prints the result. 16 | #include 17 | 18 | #include "ast.pb.h" 19 | #include "dot_printer.h" 20 | #include "graph_transformer.h" 21 | #include "labeled_graph.h" 22 | #include "morphism.h" 23 | #include "type.h" 24 | #include "value.h" 25 | 26 | int main(int argc, char **argv) { 27 | morphie::LabeledGraph graph; 28 | morphie::AST ast = morphie::ast::type::MakeInt("int label", false); 29 | morphie::TaggedAST tast; 30 | tast.set_tag("num"); 31 | *tast.mutable_ast() = morphie::ast::value::MakeInt(0); 32 | graph.Initialize({{"num", ast}}, {}, {}, {}, ast); 33 | morphie::NodeId node0 = graph.FindOrAddNode(tast); 34 | *tast.mutable_ast() = morphie::ast::value::MakeInt(1); 35 | graph.FindOrAddNode(tast); 36 | morphie::DotPrinter printer; 37 | std::cout << "Input graph." << std::endl 38 | << printer.DotGraph(graph) << std::endl 39 | << "Output graph." << std::endl 40 | << printer.DotGraph( 41 | morphie::graph::DeleteNodes(graph, {node0})->Output()) 42 | << std::endl; 43 | } 44 | -------------------------------------------------------------------------------- /build_test/jsoncpp_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Prints a empty JSON object using the JSONCpp library. 16 | #include 17 | #include "json/json.h" 18 | 19 | int main(int argc, char **argv) { 20 | Json::Value json_doc; 21 | std::cout << json_doc.asString() << std::endl; 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /build_test/labeled_graph_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Construct and print an empty labeled graph. 16 | #include 17 | 18 | #include "ast.pb.h" 19 | #include "labeled_graph.h" 20 | #include "type.h" 21 | 22 | int main(int argc, char **argv) { 23 | morphie::LabeledGraph graph; 24 | morphie::AST ast = morphie::ast::type::MakeInt("int label", false); 25 | graph.Initialize({}, {}, {}, {}, ast); 26 | std::cout << "Initialized graph." << std::endl; 27 | } 28 | -------------------------------------------------------------------------------- /build_test/logging_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Test that the logging utilities work. 16 | #include 17 | 18 | #include "util/logging.h" 19 | 20 | int main(int argc, char **argv) { 21 | morphie::util::Check(true); 22 | std::cout << "Check test passed." << std::endl; 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /build_test/morphism_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Instantiate a morphism with an empty graph and print the result. 16 | #include "morphism.h" 17 | 18 | #include 19 | 20 | #include "ast.pb.h" 21 | #include "labeled_graph.h" 22 | #include "type.h" 23 | 24 | int main(int argc, char **argv) { 25 | morphie::LabeledGraph graph; 26 | morphie::AST ast = morphie::ast::type::MakeInt("int label", false); 27 | graph.Initialize({}, {}, {}, {}, ast); 28 | morphie::graph::Morphism morphism(&graph); 29 | std::cout << "Initialized morphism." << std::endl; 30 | } 31 | -------------------------------------------------------------------------------- /build_test/plaso_analyzer_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Initialize the Plaso analyzer with an empty JSON document. 16 | #include 17 | #include 18 | #include 19 | 20 | #include "analyzers/plaso/plaso_analyzer.h" 21 | #include "json_reader.h" 22 | #include "json/json.h" 23 | 24 | int main(int argc, char **argv) { 25 | morphie::PlasoAnalyzer analyzer(false /* Do not show event sources.*/); 26 | std::istringstream stream(""); 27 | morphie::StreamJson json_stream(&stream); 28 | analyzer.Initialize(&json_stream); 29 | std::cout << "Initialized Plaso analyzer." << std::endl; 30 | } 31 | -------------------------------------------------------------------------------- /build_test/plaso_event_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Test building and linking of the PlasoEvent proto and related utilities. The 16 | // executable should print a proto representation of a filename. 17 | #include 18 | 19 | #include "plaso_event.h" 20 | #include "plaso_event.pb.h" 21 | 22 | int main(int argc, char **argv) { 23 | morphie::File file = morphie::plaso::ParseFilename("/tmp/build/test/file.txt"); 24 | std::cout << file.DebugString() << std::endl; 25 | } 26 | -------------------------------------------------------------------------------- /build_test/plaso_event_graph_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Test building and linking of the PlasoEventGraph by printing the empty graph. 16 | #include 17 | 18 | #include "plaso_event_graph.h" 19 | 20 | int main(int argc, char **argv) { 21 | morphie::PlasoEventGraph graph(false); 22 | graph.Initialize(); 23 | std::cout << "Initialized Plaso event graph." << graph.ToDot() << std::endl; 24 | } 25 | -------------------------------------------------------------------------------- /build_test/stream_dependency_graph_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Test building and linking of the StreamDependencyGraph. 16 | #include 17 | 18 | #include "stream_dependency_graph.h" 19 | 20 | int main(int argc, char **argv) { 21 | morphie::StreamDependencyGraph graph; 22 | graph.Initialize(); 23 | std::cout << "Initialized stream dependency graph." << std::endl 24 | << graph.ToDot() << std::endl; 25 | } 26 | -------------------------------------------------------------------------------- /build_test/type_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // The code below runs the type checker on an AST representing a type. 16 | #include 17 | 18 | #include "ast.pb.h" 19 | #include "type.h" 20 | #include "type_checker.h" 21 | 22 | int main(int argc, char **argv) { 23 | morphie::AST ast = morphie::ast::type::MakeBool("ast", false); 24 | std::cout << ast.DebugString(); 25 | std::string err; 26 | if (morphie::ast::type::IsType(ast, &err)) { 27 | std::cout << " is a type." << std::endl; 28 | } else { 29 | std::cout << " is not a type." << std::endl; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /build_test/type_checker_build_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "ast.pb.h" 5 | #include "type_checker.h" 6 | 7 | int main(int argc, char **argv) { 8 | morphie::AST ast; 9 | ast.set_name("name"); 10 | ast.set_is_nullable("false"); 11 | std::cout << ast.DebugString(); 12 | std::string err; 13 | if (morphie::ast::type::IsType(ast, &err)) { 14 | std::cout << std::endl << "is a type." << std::endl; 15 | } else { 16 | std::cout << std::endl << "is not a type." << std::endl; 17 | } 18 | return 0; 19 | } 20 | -------------------------------------------------------------------------------- /build_test/value_build_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // The code below runs the value checker on an AST representing a value. 16 | #include 17 | #include 18 | 19 | #include "ast.pb.h" 20 | #include "value.h" 21 | #include "value_checker.h" 22 | 23 | int main(int argc, char **argv) { 24 | morphie::AST ast = morphie::ast::value::MakeBool(false); 25 | std::string err; 26 | std::cout << ast.DebugString(); 27 | if (morphie::ast::value::IsValue(ast, &err)) { 28 | std::cout << " is a value." << std::endl; 29 | } else { 30 | std::cout << " is not a value." << std::endl; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /build_test/value_checker_build_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "ast.pb.h" 4 | #include "value_checker.h" 5 | 6 | 7 | int main(int argc, char **argv) { 8 | morphie::AST ast; 9 | std::cout << ast.DebugString(); 10 | std::string err; 11 | std::cout << ast.DebugString(); 12 | if (morphie::ast::value::IsValue(ast, &err)) { 13 | std::cout << std::endl << " is a value." << std::endl; 14 | } else { 15 | std::cout << std::endl << " is not a value." << std::endl; 16 | } 17 | return 0; 18 | } 19 | -------------------------------------------------------------------------------- /frontend.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // A frontend to the different analyzers in logle. 16 | #ifndef LOGLE_FRONTEND_H_ 17 | #define LOGLE_FRONTEND_H_ 18 | 19 | #include "analysis_options.pb.h" 20 | #include "util/status.h" 21 | 22 | namespace morphie { 23 | namespace frontend { 24 | 25 | enum class Analyzer { kCurio, kMailAccess, kPlaso }; 26 | 27 | // Returns the status generated by running the analyzer specified by the 28 | // 'options' proto. Error codes are generated in the following situations. 29 | // * The input format is not supported by the analyzer. 30 | // - The Curio analyzer requires JSON input. 31 | // - The MailAccess analyzer requires CSV input. 32 | // * File I/O errors when reading from an input file, writing to the output DOT 33 | // file, or closing either the input or output file. 34 | // * Analyzer generated errors. Consult the individual analyzer documentation 35 | // for situations in which the analyzers return errors. 36 | util::Status Run(const AnalysisOptions& options); 37 | 38 | } // namespace frontend 39 | } // namespace morphie 40 | #endif // LOGLE_FRONTEND_H_ 41 | -------------------------------------------------------------------------------- /graph/ast.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | syntax = "proto2"; 16 | 17 | package morphie; 18 | 19 | // Unstructured data from logs is typically processed and represented in a 20 | // structured way to enable analysis. The protos in this file implement an 21 | // abstract syntax tree (AST) that can represent types and values of those 22 | // types. The types that will be used are below. 23 | // ptype := BOOL | INT | STRING | TIMESTAMP 24 | // type := null | INTERVAL(ptype) | LIST(type) | TUPLE(type,...,type) | 25 | // SET(type) | type? 26 | // A type is either null, a primitive type, an interval over a primitive type, 27 | // or a composite type. A primitive type is a bool, int, string, or a timestamp 28 | // indicating the number of microseconds since the start of the Unix Epoch. A 29 | // composite type is obtained by applying a type constructor to a type. The 30 | // composite types are intervals over primitive types, or lists of some type, 31 | // sets of some type, or a tuple of types. Lists, sets and tuples are 32 | // collectively referred to as containers. A null type is neither a primitive 33 | // nor a composite type. A null type has no value. A type T is nullable, written 34 | // T?, if it can have no value. Nullable types are useful for situations where 35 | // data is missing. 36 | // 37 | // Example: A log containing timestamped, file system events can be processed 38 | // to obtain two kinds of structures. 39 | // Event: tuple(timestamp?, string?, string?, string?)) 40 | // File: list(string) 41 | // Event data is structured as a tuple of a timestamp and three strings. The 42 | // four fields are nullable because data might be missing. A file is 43 | // represented by a list of strings. Each element of the list represents one 44 | // subdirectory on the file path and the entire list represents the filename. 45 | // 46 | // The AST proto represents an Abstract Syntax Tree (AST) for types and values 47 | // of types. The standard approach to structuring data would be to use a proto 48 | // with specific fields. In the example above, event data can be represented by 49 | // a proto with an int field for a timestamp and three string fields. Such a 50 | // proto can only be used by systems that produce or consume data with this 51 | // specific structure. The alternative used here is to represent types and 52 | // values by ASTs. The AST proto is a universal data structure in the sense that 53 | // it can represent all types defined by the grammar above and also represent 54 | // values of those types. 55 | // 56 | // The trade-off is that we cannot rely on the protobuf compiler for type 57 | // checking. Instead, type checking is done at runtime. This trade-off is 58 | // feasible in this case because the grammar above, and consequently the type 59 | // checker implementation code, is simple. 60 | // 61 | // The representation of types by ASTs is documented in the file type_checker.h 62 | // and the representation of values by ASTs is documented in value_checker.h. 63 | 64 | enum PrimitiveType { 65 | BOOL = 0; 66 | INT = 1; 67 | STRING = 2; 68 | TIMESTAMP = 3; 69 | } 70 | 71 | enum Operator { 72 | INTERVAL = 0; 73 | LIST = 1; 74 | TUPLE = 2; 75 | SET = 3; 76 | } 77 | 78 | message PrimitiveValue { 79 | oneof val { 80 | bool bool_val = 1; 81 | int64 int_val = 2; 82 | string string_val= 3; 83 | int64 time_val = 4; 84 | } 85 | } 86 | 87 | message PrimitiveAST { 88 | required PrimitiveType type = 1; 89 | optional PrimitiveValue val = 2; 90 | } 91 | 92 | message CompositeAST { 93 | required Operator op = 1; 94 | repeated AST arg = 2; 95 | } 96 | 97 | message AST { 98 | optional bool is_nullable = 1; 99 | optional string name = 2; 100 | oneof node { 101 | PrimitiveAST p_ast = 3; 102 | CompositeAST c_ast = 4; 103 | } 104 | } 105 | 106 | // A tagged AST adds a tag to an AST. This allows labelling an AST by the kind 107 | // of data it represents. Example tags are "Event" or "File". 108 | message TaggedAST { 109 | required string tag = 1; 110 | optional AST ast = 2; 111 | }; 112 | -------------------------------------------------------------------------------- /graph/dot_printer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file provides utilities for rendering nodes, edges and entire graphs in 16 | // GraphViz DOT format. The functions here implement default visualizations and 17 | // provide an API for developing custom visualizations. 18 | // 19 | // Example 1. Generate a DOT representation using default settings. 20 | // LabeledGraph graph; 21 | // // Code that constructs a graph. 22 | // string dot_graph = DotPrinter().DotGraph(graph); 23 | // 24 | // Example 2. The client here is using a graph in which nodes are usernames or 25 | // filenames. A client-defined visualization NodeRenderer() will be used for 26 | // nodes but the default visualization will be used for edges. 27 | // 28 | // string CustomRenderer(const string& tag, const AST& ast) { ... } 29 | // 30 | // LabeledGraph graph; 31 | // ConstructGraph(&graph); 32 | // DotPrinter dot_printer(CustomRenderer, DotPrinter::EdgeAttribute); 33 | // string dot_graph = dot_printer.DotGraph(graph); 34 | #ifndef LOGLE_DOT_PRINTER_H_ 35 | #define LOGLE_DOT_PRINTER_H_ 36 | 37 | #include 38 | 39 | #include "base/string.h" 40 | #include "graph/labeled_graph.h" 41 | #include "ast.pb.h" 42 | 43 | namespace morphie { 44 | 45 | // An attribute function returns a GraphViz DOT attribute for a node or an edge. 46 | // The first argument represents a tag and the second is an abstract syntax tree 47 | // (AST) representing a label. See ast.proto for more on ASTs. 48 | using AttributeFn = std::function; 49 | 50 | // The DotPrinter class provides an API for generating GraphViz DOT 51 | // representations of nodes, edges and graphs. The API allows for default 52 | // visualization of edge and node labels. These functions assume that specific 53 | // information, such as a filename or a URL, is represented by ASTs with a 54 | // globally unique tag and type. For example, ASTs for filenames are assumed to 55 | // have a tag "Filename" and are assumed to be lists of strings. The utility 56 | // functions generate DOT representations for predefined combinations of a tag 57 | // and a type. 58 | // 59 | // The GraphViz DOT format declares nodes and edges and their attributes. 60 | // 61 | // Example 3. A GraphViz DOT graph. The attributes are between square-brackets. 62 | // digraph ex3 { 63 | // a [shape=Box, label="Rectangle"]; // Node declaration 64 | // b [shape=Circle, label="Ellipse"]; 65 | // a -> b [style=dashed]; // Edge declaration 66 | // } 67 | // 68 | // To enable customizable visualization, this API provides functions for 69 | // generating an entire DOT graph, generating node and edge declarations with 70 | // attributes, or generating only attributes. The constants define certain node 71 | // and edge styles, which are part of an attribute definition. 72 | class DotPrinter { 73 | public: 74 | // The constructor below, which takes no arguments, sets the node attribute 75 | // function to be DotPrinter::NodeAttribute and the edge attribute function to 76 | // be DotPrinter::EdgeAttribute. 77 | DotPrinter(); 78 | 79 | // This constructor uses the arguments to generate node and edge attributes. 80 | // It is the responsibility of the client to ensure that these attribute 81 | // functions generate syntactically correct DOT attributes. 82 | DotPrinter(const AttributeFn& node_attribute, 83 | const AttributeFn& edge_attribute); 84 | 85 | // The XXXAttribute functions generate predefined node/edge attributes. 86 | // 87 | // Requires that 'ast' is of the type ast::type::MakeFilename(). 88 | static string FileAttribute(const AST& ast); 89 | // Requires that 'ast' is of the type ast::type::MakeIPAddress(). 90 | static string IPAddressAttribute(const AST& ast); 91 | // Requires that 'ast' is of the type ast::type::MakeURL(). 92 | static string URLAttribute(const AST& ast); 93 | // Returns a predefined node attribute if one is defined for 'tag', and 94 | // otherwise returns a DOT node whose label is 'ast' as a string. 95 | static string NodeAttribute(const string& tag, const AST& ast); 96 | // Returns a predefined node attribute if one is defined for 'tag', and 97 | // otherwise returns a DOT node whose label is 'ast' as a string. 98 | static string EdgeAttribute(const string& tag, const AST& ast); 99 | 100 | // Returns a DOT node/edge declaration that is terminated with a semi-colon 101 | // but not a newline. The attribute is chosen using 'ast.tag()'. 102 | string DotNode(NodeId node_id, const TaggedAST& ast); 103 | string DotEdge(NodeId source_id, NodeId target_id, const TaggedAST& ast); 104 | 105 | // Returns a DOT declaration of all nodes/edges in the given graph. If the 106 | // graph has no nodes, AllNodesInDot returns the empty string. If the graph 107 | // has no edges, AllEdgesInDot returns the empty string. 108 | string AllNodesInDot(const LabeledGraph& graph); 109 | string AllEdgesInDot(const LabeledGraph& graph); 110 | 111 | // Returns a DOT representation of the graph. The returned string is not 112 | // newline terminated. 113 | string DotGraph(const LabeledGraph& graph); 114 | 115 | private: 116 | // The function used to generate node attributes. 117 | AttributeFn node_attribute_; 118 | // The function used to generate edge attributes. 119 | AttributeFn edge_attribute_; 120 | }; // class DotPrinter 121 | 122 | } // namespace morphie 123 | #endif // LOGLE_DOT_PRINTER_H_ 124 | -------------------------------------------------------------------------------- /graph/graph_analyzer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains utilities for analyzing graphs. 16 | #ifndef LOGLE_GRAPH_ANALYZER_H_ 17 | #define LOGLE_GRAPH_ANALYZER_H_ 18 | 19 | #include 20 | #include "labeled_graph.h" 21 | 22 | namespace morphie { 23 | 24 | // Contains methods for analyzing graphs and structures on graphs. 25 | namespace graph_analyzer { 26 | 27 | // Takes a partition of the nodes of a graph and returns a refinement of that 28 | // partition. 29 | // Example: Consider the following scenario - threads T0 and T1 both spawn a 30 | // similar series of events. These events are similar so both sets get 31 | // partitioned as E. The refinement would split E into E0 and E1 so that we 32 | // have T0 -> E0, T1 -> E1, instead of 33 | // T0 T1 34 | // \ / 35 | // E 36 | // 37 | // The relational coarsest partition problem is as follows: Given a partition P 38 | // of a set U and a binary relation E on U, find the coarsest refinement Q of P 39 | // such that Q is stable with respect to E. A partition Q is stable with respect 40 | // to E if for all blocks B_1, B_2 of Q, either B_1 is contained in, or disjoint 41 | // from E^-1(B_2) (where E^-1(B_2) is the preimage of B_2). 42 | // Paige, Robert; Tarjan, Robert E. (1987), "Three partition refinement 43 | // algorithms", SIAM Journal on Computing 16 (6): 973–989 44 | std::map RefinePartition(const LabeledGraph& graph, 45 | const std::map& partition); 46 | } // namespace graph_analyzer 47 | 48 | } // namespace morphie 49 | 50 | 51 | #endif // LOGLE_GRAPH_ANALYZER_H_ 52 | -------------------------------------------------------------------------------- /graph/graph_analyzer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #include "graph_analyzer.h" 15 | 16 | #include "gtest.h" 17 | #include "test_graphs.h" 18 | 19 | namespace morphie { 20 | namespace { 21 | 22 | std::map SingleBlockPartition(const LabeledGraph& graph) { 23 | std::map partition; 24 | auto end_it = graph.NodeSetEnd(); 25 | for (auto node_it = graph.NodeSetBegin(); node_it != end_it; ++node_it) { 26 | partition.insert({*node_it, 0}); 27 | } 28 | return partition; 29 | } 30 | 31 | std::map IdentityPartition(const LabeledGraph& graph) { 32 | std::map partition; 33 | auto end_it = graph.NodeSetEnd(); 34 | int i = 0; 35 | for (auto node_it = graph.NodeSetBegin(); node_it != end_it; ++node_it, ++i) { 36 | partition.insert({*node_it, i}); 37 | } 38 | return partition; 39 | } 40 | 41 | // Tests a single block partition on a simple graph. 42 | // input_graph: 43 | // 0 -> 1 44 | TEST(GraphAnalyzerTest, SimpleSingleBlockSplit) { 45 | test::WeightedGraph path; 46 | test::GetPathGraph(2, &path); 47 | ASSERT_EQ(2, path.NumNodes()); 48 | ASSERT_EQ(1, path.NumEdges()); 49 | const LabeledGraph& input_graph = *path.GetGraph(); 50 | 51 | auto partition = SingleBlockPartition(input_graph); 52 | auto refinement = graph_analyzer::RefinePartition(input_graph, partition); 53 | auto ref_it = refinement.begin(); 54 | int id1 = ref_it->second; 55 | ++ref_it; 56 | int id2 = ref_it->second; 57 | EXPECT_NE(id1, id2); 58 | } 59 | 60 | // Tests a single block partition on a simple graph. 61 | // input_graph: 62 | // 0 -> 1 -> 2 -> 3 -> 4 -> 5 63 | TEST(GraphAnalyzerTest, LongSingleBlockSplit) { 64 | test::WeightedGraph path; 65 | test::GetPathGraph(6, &path); 66 | ASSERT_EQ(6, path.NumNodes()); 67 | ASSERT_EQ(5, path.NumEdges()); 68 | const LabeledGraph& input_graph = *path.GetGraph(); 69 | 70 | auto partition = SingleBlockPartition(input_graph); 71 | auto refinement = graph_analyzer::RefinePartition(input_graph, partition); 72 | auto end_it = refinement.end(); 73 | std::set ids; 74 | for (auto ref_it = refinement.begin(); ref_it != end_it; ++ref_it) { 75 | ids.insert(ref_it->second); 76 | } 77 | EXPECT_EQ(6, ids.size()); 78 | } 79 | 80 | // Tests that the coarsest stable refinement of the identity partition is just 81 | // the identity. 82 | // input_graph: 83 | // 0 -> 1 -> 2 84 | // ^ | 85 | // | v 86 | // 5 <- 4 <- 3 87 | TEST(GraphAnalyzerTest, IdentityRefinement) { 88 | test::WeightedGraph cycle; 89 | test::GetCycleGraph(6, &cycle); 90 | ASSERT_EQ(6, cycle.NumNodes()); 91 | ASSERT_EQ(6, cycle.NumEdges()); 92 | const LabeledGraph& input_graph = *cycle.GetGraph(); 93 | 94 | auto partition = IdentityPartition(input_graph); 95 | auto refinement = graph_analyzer::RefinePartition(input_graph, partition); 96 | auto end_it = refinement.end(); 97 | std::set ids; 98 | for (auto ref_it = refinement.begin(); ref_it != end_it; ++ref_it) { 99 | ids.insert(ref_it->second); 100 | } 101 | EXPECT_EQ(6, ids.size()); 102 | } 103 | 104 | // Tests graph_analyzer::RefinePartition on the following partition and graph. 105 | // partition: 106 | // {0, 1, 4, 5}, {2, 3, 6, 7} 107 | // input_graph: 108 | // 0 -> 1 -> 2 109 | // ^ | 110 | // | v 111 | // 7 3 112 | // ^ | 113 | // | v 114 | // 6 <- 5 <- 4 115 | // 116 | // Expected output: 117 | // {0, 4}, {1, 5}, {2, 6}, {3, 7} 118 | TEST(GraphAnalyzerTest, CycleRefinement) { 119 | test::WeightedGraph cycle; 120 | test::GetCycleGraph(8, &cycle); 121 | ASSERT_EQ(8, cycle.NumNodes()); 122 | ASSERT_EQ(8, cycle.NumEdges()); 123 | const LabeledGraph& input_graph = *cycle.GetGraph(); 124 | 125 | std::map ids; 126 | std::map partition; 127 | NodeId current_node = *input_graph.NodeSetBegin(); 128 | for (int index = 0; index < 8; ++index) { 129 | ids.insert({current_node, index}); 130 | int partition_id = index % 4 < 2 ? 0 : 1; 131 | partition.insert({current_node, partition_id}); 132 | current_node = *(input_graph.GetSuccessors(current_node).begin()); 133 | } 134 | auto node_end_it = input_graph.NodeSetEnd(); 135 | auto refinement = graph_analyzer::RefinePartition(input_graph, partition); 136 | for (auto node_it_1 = input_graph.NodeSetBegin(); node_it_1 != node_end_it; 137 | ++node_it_1) { 138 | for (auto node_it_2 = std::next(node_it_1); node_it_2 != node_end_it; 139 | ++node_it_2) { 140 | NodeId node_1 = *node_it_1; 141 | NodeId node_2 = *node_it_2; 142 | if (ids[node_1] % 4 == ids[node_2] % 4) { 143 | EXPECT_EQ(refinement[node_1], refinement[node_2]); 144 | } else { 145 | EXPECT_NE(refinement[node_1], refinement[node_2]); 146 | } 147 | } 148 | } 149 | } 150 | 151 | } // namespace 152 | } // namespace morphie 153 | -------------------------------------------------------------------------------- /graph/graph_explorer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package graph_explorer; 4 | 5 | // GraphExplorer is a tool for interactive visualization of graphs with 6 | // hierarchical structure, rich node and edge labels, and with metadata 7 | // associated with nodes and edges. The structure of the Graph proto in this 8 | // file is inspired by the design of the GraphViz DOT language. It allows for 9 | // specification of various properties of nodes and edges in a graph. In 10 | // addition, a Graph proto includes information about hierarchical structure 11 | // that will be used by the visualizer. 12 | // 13 | // The EBNF grammar below provides an overview of the contents of the proto. 14 | // This grammar is only intended to provide a succinct overview. For details, 15 | // consult the proto definition. All features in the grammar have not been 16 | // implemented in the proto or the visualizer yet. 17 | // 18 | // graph ::= graph_attribute* node* 19 | // node ::= node_id node_attribute* edge* 20 | // edge ::= node_id edge_attribute* 21 | // node_id ::= string 22 | // 23 | // Graph, edge and node attributes are of two types. A Boolean attribute 24 | // indicates if a graph entity has a property (such as whether an edge is 25 | // directed). Valued attributes have a name and a value. Both attributes are 26 | // currently represented as key-value pairs. A Boolean attribute is a key that 27 | // starts with 'is' and the value of that attribute is true if it is present in 28 | // the map. The value of a Boolean key is ignored. The list of supported 29 | // attributes for each graph entity is given within the message defining that 30 | // entity. 31 | 32 | // An Edge message (with a capital 'E') consists of the source node of the edge 33 | // and the attributes of that edge. There is no target node in an edge 34 | // definition because graphs are represented as adjacency lists. 35 | message Edge { 36 | // The identifier of the source node. 37 | string input = 1; 38 | 39 | // * Supported attributes: 40 | // label - a plain text or HTML edge label. 41 | // color - the HTML color of the edge (hex strings are supported). 42 | // 43 | // Attributes that are being added (these can be included in the proto and 44 | // will be accepted by the visualizer but currently do not affect the output.) 45 | // 46 | // width - integer edge width in pixels. 47 | // style - the edge style as an SVG stroke-dasharray. A stroke-dasharray is a 48 | // comma separated list of SVG length and percentage values that 49 | // specifies the lengths of dashes and gaps in the line. For example a 50 | // 48 pixel edge with the style "5, 5, 1, 5" will look like 51 | // "_ . _ . _ ." 52 | // and consists a repeating sequence of a 5 pixel dash, 5 pixel space, 53 | // 1 pixel dash and 5 pixel space. 54 | // isDirected - asserts that the edge has a head and a tail end. 55 | map edge_attr = 2; 56 | } 57 | 58 | // There are two types of nodes. A 'node' or 'leaf node' is a node in the 59 | // standard sense: it may have a label and attributes but does not have 60 | // additional graph structure. A metanode is a collection of nodes or metanodes. 61 | // The metanode containment relationship must be acyclic, meaning that if a 62 | // metanode 'A' contains the metanode 'B', then 'B' cannot contain 'A'. 63 | message Node { 64 | // The identifier of a node, including the namespace hierarchy. The hierarchy 65 | // consists of a series of nodes separated by '/'. ('1/2/3' defines a node of 66 | // id '3' in subgroups of 'node 1'/'node 2'). The identifier uniquely 67 | // identifies the node in the graph. In general, an identifier is not a label, 68 | // but if no label is present, the identifier will be displayed. 69 | string name = 1; 70 | 71 | // For legacy reasons there must be an attribute present named "op". 72 | // 73 | // * Supported attributes of all nodes: 74 | // label - a plain text or HTML node label. 75 | // fillColor - any HTML color, including hex strings. 76 | // 77 | // * Supported attributes of metanodes. 78 | // isMetanode - denotes that this is a metanode. 79 | // isLabelPersistent - denotes whether the label should be displayed when the 80 | // metanode is expanded. 81 | // orientation - direction in which the subgraph within this node is laid out. 82 | // Orientation values are are TB, BT, LR, and RL, representing the 83 | // directions Top, Bottom, Left and Right. 84 | // 85 | // * Node and Metanode attributes that are not supported yet. 86 | // summaryLabel - a text label that appears in the details pane. 87 | // outlineColor - any HTML color, including hex strings. 88 | // outlineStyle - node outline options. Eg. solid, dashed, dotted, etc. 89 | // height - in pixels. 90 | // width - in pixels. 91 | map node_attr = 2; 92 | 93 | // The edges representing inputs into this node. 94 | repeated Edge edge = 3; 95 | }; 96 | 97 | message GraphDef { 98 | // List of nodes and metanodes in the graph. Metanodes do not have to be 99 | // listed explicitly. 100 | repeated Node node = 1; 101 | }; 102 | -------------------------------------------------------------------------------- /graph/graph_exporter.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "graph/graph_exporter.h" 16 | 17 | #include 18 | 19 | #include "graph/ast.h" 20 | #include "util/string_utils.h" 21 | 22 | namespace morphie { 23 | namespace viz { 24 | 25 | GraphExporter::GraphExporter(const LabeledGraph& graph) 26 | : graph_(graph), node_label_(TextLabel) {} 27 | 28 | GraphExporter::GraphExporter(const LabeledGraph& graph, 29 | const LabelFn& node_label) 30 | : graph_(graph), node_label_(node_label) {} 31 | 32 | // Returns a serialization of 'ast' with '/' as a separator and no bounding 33 | // delimiters. The serialization is prefixed by "tag/". 34 | string GraphExporter::TextLabel(const string& tag, const AST& ast) { 35 | string label = tag; 36 | label += "/"; 37 | label += ast::ToString( 38 | ast, ast::PrintConfig("", "", "/", ast::PrintOption::kValue)); 39 | return label; 40 | } 41 | 42 | string GraphExporter::HTMLLabel(const string& tag, const AST& ast) { 43 | ast::PrintConfig config; 44 | if (tag == "Event") { 45 | config = 46 | ast::PrintConfig("
", "
", "
", ast::PrintOption::kValue); 47 | } else { 48 | config = ast::PrintConfig("", "", "/", ast::PrintOption::kValue); 49 | } 50 | return ast::ToString(ast, config); 51 | } 52 | 53 | ge::GraphDef GraphExporter::Graph() { 54 | ge::GraphDef vis_graph; 55 | for (auto node_it = graph_.NodeSetBegin(); node_it != graph_.NodeSetEnd(); 56 | ++node_it) { 57 | ge::Node* vis_node = vis_graph.add_node(); 58 | *vis_node = Node(*node_it); 59 | } 60 | return vis_graph; 61 | } 62 | 63 | string GraphExporter::GraphAsString() { return Graph().DebugString(); } 64 | 65 | string GraphExporter::NodeName(NodeId node_id, const string& tag, 66 | const AST& ast) { 67 | string label = TextLabel(tag, ast); 68 | label += "/"; 69 | label += std::to_string(node_id); 70 | return label; 71 | } 72 | 73 | string GraphExporter::FindOrAddName(NodeId node_id) { 74 | auto node_name_it = node_name_.find(node_id); 75 | if (node_name_it != node_name_.end()) { 76 | return node_name_it->second; 77 | } else { 78 | TaggedAST node_label = graph_.GetNodeLabel(node_id); 79 | string label = NodeName(node_id, node_label.tag(), node_label.ast()); 80 | node_name_.emplace(node_id, label); 81 | return label; 82 | } 83 | } 84 | 85 | ge::Node GraphExporter::Node(NodeId node_id) { 86 | ge::Node vis_node; 87 | if (!graph_.HasNode(node_id)) { 88 | return vis_node; 89 | } 90 | // The node name is an identifier for the node. 91 | string node_name = FindOrAddName(node_id); 92 | vis_node.set_name(node_name); 93 | // The label is the string displayed on the node. 94 | TaggedAST label_ast = graph_.GetNodeLabel(node_id); 95 | string label_str = node_label_(label_ast.tag(), label_ast.ast()); 96 | // vis_node.set_label(HTMLLabel(node_label.tag(), node_label.ast())); 97 | // Set node attributes. 98 | auto& node_attr = *vis_node.mutable_node_attr(); 99 | node_attr["label"] = label_str; 100 | // The value of this field can be used to automatically color a metanode by 101 | // the frequency of types of nodes within the metanode. 102 | node_attr["op"] = "op"; 103 | std::set in_nodes = graph_.GetPredecessors(node_id); 104 | string in_node_name; 105 | for (auto in_node : in_nodes) { 106 | in_node_name = FindOrAddName(in_node); 107 | TaggedAST in_node_label = graph_.GetNodeLabel(in_node); 108 | string in_node_name = 109 | NodeName(in_node, in_node_label.tag(), in_node_label.ast()); 110 | ge::Edge* edge = vis_node.add_edge(); 111 | edge->set_input(in_node_name); 112 | } 113 | return vis_node; 114 | } 115 | 116 | } // namespace viz 117 | } // namespace morphie 118 | -------------------------------------------------------------------------------- /graph/graph_exporter.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains utilities for exporting a Logle LabeledGraph to a graph 16 | // representation that can be used by an internal visualizer. The visualizer is 17 | // currently experimental but once it is mature, will also be released. 18 | #ifndef LOGLE_GRAPH_EXPORTER_H_ 19 | #define LOGLE_GRAPH_EXPORTER_H_ 20 | 21 | #include 22 | 23 | #include "base/string.h" 24 | #include "graph/labeled_graph.h" 25 | #include "ast.pb.h" 26 | #include "graph_explorer.pb.h" 27 | 28 | namespace morphie { 29 | 30 | // A namespace containing visualization-related utilities. 31 | namespace viz { 32 | 33 | namespace ge = ::graph_explorer; 34 | // A node identifier is a string that matches the regexp: 35 | // "[A-Za-z0-9.][A-Za-z0-9_./]*" 36 | // 37 | // Identifiers serve as a clustering mechanism. A common, slash-separated prefix 38 | // defines one node in the hierarchy. 39 | // 40 | // An identifier function takes a tag and an AST that label a graph node or edge 41 | // and returns a TensorFlow label. See ast.proto for more on ASTs. 42 | using LabelFn = std::function; 43 | 44 | // The GraphExporter class below uses an internal map from LabeledGraph node ids 45 | // to GraphDef node identifiers. A separate GraphExporter object has to be 46 | // created for each graph that is to be exported. 47 | class GraphExporter { 48 | public: 49 | // This constructor sets the default node label function, which is 50 | // GraphExporter::NodeLabel. 51 | GraphExporter(const LabeledGraph& graph); 52 | 53 | // This constructor uses the 'node_label' argument to customize the generation 54 | // of node labels. The caller must ensure that the label function generates a 55 | // syntactically correct TensorFlow graph label, as described in the 56 | // class-level comment. 57 | GraphExporter(const LabeledGraph& graph, const LabelFn& node_label); 58 | 59 | // Returns the tag and contents of the AST as a slash-delimited string. 60 | static string TextLabel(const string& tag, const AST& ast); 61 | // Returns the AST as an HTML string. Primary AST values are treated as plain 62 | // strings and composite values are represented as a table. 63 | static string HTMLLabel(const string& tag, const AST& ast); 64 | 65 | // Returns the TensorFlow graph for the internally stored LabeledGraph. 66 | ge::GraphDef Graph(); 67 | 68 | // Returns a human-readable serialization of the GraphDef proto above. 69 | string GraphAsString(); 70 | 71 | private: 72 | // Returns the node label as a text string followed by the node identifier 73 | // from the internal representation of the graph. 74 | string NodeName(NodeId node_id, const string& tag, const AST& ast); 75 | // Returns the label string for 'node_id' in 'graph'. Returns a cached string 76 | // if this label has been computed previously and generates a new label 77 | // otherwise. 78 | string FindOrAddName(NodeId node_id); 79 | // Returns a representation of 'node_id' and its predecessors for 80 | // visualization. 81 | ge::Node Node(NodeId node_id); 82 | 83 | const LabeledGraph& graph_; 84 | // The function used to generate node labels. 85 | LabelFn node_label_; 86 | // The map from LabeledGraph node identifiers to TensorFlow NodeDef names. 87 | unordered_map node_name_; 88 | }; // class GraphExporter 89 | 90 | } // namespace viz 91 | } // namespace morphie 92 | #endif // LOGLE_GRAPH_EXPORTER_H_ 93 | -------------------------------------------------------------------------------- /graph/graph_interface.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | // 15 | #ifndef LOGLE_GRAPH_INTERFACE_H_ 16 | #define LOGLE_GRAPH_INTERFACE_H_ 17 | 18 | #include "base/string.h" 19 | #include "util/status.h" 20 | 21 | namespace morphie { 22 | 23 | // The GraphInterface class is an abstract class that specifies the common API 24 | // for instantiations of the labeled graph. 25 | 26 | class GraphInterface { 27 | public: 28 | 29 | virtual ~GraphInterface() {} 30 | 31 | // Initializes the graph. This function must be called before all other 32 | // functions in this class. Returns 33 | // - Status::OK - if a graph with the appropriate types was created. 34 | // - Status::INTERNAL - otherwise, with the reason accessible via the 35 | // Status::error_message() function of the Status object. 36 | virtual util::Status Initialize() = 0; 37 | 38 | // Functions for statistics about nodes and edges. 39 | virtual int NumNodes() const = 0; 40 | virtual int NumEdges() const = 0; 41 | 42 | // Return a representation of the graph in Graphviz DOT format. 43 | virtual string ToDot() const = 0; 44 | }; // class GraphInterface 45 | 46 | } // namespace morphie 47 | #endif // LOGLE_GRAPH_INTERFACE_H_ 48 | 49 | -------------------------------------------------------------------------------- /graph/morphism.cc: -------------------------------------------------------------------------------- 1 | #include "morphism.h" 2 | 3 | #include "util/map_utils.h" 4 | #include "util/status.h" 5 | 6 | namespace morphie { 7 | namespace graph { 8 | 9 | std::unique_ptr Morphism::TakeOutput() { 10 | node_map_.clear(); 11 | node_preimage_.clear(); 12 | return std::move(output_graph_); 13 | } 14 | 15 | void Morphism::CopyInputType() { 16 | output_graph_.reset(new LabeledGraph()); 17 | util::Status status = output_graph_->Initialize( 18 | input_graph_.GetNodeTypes(), input_graph_.GetUniqueNodeTags(), 19 | input_graph_.GetEdgeTypes(), input_graph_.GetUniqueEdgeTags(), 20 | input_graph_.GetGraphType()); 21 | if (!status.ok()) { 22 | output_graph_.reset(nullptr); 23 | } 24 | } 25 | 26 | NodeId Morphism::FindOrCopyNode(NodeId input_node) { 27 | TaggedAST label = input_graph_.GetNodeLabel(input_node); 28 | return FindOrMapNode(input_node, label); 29 | } 30 | 31 | NodeId Morphism::FindOrMapNode(NodeId input_node, TaggedAST label) { 32 | auto map_it = node_map_.find(input_node); 33 | if (map_it != node_map_.end()) { 34 | return map_it->second; 35 | } 36 | NodeId output_node = output_graph_->FindOrAddNode(label); 37 | node_map_.insert({input_node, output_node}); 38 | auto preimage_it = node_preimage_.find(output_node); 39 | if (preimage_it == node_preimage_.end()) { 40 | node_preimage_.insert({output_node, {input_node}}); 41 | } else { 42 | preimage_it->second.insert(input_node); 43 | } 44 | return output_node; 45 | } 46 | 47 | EdgeId Morphism::FindOrCopyEdge(EdgeId input_edge) { 48 | TaggedAST label = input_graph_.GetEdgeLabel(input_edge); 49 | return FindOrMapEdge(input_edge, label); 50 | } 51 | 52 | EdgeId Morphism::FindOrMapEdge(EdgeId input_edge, TaggedAST label) { 53 | NodeId src = FindOrCopyNode(input_graph_.Source(input_edge)); 54 | NodeId tgt = FindOrCopyNode(input_graph_.Target(input_edge)); 55 | EdgeId output_edge = output_graph_->FindOrAddEdge(src, tgt, label); 56 | return output_edge; 57 | } 58 | 59 | util::Status Morphism::ComposeWith(Morphism* morphism) { 60 | if (output_graph_.get() != &morphism->input_graph_) { 61 | return util::Status(Code::INVALID_ARGUMENT, 62 | "Trying to compose incompatible morphisms."); 63 | } 64 | node_map_ = util::Compose(node_map_, morphism->node_map_); 65 | node_preimage_ = util::Preimage(node_map_); 66 | output_graph_ = morphism->TakeOutput(); 67 | return util::Status::OK; 68 | } 69 | 70 | } // namespace graph 71 | } // namespace morphie 72 | -------------------------------------------------------------------------------- /graph/morphism.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // A graph transformation generates an output graph from an input graph. For 16 | // interactive visualization and graph summarization, it is useful to be able to 17 | // relate nodes and edges in the input graph and output graphs. 18 | // 19 | // Example 1. Consider a graph with edges {1 --> 1, 1 -> 2} and the graph 20 | // {1 --> 1} obtained by deleting the node 2. The node '1' and its self-edge map 21 | // to the output graph but the node '2' and its incoming edge do not exist in 22 | // the output. 23 | // 24 | // Example 2. Consider a graph with edges {1 --> 2, 2 -> 1} and the graph 25 | // {a --> a} obtained by merging the nodes 1 and 2, while preserving their edge 26 | // relationships. The input nodes '1' and '2' map to the output node 'a' and 27 | // both input edges map to the same output edge. If self-loops are ommitted in 28 | // the output, the nodes '1' and'2' will map to 'a' but the edges between them 29 | // will not map to anything. 30 | // 31 | // This file defines the Morphism class for tracking relationships between nodes 32 | // and edges in input and output graphs. The main feature of Morphisms is that 33 | // they compose, so one can apply a series of transformations to an input graph 34 | // and only obtain the relationship between the first and large graph generated 35 | // by these transformations. 36 | #ifndef LOGLE_MORPHISM_H_ 37 | #define LOGLE_MORPHISM_H_ 38 | 39 | #include 40 | #include 41 | 42 | #include "labeled_graph.h" 43 | #include "ast.pb.h" 44 | 45 | namespace morphie { 46 | namespace graph { 47 | 48 | // A morphism, with lower-case 'm', is a pair of partial functions from the 49 | // nodes and edges of an input graph to the nodes an edges of the output graph. 50 | // Some nodes and edges in the input graph may have no corresponding elements in 51 | // the output graph, which is why the functions are partial. 52 | // 53 | // The Morphism class provides helper functions for creating and manipulating 54 | // morphisms. 55 | class Morphism { 56 | public: 57 | // A Morphism requires a non-null pointer and will not own the input graph. 58 | explicit Morphism(const LabeledGraph* graph) : input_graph_(*graph) {} 59 | 60 | bool HasOutputGraph() { return output_graph_ != nullptr; } 61 | // Functions that return the input and output graphs. 62 | const LabeledGraph& Input() const { return input_graph_; } 63 | const LabeledGraph& Output() const { return *output_graph_; } 64 | LabeledGraph* MutableOutput() { return output_graph_.get(); } 65 | // Returns and gives up ownership of the output graph and clears the internal 66 | // maps between input and output nodes. 67 | std::unique_ptr TakeOutput(); 68 | 69 | // Creates a new output graph that has the same node and edge types as the 70 | // input graph. An output graph that already exists will no longer be 71 | // accessible. 72 | void CopyInputType(); 73 | 74 | // Returns the id of an output node with the same label as input_node. Adds a 75 | // node to the output graph if no such node exists. 76 | NodeId FindOrCopyNode(NodeId input_node); 77 | // Returns the id of the output node that the input node maps to in the 78 | // morphism. Adds a new node to the output graph if no such node exists. 79 | NodeId FindOrMapNode(NodeId input_node, TaggedAST label); 80 | 81 | // These functions are similar to the functions for adding nodes above. 82 | EdgeId FindOrCopyEdge(EdgeId input_edge); 83 | EdgeId FindOrMapEdge(EdgeId input_edge, TaggedAST label); 84 | 85 | // Composes this morphism with the input and takes ownership of the output 86 | // graph in the input morphism. The output graph that existed before 87 | // composition cannot be access after the composition. 88 | util::Status ComposeWith(Morphism* morphism); 89 | 90 | private: 91 | const LabeledGraph& input_graph_; 92 | std::unordered_map node_map_; 93 | std::unordered_map> node_preimage_; 94 | std::unique_ptr output_graph_; 95 | }; // class Morphism 96 | 97 | } // namespace graph 98 | } // namespace morphie 99 | #endif // LOGLE_MORPHISM_H_ 100 | -------------------------------------------------------------------------------- /graph/morphism_test.cc: -------------------------------------------------------------------------------- 1 | #include "morphism.h" 2 | 3 | #include "gtest.h" 4 | #include "test_graphs.h" 5 | #include "value.h" 6 | 7 | namespace morphie { 8 | namespace graph { 9 | namespace { 10 | 11 | const char kNodeWeightTag[] = "Node-Weight"; 12 | const char kEdgeWeightTag[] = "Edge-Weight"; 13 | 14 | TEST(MorphismTest, ConstructorDoesNotCreateOutputGraph) { 15 | test::WeightedGraph weighted_graph; 16 | test::GetPathGraph(2, &weighted_graph); 17 | Morphism morphism(weighted_graph.GetGraph()); 18 | EXPECT_FALSE(morphism.HasOutputGraph()); 19 | } 20 | 21 | TEST(MorphismTest, CopyGraphCreatesEmptyOutputGraph) { 22 | test::WeightedGraph weighted_graph; 23 | test::GetPathGraph(2, &weighted_graph); 24 | Morphism morphism(weighted_graph.GetGraph()); 25 | morphism.CopyInputType(); 26 | EXPECT_TRUE(morphism.HasOutputGraph()); 27 | EXPECT_EQ(0, morphism.Output().NumNodes()); 28 | } 29 | 30 | TEST(MorphismTest, OutputGraphProperties) { 31 | test::WeightedGraph weighted_graph; 32 | test::GetPathGraph(2, &weighted_graph); 33 | Morphism morphism(weighted_graph.GetGraph()); 34 | morphism.CopyInputType(); 35 | // Add a node to the output graph. 36 | TaggedAST label; 37 | label.set_tag(kNodeWeightTag); 38 | *label.mutable_ast() = ast::value::MakeInt(2); 39 | morphism.MutableOutput()->FindOrAddNode(label); 40 | EXPECT_EQ(1, morphism.Output().NumNodes()); 41 | } 42 | 43 | TEST(MorphismTest, TakeOutputProperties) { 44 | test::WeightedGraph weighted_graph; 45 | test::GetPathGraph(2, &weighted_graph); 46 | Morphism morphism(weighted_graph.GetGraph()); 47 | morphism.CopyInputType(); 48 | ASSERT_TRUE(morphism.HasOutputGraph()); 49 | auto output_graph = morphism.TakeOutput(); 50 | EXPECT_FALSE(morphism.HasOutputGraph()); 51 | } 52 | 53 | } // namespace 54 | } // namespace graph 55 | } // namespace morphie 56 | -------------------------------------------------------------------------------- /graph/test_graphs.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains the definitions of different types of labeled graphs for 16 | // use in testing graph algorithms. 17 | #ifndef LOGLE_TEST_GRAPHS_H_ 18 | #define LOGLE_TEST_GRAPHS_H_ 19 | 20 | #include 21 | 22 | #include "base/string.h" 23 | #include "graph/labeled_graph.h" 24 | #include "util/status.h" 25 | 26 | namespace morphie { 27 | namespace test { 28 | 29 | // A weighted graph is one in which the nodes and edges have int weights. There 30 | // can be multiple nodes with the same weight and multiple edges of the same 31 | // weight between a given pair of nodes. A WeightedGraph must be initialized by 32 | // calling WeightedGraph::Initialize() before it can be used. 33 | class WeightedGraph { 34 | public: 35 | WeightedGraph() : is_initialized_(false) {} 36 | 37 | // Initializes the graph and returns one of the following error codes: 38 | // - Status::OK - if a graph with the appropriate types was created. 39 | // - Status::INTERNAL - otherwise, with the reason accessible via the 40 | // Status::error_message() function of the Status object. 41 | util::Status Initialize(); 42 | 43 | // Adds a new node with the given weight to the graph and returns the 44 | // identifier for that node. 45 | NodeId AddNode(int node_weight); 46 | 47 | // Adds a new, directed edge with the given weight to the graph and returns 48 | // the identifier for that edge. 49 | EdgeId AddEdge(NodeId src, NodeId tgt, int edge_weight); 50 | 51 | // Returns the set of identifiers of the nodes with a given weight. 52 | std::set GetNodes(int node_weight) const; 53 | 54 | const LabeledGraph* GetGraph() const; 55 | 56 | int NumNodes() const; 57 | int NumEdges() const; 58 | 59 | // Returns a representation of the graph in Graphviz DOT format. 60 | string ToDot() const; 61 | 62 | private: 63 | bool is_initialized_; 64 | LabeledGraph graph_; 65 | }; // class IntGraph 66 | 67 | // A path graph with k+1 nodes, for k greater than or equal to 0, has 68 | // - a node with weight j for every j in [0,k], and 69 | // - an edge j -> j+1 with weight j for j in [0, k-1]. 70 | // A path graph can be drawn as a straight line. 71 | void GetPathGraph(int num_nodes, WeightedGraph* graph); 72 | 73 | // A cycle graph with k+1 nodes, for k greater than or equal to 0, has 74 | // - a node with weight j for every j in [0,k], and 75 | // - an edge j -> (j+1 mod k) with weight j for j in [0, k]. 76 | // A cycle graph can be drawn as a simple cycle. 77 | void GetCycleGraph(int num_nodes, WeightedGraph* graph); 78 | 79 | // Returns true if 'graph' is a path graph. 80 | bool IsPath(const LabeledGraph& graph); 81 | 82 | // Returns true if 'graph' is a cycle graph. 83 | bool IsCycle(const LabeledGraph& graph); 84 | 85 | } // namespace test 86 | } // namespace morphie 87 | #endif // LOGLE_TEST_GRAPHS_H_ 88 | -------------------------------------------------------------------------------- /graph/test_graphs_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "test_graphs.h" 16 | 17 | #include "gtest.h" 18 | 19 | namespace morphie { 20 | namespace test { 21 | namespace { 22 | 23 | // Test that path-shaped graphs labeled with increasing numbers are created. 24 | TEST(TestGraphsTest, CreatesPathGraphs) { 25 | // Create the graph with one node and no edges. 26 | WeightedGraph one_path; 27 | test::GetPathGraph(1, &one_path); 28 | EXPECT_EQ(1, one_path.NumNodes()); 29 | EXPECT_EQ(0, one_path.NumEdges()); 30 | // Create the graph {0 -0-> 1, ... , 8 -8-> 9 } with 10 nodes and 8 edges. 31 | WeightedGraph ten_path; 32 | test::GetPathGraph(10, &ten_path); 33 | EXPECT_EQ(10, ten_path.NumNodes()); 34 | EXPECT_EQ(9, ten_path.NumEdges()); 35 | } 36 | 37 | // Test that cycle-shaped graphs labeled with increasing numbers are created. 38 | TEST(TestGraphsTest, CreatesCycleGraphs) { 39 | // Create the graph {0 -0-> 0 } with one node and one edge. 40 | WeightedGraph one_cycle; 41 | test::GetCycleGraph(1, &one_cycle); 42 | EXPECT_EQ(1, one_cycle.NumNodes()); 43 | EXPECT_EQ(1, one_cycle.NumEdges()); 44 | // Create the graph {0 -0-> 1, ... , 9 -9-> 0 } with 10 nodes and 10 edges. 45 | WeightedGraph ten_cycle; 46 | test::GetCycleGraph(10, &ten_cycle); 47 | EXPECT_EQ(10, ten_cycle.NumNodes()); 48 | EXPECT_EQ(10, ten_cycle.NumEdges()); 49 | } 50 | 51 | TEST(TestGraphsTest, IsPathGraph) { 52 | WeightedGraph one_cycle; 53 | test::GetCycleGraph(1, &one_cycle); 54 | ASSERT_EQ(1, one_cycle.NumNodes()); 55 | ASSERT_EQ(1, one_cycle.NumEdges()); 56 | const auto& one_cycle_graph = *one_cycle.GetGraph(); 57 | EXPECT_FALSE(test::IsPath(one_cycle_graph)); 58 | 59 | WeightedGraph ten_cycle; 60 | test::GetCycleGraph(10, &ten_cycle); 61 | ASSERT_EQ(10, ten_cycle.NumNodes()); 62 | ASSERT_EQ(10, ten_cycle.NumEdges()); 63 | const auto& ten_cycle_graph = *ten_cycle.GetGraph(); 64 | EXPECT_FALSE(test::IsPath(ten_cycle_graph)); 65 | 66 | WeightedGraph one_path; 67 | test::GetPathGraph(1, &one_path); 68 | ASSERT_EQ(1, one_path.NumNodes()); 69 | ASSERT_EQ(0, one_path.NumEdges()); 70 | const auto& one_path_graph = *one_path.GetGraph(); 71 | EXPECT_TRUE(test::IsPath(one_path_graph)); 72 | 73 | WeightedGraph ten_path; 74 | test::GetPathGraph(10, &ten_path); 75 | ASSERT_EQ(10, ten_path.NumNodes()); 76 | ASSERT_EQ(9, ten_path.NumEdges()); 77 | const auto& ten_path_graph = *ten_path.GetGraph(); 78 | EXPECT_TRUE(test::IsPath(ten_path_graph)); 79 | } 80 | 81 | TEST(TestGraphsTest, IsCycleGraph) { 82 | WeightedGraph one_cycle; 83 | test::GetCycleGraph(1, &one_cycle); 84 | ASSERT_EQ(1, one_cycle.NumNodes()); 85 | ASSERT_EQ(1, one_cycle.NumEdges()); 86 | const auto& one_cycle_graph = *one_cycle.GetGraph(); 87 | EXPECT_TRUE(test::IsCycle(one_cycle_graph)); 88 | 89 | WeightedGraph ten_cycle; 90 | test::GetCycleGraph(10, &ten_cycle); 91 | ASSERT_EQ(10, ten_cycle.NumNodes()); 92 | ASSERT_EQ(10, ten_cycle.NumEdges()); 93 | const auto& ten_cycle_graph = *ten_cycle.GetGraph(); 94 | EXPECT_TRUE(test::IsCycle(ten_cycle_graph)); 95 | 96 | WeightedGraph one_path; 97 | test::GetPathGraph(1, &one_path); 98 | ASSERT_EQ(1, one_path.NumNodes()); 99 | ASSERT_EQ(0, one_path.NumEdges()); 100 | const auto& one_path_graph = *one_path.GetGraph(); 101 | EXPECT_FALSE(test::IsCycle(one_path_graph)); 102 | 103 | WeightedGraph ten_path; 104 | test::GetPathGraph(10, &ten_path); 105 | ASSERT_EQ(10, ten_path.NumNodes()); 106 | ASSERT_EQ(9, ten_path.NumEdges()); 107 | const auto& ten_path_graph = *ten_path.GetGraph(); 108 | EXPECT_FALSE(test::IsCycle(ten_path_graph)); 109 | } 110 | 111 | } // namespace 112 | } // namespace test 113 | } // namespace morphie 114 | -------------------------------------------------------------------------------- /graph/type.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "graph/type.h" 16 | 17 | #include 18 | 19 | #include "graph/ast.h" 20 | #include "graph/type_checker.h" 21 | #include "util/logging.h" 22 | 23 | namespace morphie { 24 | namespace ast { 25 | namespace type { 26 | 27 | namespace { 28 | 29 | const char kIntervalArg[] = "bound"; 30 | // Arguments of lists and sets do not have a name. List elements will be 31 | // referred to by their position in the list. Set elements will be referred to 32 | // their position in an enumeration of the set contents. This enumeration is not 33 | // guaranteed to be unique. 34 | const char kContainerArg[] = ""; 35 | 36 | void SetFields(const string& name, bool is_nullable, AST* ast) { 37 | ast->set_name(name); 38 | ast->set_is_nullable(is_nullable); 39 | } 40 | 41 | } // namespace 42 | 43 | // Neither of the oneof options is chosen in a null type. 44 | AST MakeNull(const string& name) { 45 | AST ast; 46 | SetFields(name, true, &ast); 47 | return ast; 48 | } 49 | 50 | AST MakeBool(const string& name, bool is_nullable) { 51 | return MakePrimitive(name, is_nullable, PrimitiveType::BOOL); 52 | } 53 | 54 | AST MakeInt(const string& name, bool is_nullable) { 55 | return MakePrimitive(name, is_nullable, PrimitiveType::INT); 56 | } 57 | 58 | AST MakeString(const string& name, bool is_nullable) { 59 | return MakePrimitive(name, is_nullable, PrimitiveType::STRING); 60 | } 61 | 62 | AST MakeTimestamp(const string& name, bool is_nullable) { 63 | return MakePrimitive(name, is_nullable, PrimitiveType::TIMESTAMP); 64 | } 65 | 66 | AST MakePrimitive(const string& name, bool is_nullable, PrimitiveType ptype) { 67 | AST ast; 68 | SetFields(name, is_nullable, &ast); 69 | ast.mutable_p_ast()->set_type(ptype); 70 | ast.mutable_p_ast()->clear_val(); 71 | return ast; 72 | } 73 | 74 | AST MakeInterval(const string& name, PrimitiveType ptype) { 75 | AST arg = MakePrimitive(kIntervalArg, true, ptype); 76 | std::vector args; 77 | args.emplace_back(arg); 78 | return MakeComposite(name, true, Operator::INTERVAL, args); 79 | } 80 | 81 | AST MakeList(const string& name, bool is_nullable, const AST& arg) { 82 | return MakeContainer(name, is_nullable, Operator::LIST, arg); 83 | } 84 | 85 | AST MakeSet(const string& name, bool is_nullable, const AST& arg) { 86 | return MakeContainer(name, is_nullable, Operator::SET, arg); 87 | } 88 | 89 | AST MakeContainer(const string& name, bool is_nullable, Operator op, 90 | const AST& arg) { 91 | std::vector args; 92 | args.emplace_back(arg); 93 | AST ast = MakeComposite(name, is_nullable, op, args); 94 | ast.mutable_c_ast()->mutable_arg(0)->set_name(kContainerArg); 95 | return ast; 96 | } 97 | 98 | AST MakeTuple(const string& name, bool is_nullable, 99 | const std::vector& args) { 100 | return MakeComposite(name, is_nullable, Operator::TUPLE, args); 101 | } 102 | 103 | AST MakeComposite(const string& name, bool is_nullable, Operator op, 104 | const std::vector& args) { 105 | AST ast; 106 | SetFields(name, is_nullable, &ast); 107 | ast.mutable_c_ast()->set_op(op); 108 | 109 | string tmp_err; 110 | AST* new_arg; 111 | for (const auto& arg : args) { 112 | CHECK(IsType(arg, &tmp_err), tmp_err); 113 | new_arg = ast.mutable_c_ast()->add_arg(); 114 | *new_arg = arg; 115 | } 116 | return ast; 117 | } 118 | 119 | AST MakeDirectory() { 120 | return MakeList(kDirectory, true, MakeString(kFilePathPart, false)); 121 | } 122 | 123 | AST MakeFile() { 124 | std::vector args; 125 | args.push_back(MakeDirectory()); 126 | args.push_back(MakeString(kFilename, true)); 127 | return MakeTuple(kFileTag, true, args); 128 | } 129 | 130 | AST MakeIPAddress() { 131 | return MakeString(kIPAddressTag, false); 132 | } 133 | 134 | AST MakeURL() { 135 | return MakeString(kURLTag, false); 136 | } 137 | 138 | } // namespace type 139 | } // namespace ast 140 | } // namespace morphie 141 | -------------------------------------------------------------------------------- /graph/type.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains functions for constructing ASTs that represent types. See 16 | // third_party/logle/graph/ast.proto and third_party/logle/graph/type_checker.h 17 | // for details on types. 18 | #ifndef LOGLE_TYPE_H_ 19 | #define LOGLE_TYPE_H_ 20 | 21 | #include "base/string.h" 22 | #include "base/vector.h" 23 | #include "ast.pb.h" 24 | 25 | namespace morphie { 26 | namespace ast { 27 | namespace type { 28 | 29 | // The Make[Type](name, is_nullable, ...) functions return an AST for a type 30 | // with the 'name' and 'is_nullable' fields set to the given inputs. 31 | // 32 | // Return an AST for the null type. 33 | AST MakeNull(const string& name); 34 | 35 | // The functions below create primitive types. 36 | AST MakeBool(const string& name, bool is_nullable); 37 | AST MakeInt(const string& name, bool is_nullable); 38 | AST MakeString(const string& name, bool is_nullable); 39 | AST MakeTimestamp(const string& name, bool is_nullable); 40 | AST MakePrimitive(const string& name, bool is_nullable, PrimitiveType ptype); 41 | 42 | // This function returns an interval type with arguments of type 'ptype'. The 43 | // interval is nullable and has nullable arguments. 44 | // - Requires that 'ptype' is ordered, which can be checked by calling 45 | // ast::IsOrdered(ptype). 46 | AST MakeInterval(const string& name, PrimitiveType ptype); 47 | 48 | // The functions below create composite types with 'arg' and 'args' containing 49 | // the arguments to the type constructor. 50 | // - These functions require that 'arg' and 'args' contain types, which can 51 | // be checked by calling type::IsAType(arg). 52 | AST MakeList(const string& name, bool is_nullable, const AST& arg); 53 | AST MakeSet(const string& name, bool is_nullable, const AST& arg); 54 | AST MakeContainer(const string& name, bool is_nullable, Operator op, 55 | const AST& arg); 56 | AST MakeTuple(const string& name, bool is_nullable, 57 | const std::vector& args); 58 | AST MakeComposite(const string& name, bool is_nullable, Operator op, 59 | const std::vector& args); 60 | 61 | // The Make[Data] functions below generate predefined ASTs for commonly 62 | // occurring types of data. 63 | // 64 | // A directory is a list of strings representing a path. 65 | AST MakeDirectory(); 66 | // A file consists of a directory containing the file and a filename. 67 | AST MakeFile(); 68 | // An IP address and a URL are represented as strings. 69 | AST MakeIPAddress(); 70 | AST MakeURL(); 71 | 72 | } // namespace type 73 | } // namespace ast 74 | } // namespace morphie 75 | 76 | #endif // LOGLE_TYPE_H_ 77 | -------------------------------------------------------------------------------- /graph/type_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "graph/type.h" 16 | 17 | #include "base/vector.h" 18 | #include "graph/type_checker.h" 19 | #include "gtest.h" 20 | 21 | namespace morphie { 22 | namespace ast { 23 | namespace type { 24 | namespace { 25 | 26 | class TypeTest : public ::testing::Test { 27 | protected: 28 | AST type_ast_; 29 | AST arg_ast_; 30 | string err_; 31 | }; 32 | 33 | // Test creation of the null type. 34 | TEST_F(TypeTest, CreatesNullType) { 35 | type_ast_ = MakeNull("foo"); 36 | EXPECT_TRUE(IsType(type_ast_, &err_)); 37 | } 38 | 39 | // Test creation of primitive types. 40 | TEST_F(TypeTest, CreatesABool) { 41 | type_ast_ = MakeBool("foo", true); 42 | EXPECT_TRUE(IsType(type_ast_, &err_)); 43 | } 44 | 45 | TEST_F(TypeTest, CreatesAnInt) { 46 | type_ast_ = MakeInt("foo", false); 47 | EXPECT_TRUE(IsType(type_ast_, &err_)); 48 | } 49 | 50 | TEST_F(TypeTest, CreatesAString) { 51 | type_ast_ = MakeString("foo", false); 52 | EXPECT_TRUE(IsType(type_ast_, &err_)); 53 | } 54 | 55 | TEST_F(TypeTest, CreatesATimestamp) { 56 | type_ast_ = MakeTimestamp("foo", false); 57 | EXPECT_TRUE(IsType(type_ast_, &err_)); 58 | } 59 | 60 | // Test creation of intervals. 61 | TEST_F(TypeTest, CreatesAnInterval) { 62 | type_ast_ = MakeInterval("foo", PrimitiveType::TIMESTAMP); 63 | EXPECT_TRUE(IsType(type_ast_, &err_)); 64 | } 65 | 66 | // Test creation of composite types. Create a type for a list of timestamps. 67 | TEST_F(TypeTest, CreatesAList) { 68 | arg_ast_ = MakeTimestamp("foo", false); 69 | type_ast_ = MakeList("bar", false, arg_ast_); 70 | EXPECT_TRUE(IsType(type_ast_, &err_)); 71 | } 72 | 73 | // Create a type for a set of timestamps. 74 | TEST_F(TypeTest, CreatesASet) { 75 | arg_ast_ = MakeTimestamp("foo", false); 76 | type_ast_ = MakeSet("bar", false, arg_ast_); 77 | EXPECT_TRUE(IsType(type_ast_, &err_)); 78 | } 79 | 80 | // Create a type for a tuple of a timestamp and a string. 81 | TEST_F(TypeTest, CreatesATuple) { 82 | std::vector args; 83 | arg_ast_ = MakeTimestamp("foo", false); 84 | args.emplace_back(arg_ast_); 85 | arg_ast_ = MakeString("bar", false); 86 | args.emplace_back(arg_ast_); 87 | type_ast_ = MakeTuple("baz", false, args); 88 | EXPECT_TRUE(IsType(type_ast_, &err_)); 89 | } 90 | 91 | // Creates types for directories and files. 92 | TEST_F(TypeTest, FileTypes) { 93 | type_ast_ = MakeFile(); 94 | // A file is a tuple consisting of a file path and a file name. 95 | // EXPECT_TRUE(type_ast_.has_c_ast()); 96 | EXPECT_TRUE(type_ast_.c_ast().op() == Operator::TUPLE); 97 | EXPECT_TRUE(type_ast_.c_ast().arg_size() == 2); 98 | // The file path is a list of strings. 99 | AST path = type_ast_.c_ast().arg(0); 100 | EXPECT_TRUE(path.c_ast().op() == Operator::LIST); 101 | EXPECT_TRUE(path.c_ast().arg_size() == 1); 102 | EXPECT_TRUE(path.c_ast().arg(0).has_p_ast()); 103 | EXPECT_TRUE(path.c_ast().arg(0).p_ast().type() == PrimitiveType::STRING); 104 | // The filename is a string. 105 | AST filename = type_ast_.c_ast().arg(1); 106 | EXPECT_TRUE(filename.p_ast().type() == PrimitiveType::STRING); 107 | } 108 | 109 | // Check that non-null but non-type arguments make composite methods crash. 110 | TEST(TypeDeathTest, NonTypeASTToListArg) { 111 | AST arg, type; 112 | arg.clear_name(); 113 | EXPECT_DEATH({ type = MakeList("foo", false, arg); }, ".*"); 114 | } 115 | 116 | TEST(TypeDeathTest, NonTypeASTToSetArg) { 117 | AST arg, type; 118 | arg.clear_name(); 119 | EXPECT_DEATH({ type = MakeSet("foo", false, arg); }, ".*"); 120 | } 121 | 122 | TEST(TypeDeathTest, NonTypeASTToTupleArg) { 123 | AST arg, type; 124 | std::vector args; 125 | arg = MakeBool("foo", false); 126 | args.emplace_back(arg); 127 | arg.clear_is_nullable(); 128 | args.emplace_back(arg); 129 | EXPECT_DEATH({ type = MakeTuple("bar", false, args); }, ".*"); 130 | } 131 | 132 | TEST(TypeDeathTest, NonTypeASTToCompositeArg) { 133 | AST arg, type; 134 | std::vector args; 135 | arg = MakeBool("foo", false); 136 | args.emplace_back(arg); 137 | arg.clear_is_nullable(); 138 | args.emplace_back(arg); 139 | EXPECT_DEATH({ 140 | type = MakeComposite("bar", false, Operator::TUPLE, args); }, ".*"); 141 | } 142 | 143 | } // namespace 144 | } // namespace type 145 | } // namespace ast 146 | } // namespace morphie 147 | -------------------------------------------------------------------------------- /gtest.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #ifndef LOGLE_GTEST_H_ 16 | #define LOGLE_GTEST_H_ 17 | 18 | #if defined(PLATFORM_GOOGLE) 19 | #include "testing/base/public/gunit.h" 20 | #else 21 | #include 22 | #endif 23 | 24 | #endif // LOGLE_GTEST_H_ 25 | -------------------------------------------------------------------------------- /logle.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Logle is a tool for analyzing logs using graph algorithms and graph 16 | // visualization techniques. It can run one of many different analyses on a log 17 | // file in either CSV or JSON format. 18 | // 19 | // Development is at an early stage. 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "analysis_options.pb.h" 26 | #include "gflags/gflags.h" 27 | #include "frontend.h" 28 | #include "util/status.h" 29 | 30 | namespace protobuf = google::protobuf; 31 | using Analyzer = morphie::frontend::Analyzer; 32 | using std::string; 33 | 34 | // Flags that determine which analyzer to use and the options with which to 35 | // invoke that analyzer. 36 | DEFINE_string(analysis_options, "", "Analysis options as a protocol buffer."); 37 | 38 | // Returns true and logs an error message if 'value' is empty. This function is 39 | // called by InitGoogle. More complex validation takes place separately. 40 | static bool CheckFlagValueNotEmpty(const char* flagname, const string& value) { 41 | if (value.empty()) { 42 | std::cerr << flagname << " flag must be non-empty."; 43 | return false; 44 | } 45 | return true; 46 | } 47 | 48 | static bool is_config_empty = 49 | google::RegisterFlagValidator(&FLAGS_analysis_options, &CheckFlagValueNotEmpty); 50 | 51 | int main(int argc, char** argv) { 52 | gflags::ParseCommandLineFlags(&argc, &argv, true); 53 | morphie::AnalysisOptions options; 54 | string out; 55 | if (!protobuf::TextFormat::ParseFromString(FLAGS_analysis_options, 56 | &options)) { 57 | std::cerr << "The 'analysis_options' flag must have the format of an " 58 | "AnalysisOptions proto."; 59 | return -1; 60 | } 61 | morphie::util::Status status = morphie::frontend::Run(options); 62 | if (!status.ok()) { 63 | std::cerr << status.message(); 64 | return -1; 65 | } 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Download and install Google Test libraries. 2 | # 3 | # Create a directory called 'googletest' in the build area and copy the file 4 | # CMakeLists.txt.googletest there. 5 | configure_file(CMakeLists.txt.googletest 6 | googletest/CMakeLists.txt) 7 | # Invoke a separate run of cmake as an external command on the copied 8 | # CMakeLists.txt file. This invocation ensures that googletest is unpacked and 9 | # installed during configuration time. If the command below is not invoked, a 10 | # separate script would have to be run to go through the build steps for 11 | # dependencies in the right order. 12 | set(gtest_download_dir "${CMAKE_BINARY_DIR}/third_party/googletest") 13 | execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 14 | WORKING_DIRECTORY ${gtest_download_dir} ) 15 | execute_process(COMMAND ${CMAKE_COMMAND} --build . 16 | WORKING_DIRECTORY ${gtest_download_dir} ) 17 | # Adds the build options 'gtest' and 'gtest_main', which are defined in the 18 | # CMakeLists file of googletest to this build. 19 | add_compile_options(-fexceptions) 20 | add_subdirectory(${CMAKE_BINARY_DIR}/googletest-src 21 | ${CMAKE_BINARY_DIR}/googletest-build) 22 | 23 | # Download and install the open source JSON parser. 24 | # 25 | # Create a directory called 'jsoncpp' in the build area and copy the file 26 | # CMakeLists.txt.jsoncpp there. 27 | configure_file(CMakeLists.txt.jsoncpp 28 | jsoncpp/CMakeLists.txt) 29 | message(STATUS "DOWNLOADING JSONCPP") 30 | set(jsoncpp_build_dir "${CMAKE_BINARY_DIR}/third_party/jsoncpp") 31 | message(STATUS "RAN JSONCPP CMAKE in ${jsoncpp_build_dir}") 32 | execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 33 | WORKING_DIRECTORY ${jsoncpp_build_dir} ) 34 | message(STATUS "RAN JSONCPP CMAKE in ${jsoncpp_build_dir}") 35 | set(CMAKE_INSTALL_COMPONENT "${jsoncpp_build_dir}") 36 | execute_process(COMMAND ${CMAKE_COMMAND} --build . 37 | WORKING_DIRECTORY ${jsoncpp_build_dir} ) 38 | message(STATUS "RAN JSONCPP MAKE") 39 | message(STATUS "DONE WITH JSON") 40 | 41 | # Download and install the gflags command line library. 42 | # 43 | # Create a directory called 'gflags' in the build area and copy the file 44 | # CMakeLists.txt.gflags there. 45 | configure_file(CMakeLists.txt.gflags 46 | gflags/CMakeLists.txt) 47 | set(gflags_download_dir "${CMAKE_BINARY_DIR}/third_party/gflags") 48 | execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 49 | WORKING_DIRECTORY ${gflags_download_dir} ) 50 | execute_process(COMMAND ${CMAKE_COMMAND} --build . 51 | WORKING_DIRECTORY ${gflags_download_dir} ) 52 | # Adds the build options 'gtest' and 'gtest_main', which are defined in the 53 | # CMakeLists file of googletest to this build. 54 | # add_subdirectory(${CMAKE_BINARY_DIR}/gflags-src 55 | # ${CMAKE_BINARY_DIR}/gflags-build) 56 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt.gflags: -------------------------------------------------------------------------------- 1 | # CMake configuration for downloading and installing gflags from GitHub. The 2 | # glflags was originally called "Google Commandline Flags" but is now an open 3 | # source project maintained by a developer not at Google. 4 | cmake_minimum_required(VERSION 2.8.12) 5 | 6 | project(gflags-download NONE) 7 | 8 | include(ExternalProject) 9 | ExternalProject_Add(gflags 10 | GIT_REPOSITORY https://github.com/gflags/gflags.git 11 | GIT_TAG master 12 | SOURCE_DIR "${CMAKE_BINARY_DIR}/gflags-src" 13 | BINARY_DIR "${CMAKE_BINARY_DIR}/gflags-build" 14 | INSTALL_DIR "${CMAKE_BINARY_DIR}/gflags-build" 15 | CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" 16 | ) 17 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt.googletest: -------------------------------------------------------------------------------- 1 | # CMake configuration for downloading and installing googletest from GitHub. 2 | cmake_minimum_required(VERSION 2.8.12) 3 | 4 | project(googletest-download NONE) 5 | 6 | include(ExternalProject) 7 | ExternalProject_Add(googletest 8 | GIT_REPOSITORY https://github.com/google/googletest.git 9 | GIT_TAG master 10 | SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src" 11 | BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build" 12 | CONFIGURE_COMMAND "" 13 | BUILD_COMMAND "" 14 | INSTALL_COMMAND "" 15 | TEST_COMMAND "" 16 | ) 17 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt.jsoncpp: -------------------------------------------------------------------------------- 1 | # CMake configuration for downloading and installing an open-source, JSON parser written in C++. 2 | cmake_minimum_required(VERSION 2.8.12) 3 | 4 | project(jsoncpp-download NONE) 5 | 6 | message(STATUS "IN JSONCPP DOWNLOAD FILE") 7 | # The install prefix is the location in which the static libraries 8 | # will be installed. 9 | set(jsoncpp_install_prefix "${CMAKE_BINARY_DIR}/jsoncpp-lib") 10 | 11 | # The ExternalProject CMake module provides functionality for 12 | # downloading and installing a library from a remote repository. In 13 | # this case, branch tagged 'master' from the jsoncpp GitHub 14 | # repository is downloaded. The ExternalProject_Add module also uses a 15 | # variable called INSTALL_DIR but setting this directory is not 16 | # sufficient to ensure that the libraries are installed in the desired 17 | # location because it does not cause the CMAKE_INSTALL_PREFIX variable 18 | # to be set. 19 | include(ExternalProject) 20 | ExternalProject_Add(jsoncpp-parser 21 | GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp.git 22 | GIT_TAG master 23 | SOURCE_DIR "${CMAKE_BINARY_DIR}/jsoncpp-src" 24 | BINARY_DIR "${CMAKE_BINARY_DIR}/jsoncpp-build" 25 | INSTALL_DIR "${CMAKE_BINARY_DIR}/jsoncpp-build" 26 | CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" 27 | ) 28 | -------------------------------------------------------------------------------- /util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Generic algorithmic and data structure utilities. 3 | 4 | add_library(util_csv csv.h csv.cc) 5 | target_compile_options(util_csv PRIVATE -fexceptions) 6 | 7 | add_library(util_logging STATIC logging.h logging.cc) 8 | 9 | add_library(util_map_utils STATIC map_utils.h) 10 | set_target_properties(util_map_utils PROPERTIES LINKER_LANGUAGE CXX) 11 | 12 | add_library(util_status STATIC status.h status.cc) 13 | 14 | add_library(util_string_utils STATIC string_utils.h string_utils.cc) 15 | 16 | add_library(util_time_utils STATIC time_utils.h time_utils.cc) 17 | 18 | -------------------------------------------------------------------------------- /util/csv.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #include "util/csv.h" 15 | #include // NOLINT 16 | #include // NOLINT 17 | 18 | #include 19 | 20 | // The Clang compiler does not support exceptions. Boost is compiled with Clang, 21 | // the client is required to define this function. 22 | // void boost::throw_exception(std::exception const& e) {} 23 | 24 | namespace morphie { 25 | namespace util { 26 | 27 | CSVParser::Iterator& CSVParser::Iterator::operator++() { 28 | parser_->Advance(); 29 | return *this; 30 | } 31 | 32 | CSVParser::CSVParser(std::istream* input) : CSVParser(input, ',') {} 33 | 34 | CSVParser::CSVParser(std::istream* input, char delim) 35 | : delim_(delim), 36 | begin_iter_(this, false), 37 | end_iter_(this, true), 38 | state_(State::kReading) { 39 | Init(input); 40 | } 41 | 42 | void CSVParser::Init(std::istream* input) { 43 | if (input == nullptr || !input->good()) { 44 | input_.reset(nullptr); 45 | state_ = State::kOutputEmpty; 46 | record_.status_ = util::Status(Code::INVALID_ARGUMENT, "Input is null."); 47 | return; 48 | } 49 | input_.reset(input); 50 | Advance(); 51 | } 52 | 53 | // The parser begins in the kReading state and makes the following state 54 | // transitions. 55 | // - kReading -> kInputEmpty if there is exactly one line left in the input. 56 | // - kReading -> kReading if there is more than one line in the input. In both 57 | // transitions out of kReading, the next input line is parsed, so the output 58 | // is not empty. 59 | // - kInputEmpty -> kOutputEmpty. 60 | // - kOutputEmpty-> kOutputEmpty. 61 | // The transitions from kInputEmpty and kOutputEmpty are not based on other 62 | // conditions. 63 | void CSVParser::Advance() { 64 | record_.fields_.clear(); 65 | switch (state_) { 66 | case State::kOutputEmpty: 67 | return; 68 | case State::kInputEmpty: { 69 | state_ = State::kOutputEmpty; 70 | return; 71 | } 72 | case State::kReading: { 73 | string line; 74 | std::getline(*input_, line); 75 | try { 76 | boost::escaped_list_separator sep("\\", string(1, delim_), "\""); 77 | boost::tokenizer> tokenizer(line, 78 | sep); 79 | record_.fields_.assign(tokenizer.begin(), tokenizer.end()); 80 | } catch (boost::exception& e) { 81 | record_.status_ = 82 | util::Status(Code::INVALID_ARGUMENT, "Error tokenizing line."); 83 | } 84 | if (!input_->good()) { 85 | input_.reset(nullptr); 86 | state_ = State::kInputEmpty; 87 | } 88 | } 89 | } 90 | } 91 | 92 | } // namespace util 93 | } // namespace morphie 94 | -------------------------------------------------------------------------------- /util/json_reader.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "base/string.h" 5 | #include "json_reader.h" 6 | #include "util/logging.h" 7 | 8 | namespace morphie { 9 | 10 | std::unique_ptr GetJsonDoc(std::istream* json_stream) { 11 | Json::Reader json_reader; 12 | std::unique_ptr json_doc(new Json::Value); 13 | bool success = json_reader.parse( 14 | *json_stream, *json_doc, false /*Do not parse comments*/); 15 | CHECK(success, "File is not in JSON format."); 16 | CHECK(json_doc->isObject(), "JSON object is not a dict."); 17 | return json_doc; 18 | } 19 | 20 | FullJson::FullJson(std::istream* json_stream) { 21 | json_doc_ = GetJsonDoc(json_stream); 22 | members_ = json_doc_->getMemberNames(); 23 | } 24 | 25 | bool FullJson::HasNext() { 26 | return current_index_ < members_.size(); 27 | } 28 | 29 | const Json::Value* FullJson::Next() { 30 | CHECK(HasNext(), "Called Next after last object is received."); 31 | std::string object_id = members_[current_index_]; 32 | ++current_index_; 33 | Json::Value* json_object; 34 | json_object = &((*json_doc_)[object_id]); 35 | return json_object; 36 | } 37 | 38 | FullJson::~FullJson() { 39 | } 40 | 41 | StreamJson::StreamJson(std::istream* json_stream) { 42 | input_file_ = json_stream; 43 | } 44 | 45 | bool StreamJson::HasNext() { 46 | return input_file_->peek() != '\n' && !input_file_->eof(); 47 | } 48 | 49 | const Json::Value* StreamJson::Next() { 50 | CHECK(HasNext(), "Called Next at the end of a stream."); 51 | 52 | Json::Reader json_reader; 53 | std::string line; 54 | getline(*(input_file_), line); 55 | 56 | bool success = json_reader.parse( 57 | line.c_str(), current_object_, false /*Do not parse comments*/); 58 | CHECK(success, "Line is not in JSON format"); 59 | 60 | return &(current_object_); 61 | } 62 | 63 | StreamJson::~StreamJson() { 64 | } 65 | 66 | } // namespace morphie 67 | -------------------------------------------------------------------------------- /util/json_reader.h: -------------------------------------------------------------------------------- 1 | #ifndef LOGLE_JSON_READER_H 2 | #define LOGLE_JSON_READER_H 3 | 4 | #include 5 | #include 6 | 7 | #include "json/json.h" 8 | 9 | namespace morphie { 10 | 11 | // Contains classes that handle loading of input files in different formats. 12 | // It supports JSON and JSON stream format. 13 | // 14 | // JSON format consists of one monolithic JSON object (dict of objects). 15 | // It is easier to use, but parsing a JSON file has big memory overhead and 16 | // limits usage of big input files. Moreover we do not need all fields from 17 | // objects, so there is no need to have them in memory for more time than 18 | // necessary. 19 | 20 | // Each line of JSON stream file contains one JSON object. 21 | // This can be easily read and parsed one line at a time and we only 22 | // need to store one object at a time. 23 | 24 | // Interface that handles loading of input JSON file. 25 | class JsonDocumentIterator{ 26 | public: 27 | virtual ~JsonDocumentIterator() {} 28 | 29 | // Returns whether Next() could be called. 30 | // This will not change the iterator. 31 | virtual bool HasNext() = 0; 32 | // Returns current object and advances the iterator. 33 | // The iterator initially points to the first object. 34 | virtual const Json::Value* Next() = 0; 35 | }; 36 | 37 | // Support for loading objects from one monolithic JSON object. 38 | // This is memory demanding but widely used format. 39 | // Example: 40 | // {"object1": {"field1": "value1"}, 41 | // "object2": {"field1": "value2"} 42 | // } 43 | class FullJson: public JsonDocumentIterator{ 44 | public: 45 | FullJson(std::istream* json_stream); 46 | ~FullJson(); 47 | bool HasNext(); 48 | const Json::Value* Next(); 49 | private: 50 | std::unique_ptr json_doc_; 51 | // Contains object names (keys from JSON dict). 52 | Json::Value::Members members_; 53 | // Index to members_ specifying the name of next object. 54 | int current_index_ = 0; 55 | }; 56 | 57 | // Support for JSON stream format. 58 | // Each line of input file is one JSON encoded object 59 | // Example: 60 | // {"field1": "value1"} 61 | // {"field1": "value2"} 62 | // 63 | class StreamJson: public JsonDocumentIterator{ 64 | public: 65 | StreamJson(std::istream* json_stream); 66 | ~StreamJson(); 67 | bool HasNext(); 68 | const Json::Value* Next(); 69 | private: 70 | std::istream* input_file_; 71 | // contains last read JSON object 72 | Json::Value current_object_; 73 | }; 74 | 75 | } // namespace morphie 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "util/logging.h" 16 | 17 | #include // for assert() 18 | #include // for abort() 19 | #include // for std::cerr 20 | 21 | namespace morphie { 22 | namespace util { 23 | 24 | void Check(bool condition, const string& location, const string& err) { 25 | if (!condition) { 26 | std::cerr << location << ": " << err; 27 | std::abort(); 28 | } 29 | } 30 | 31 | void Check(bool condition, const string& location) { 32 | Check(condition, location, ""); 33 | } 34 | 35 | void Check(bool condition) { Check(condition, "", ""); } 36 | 37 | } // namespace util 38 | } // namespace morphie 39 | -------------------------------------------------------------------------------- /util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file defines a function for checking assertions and printing an error 16 | // if the assertion fails. 17 | #ifndef LOGLE_UTIL_LOGGING_H_ 18 | #define LOGLE_UTIL_LOGGING_H_ 19 | 20 | #include "base/string.h" 21 | 22 | #define MAKE_STR(x) #x 23 | #define TOSTRING(x) MAKE_STR(x) 24 | #define LOCATION_STR __FILE__ ":" TOSTRING(__LINE__) 25 | #define CHECK(c, err) util::Check(c, LOCATION_STR, err) 26 | #define FAIL(err) util::Check(false, LOCATION_STR, err) 27 | 28 | namespace morphie { 29 | namespace util { 30 | 31 | // Produces an error message and aborts if the condition is false. 32 | void Check(bool condition, const string& location, const string& err); 33 | void Check(bool condition, const string& location); 34 | void Check(bool condition); 35 | 36 | } // namespace util 37 | } // namespace morphie 38 | 39 | #endif // LOGLE_UTIL_LOGGING_H_ 40 | -------------------------------------------------------------------------------- /util/map_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Utilities for implementing simple mathematical operations on maps. 16 | #ifndef LOGLE_UTIL_MAP_UTILS_H_ 17 | #define LOGLE_UTIL_MAP_UTILS_H_ 18 | 19 | #include 20 | #include 21 | 22 | namespace morphie { 23 | namespace util { 24 | 25 | // The type unordered_map can be viewed as a partial function from values 26 | // of type A to values of type B. The composition of f: A --> B with g: B --> C 27 | // is the function h : A --> C that maps each element 'a' in the domain of 'f' 28 | // to the element 'g(f(a))' provided 'f(a)' is in the domain of 'g'. The C++ 29 | // functions below assume that the domain and the range of the function are over 30 | // the same set. 31 | // 32 | // Returns the composition of the keys map with the values map. The result 33 | // contains keys from the first map and values from the second map. 34 | template 35 | std::unordered_map Compose( 36 | const std::unordered_map& keys, 37 | const std::unordered_map& values) { 38 | std::unordered_map composition; 39 | for (const auto& key_pair : keys) { 40 | auto value_it = values.find(key_pair.second); 41 | if (value_it == values.end()) { 42 | continue; 43 | } 44 | composition.insert({key_pair.first, value_it->second}); 45 | } 46 | return composition; 47 | } 48 | 49 | // The preimage of a function f : A --> B is a function from B to the powerset 50 | // of A that maps each element 'b' to the set {a | f(a) = b}. 51 | // 52 | // Returns the preimage of the input function. 53 | template 54 | std::unordered_map> Preimage( 55 | const std::unordered_map& fn) { 56 | std::unordered_map> preimage; 57 | for (const auto& fn_pair : fn) { 58 | // If the insertion below fails, there is already an entry for this value in 59 | // the preimage map, so the key has to be inserted into the preimage set. 60 | auto insert_pair = preimage.insert({fn_pair.second, {fn_pair.first}}); 61 | if (insert_pair.second == false) { 62 | insert_pair.first->second.insert(fn_pair.first); 63 | } 64 | } 65 | return preimage; 66 | } 67 | 68 | } // namespace util 69 | } // namespace morphie 70 | 71 | #endif // LOGLE_UTIL_MAP_UTILS_H_ 72 | -------------------------------------------------------------------------------- /util/map_utils_test.cc: -------------------------------------------------------------------------------- 1 | #include "util/map_utils.h" 2 | 3 | #include "gtest.h" 4 | 5 | namespace morphie { 6 | namespace util { 7 | 8 | namespace { 9 | 10 | TEST(MapCompositionTest, EmptyArgumentProducesEmptyMap) { 11 | std::unordered_map empty_map; 12 | std::unordered_map one_two{{1, 2}}; 13 | // If one of the maps to compose with is empty, the composition is empty. 14 | EXPECT_TRUE(Compose(empty_map, empty_map).empty()); 15 | EXPECT_TRUE(Compose(empty_map, one_two).empty()); 16 | EXPECT_TRUE(Compose(one_two, empty_map).empty()); 17 | // If no value of the first map is a key of the second map, the composition is 18 | // empty. 19 | EXPECT_TRUE(Compose(one_two, one_two).empty()); 20 | } 21 | 22 | // The composition of identity maps should be the identity map over the common 23 | // set of keys and values. 24 | TEST(MapCompositionTest, CompositionOfIdentityIsIdentity) { 25 | std::unordered_map identity_02{{0, 0}, {1, 1}, {2, 2}}; 26 | std::unordered_map identity_01{{0, 0}, {1, 1}}; 27 | std::unordered_map identity_12{{1, 1}, {2, 2}}; 28 | std::unordered_map identity_11{{1, 1}}; 29 | EXPECT_EQ(identity_02, Compose(identity_02, identity_02)); 30 | // If the entries in one identity map are contained in the other, the result 31 | // is the smaller identity map. 32 | EXPECT_EQ(identity_01, Compose(identity_02, identity_01)); 33 | EXPECT_EQ(identity_01, Compose(identity_01, identity_02)); 34 | // The composition only has values in the first that appear as keys in the 35 | // second. 36 | EXPECT_EQ(identity_11, Compose(identity_01, identity_12)); 37 | } 38 | 39 | // A map 'm' is injective if for all distinct keys 'a' and 'b', the values 40 | // 'm[a]' and 'm[b]' are also different. 41 | TEST(MapCompositionTest, CompositionOfInjectiveMaps) { 42 | std::unordered_map increments{{0, 1}, {1, 2}, {2, 3}}; 43 | std::unordered_map decrements{{1, 0}, {2, 1}, {3, 2}}; 44 | std::unordered_map identity_02{{0, 0}, {1, 1}, {2, 2}}; 45 | EXPECT_EQ(identity_02, Compose(increments, decrements)); 46 | } 47 | 48 | TEST(MapPreimageTest, PreimageProperties) { 49 | // The preimage of the empty function is an empty function. 50 | std::unordered_map empty_map; 51 | std::unordered_map> empty_pre; 52 | EXPECT_EQ(empty_pre, Preimage(empty_map)); 53 | // An injective function will have singleton preimages. 54 | std::unordered_map increments{{0, 1}, {1, 2}, {2, 3}}; 55 | std::unordered_map> pre_increments{ 56 | {1, {0}}, {2, {1}}, {3, {2}}}; 57 | EXPECT_EQ(pre_increments, Preimage(increments)); 58 | // A non-injective function will have non-singleton preimages. 59 | std::unordered_map parity{{0, 0}, {1, 1}, {2, 0}, {3, 1}}; 60 | std::unordered_map> pre_parity{{0, {0, 2}}, 61 | {1, {1, 3}}}; 62 | EXPECT_EQ(pre_parity, Preimage(parity)); 63 | } 64 | 65 | } // namespace 66 | } // namespace util 67 | } // namespace morphie 68 | -------------------------------------------------------------------------------- /util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #include "util/status.h" 15 | 16 | namespace morphie { 17 | namespace util { 18 | 19 | namespace { 20 | 21 | const Status& GetOkStatus() { 22 | static const Status status; 23 | return status; 24 | } 25 | 26 | } // unnamed namespace 27 | 28 | const Status& Status::OK = GetOkStatus(); 29 | 30 | } // namespace util 31 | } // namespace morphie 32 | -------------------------------------------------------------------------------- /util/status.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #ifndef LOGLE_UTIL_STATUS_H_ 15 | #define LOGLE_UTIL_STATUS_H_ 16 | 17 | // This file defines a Status class for representing the success of an 18 | // operation. A function that operates successfully returns an OK status object. 19 | // A function whose operation fails returns a non-OK object, which contains 20 | // details about why the operation failed. 21 | #include "base/string.h" 22 | 23 | namespace morphie { 24 | enum class Code { 25 | // No error. Returned on success. 26 | OK = 0, 27 | // The error is caused by the client providing invalid data. 28 | INVALID_ARGUMENT = 1, 29 | // An invariant of the system has been violated. 30 | INTERNAL = 2, 31 | // The error is caused by interaction with an external system. For example, an 32 | // attempt to open a file may have failed. 33 | EXTERNAL = 3 34 | }; 35 | 36 | namespace util { 37 | 38 | // A Status object consists of an error code and an error message. This class 39 | // adopts the convention that a status object with code Code::OK has no message 40 | // and also defines a unique OK status object. 41 | class Status { 42 | public: 43 | // Creates a status object with code OK and an empty message. 44 | Status() : code_(Code::OK), message_("") {} 45 | 46 | // Creates a status object with the provided code. Sets the status message to 47 | // the second argument if the code is not Code::OK and ignores the second 48 | // argument otherwise. 49 | Status(Code code, const string& message) : code_(code), message_(message) { 50 | if (code_ == Code::OK) { 51 | message_.clear(); 52 | } 53 | } 54 | 55 | // Copy constructor. 56 | Status(const Status& other) : code_(other.code_), message_(other.message_) {} 57 | 58 | // Assignment operator. 59 | Status& operator=(const Status& other) { 60 | code_ = other.code_; 61 | message_ = other.message_; 62 | return *this; 63 | } 64 | 65 | // A status object with code OK and an empty message. 66 | static const Status& OK; 67 | 68 | // Returns true if the status code is OK. 69 | bool ok() const { return code_ == Code::OK; } 70 | 71 | Code code() const { return code_; } 72 | const string& message() const { return message_; } 73 | 74 | private: 75 | Code code_; 76 | string message_; 77 | }; 78 | 79 | } // namespace util 80 | } // namespace morphie 81 | 82 | #endif // LOGLE_UTIL_STATUS_H_ 83 | -------------------------------------------------------------------------------- /util/status_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #include "util/status.h" 15 | 16 | #include "gtest.h" 17 | 18 | namespace morphie { 19 | namespace util { 20 | namespace { 21 | 22 | TEST(StatusTest, MemberFunctions) { 23 | Status s; 24 | // The default status object has the OK error code and empty message. 25 | EXPECT_TRUE(s.ok()); 26 | EXPECT_EQ(Code::OK, s.code()); 27 | EXPECT_EQ("", s.message()); 28 | // Status::OK has the same properties as the default status object. 29 | EXPECT_TRUE(Status::OK.ok()); 30 | EXPECT_EQ(Code::OK, Status::OK.code()); 31 | EXPECT_EQ("", Status::OK.message()); 32 | // Construct an error object with code different from OK. 33 | s = Status(Code::INTERNAL, "Internal error."); 34 | EXPECT_FALSE(s.ok()); 35 | EXPECT_EQ(Code::INTERNAL, s.code()); 36 | EXPECT_EQ("Internal error.", s.message()); 37 | } 38 | 39 | } // unnamed namespace 40 | } // namespace util 41 | } // namespace morphie 42 | -------------------------------------------------------------------------------- /util/string_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #include "util/string_utils.h" 15 | 16 | namespace morphie { 17 | namespace util { 18 | 19 | std::set SplitToSet(const string& str, char delim) { 20 | std::vector vec = SplitToVector(str, delim); 21 | return std::set(vec.begin(), vec.end()); 22 | } 23 | 24 | std::vector SplitToVector(const string& str, char delim) { 25 | if (str.empty()) { 26 | return {""}; 27 | } 28 | std::vector strings; 29 | std::stringstream stream(str); 30 | string token; 31 | while (std::getline(stream, token, delim)) { 32 | strings.push_back(token); 33 | } 34 | if (str.back() == delim) { 35 | strings.push_back(""); 36 | } 37 | return strings; 38 | } 39 | 40 | string StrCat(const string& str1, const string& str2) { 41 | return StrCat(str1, str2, "", "", "", ""); 42 | } 43 | 44 | string StrCat(const string& str1, const string& str2, const string& str3) { 45 | return StrCat(str1, str2, str3, "", "", ""); 46 | } 47 | 48 | string StrCat(const string& str1, const string& str2, const string& str3, 49 | const string& str4) { 50 | return StrCat(str1, str2, str3, str4, "", ""); 51 | } 52 | 53 | string StrCat(const string& str1, const string& str2, const string& str3, 54 | const string& str4, const string& str5) { 55 | return StrCat(str1, str2, str3, str4, str5, ""); 56 | } 57 | 58 | string StrCat(const string& str1, const string& str2, const string& str3, 59 | const string& str4, const string& str5, const string& str6) { 60 | string out; 61 | StrAppend(&out, str1, str2, str3, str4, str5, str6); 62 | return out; 63 | } 64 | 65 | void StrAppend(string* out, const string& str1) { 66 | StrAppend(out, str1, "", "", "", "", ""); 67 | } 68 | 69 | void StrAppend(string* out, const string& str1, const string& str2) { 70 | StrAppend(out, str1, str2, "", "", "", ""); 71 | } 72 | 73 | void StrAppend(string* out, const string& str1, const string& str2, 74 | const string& str3) { 75 | StrAppend(out, str1, str2, str3, "", "", ""); 76 | } 77 | 78 | void StrAppend(string* out, const string& str1, const string& str2, 79 | const string& str3, const string& str4) { 80 | StrAppend(out, str1, str2, str3, str4, "", ""); 81 | } 82 | 83 | void StrAppend(string* out, const string& str1, const string& str2, 84 | const string& str3, const string& str4, const string& str5) { 85 | StrAppend(out, str1, str2, str3, str4, str5, ""); 86 | } 87 | 88 | void StrAppend(string* out, const string& str1, const string& str2, 89 | const string& str3, const string& str4, const string& str5, 90 | const string& str6) { 91 | out->reserve(out->size() + str1.size() + str2.size() + str3.size() + 92 | str4.size() + str5.size() + str6.size()); 93 | *out += str1; 94 | *out += str2; 95 | *out += str3; 96 | *out += str4; 97 | *out += str5; 98 | *out += str6; 99 | } 100 | 101 | } // namespace util 102 | } // namespace morphie 103 | -------------------------------------------------------------------------------- /util/string_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // Utilities for manipulating strings. 16 | #ifndef LOGLE_UTIL_STRING_UTILS_H_ 17 | #define LOGLE_UTIL_STRING_UTILS_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "base/string.h" 24 | 25 | namespace morphie { 26 | namespace util { 27 | 28 | // Returns a container of strings obtained by splitting 'str' using the 29 | // delimiter provided. 30 | std::set SplitToSet(const string& str, char delim); 31 | // Some examples to illustrate the semantics. 32 | // SplitToVector("", ',') == {""} 33 | // SplitToVector("abc", ',') == {"abc"} 34 | // SplitToVector("/usr/local/bin", '/') == {"", "local", "bin"} 35 | // SplitToVector("/usr/local/bin/", '/') == {"", "local", "bin", ""} 36 | std::vector SplitToVector(const string& str, char delim); 37 | 38 | // The StrCat functions return the concatenation of their arguments. 39 | string StrCat(const string& str1, const string& str2); 40 | string StrCat(const string& str1, const string& str2, const string& str3); 41 | string StrCat(const string& str1, const string& str2, const string& str3, 42 | const string& str4); 43 | string StrCat(const string& str1, const string& str2, const string& str3, 44 | const string& str4, const string& str5); 45 | string StrCat(const string& str1, const string& str2, const string& str3, 46 | const string& str4, const string& str5, const string& str6); 47 | 48 | // The StrAppend functions append the second and subsequent arguments to the 49 | // string in the first argument. These functions should not be called with the 50 | // output string as one of the second or other arguments. Please avoid 51 | // anti-patterns like this. 52 | // StrAppend(&out, str1, out); 53 | void StrAppend(string* out, const string& str1); 54 | void StrAppend(string* out, const string& str1, const string& str2); 55 | void StrAppend(string* out, const string& str1, const string& str2, 56 | const string& str3); 57 | void StrAppend(string* out, const string& str1, const string& str2, 58 | const string& str3, const string& str4); 59 | void StrAppend(string* out, const string& str1, const string& str2, 60 | const string& str3, const string& str4, const string& str5); 61 | void StrAppend(string* out, const string& str1, const string& str2, 62 | const string& str3, const string& str4, const string& str5, 63 | const string& str6); 64 | 65 | // Returns a string equivalent to the concatenation of strings in 'args' using 66 | // 'sep' as a separator. Assumes that stream operators can be used to obtain a 67 | // serialization of elements of type 'T'. 68 | template 69 | string SetJoin(const std::set& args, const string& sep) { 70 | if (args.empty()) { 71 | return ""; 72 | } 73 | 74 | std::stringstream args_str; 75 | auto set_it = args.begin(); 76 | args_str << *set_it; 77 | for (++set_it; set_it != args.end(); ++set_it) { 78 | args_str << sep << *set_it; 79 | } 80 | return args_str.str(); 81 | } 82 | 83 | } // namespace util 84 | } // namespace morphie 85 | 86 | #endif // LOGLE_UTIL_STRING_UTILS_H_ 87 | -------------------------------------------------------------------------------- /util/string_utils_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | #ifndef LOGLE_UTIL_STRING_UTILS_TEST_H_ 15 | #define LOGLE_UTIL_STRING_UTILS_TEST_H_ 16 | 17 | #include "util/string_utils.h" 18 | 19 | #include 20 | 21 | #include "base/string.h" 22 | #include "gtest.h" 23 | 24 | namespace morphie { 25 | namespace util { 26 | namespace { 27 | 28 | TEST(VectorSplitTest, Correctness) { 29 | std::vector result; 30 | // Empty string. 31 | result = {""}; 32 | EXPECT_EQ(result, SplitToVector("", '.')); 33 | // String with no delimiters. 34 | result = {"abc"}; 35 | EXPECT_EQ(result, SplitToVector("abc", '.')); 36 | // Delimiters only. 37 | result = {"", ""}; 38 | EXPECT_EQ(result, SplitToVector(",", ',')); 39 | result = {"", "a"}; 40 | EXPECT_EQ(result, SplitToVector(",a", ',')); 41 | result = {"", "a", ""}; 42 | EXPECT_EQ(result, SplitToVector(",a,", ',')); 43 | result = {"a", "b", "a"}; 44 | EXPECT_EQ(result, SplitToVector("a/b/a", '/')); 45 | result = {"a", "b", "a", ""}; 46 | EXPECT_EQ(result, SplitToVector("a/b/a/", '/')); 47 | result = {"", "a", "b", "a", ""}; 48 | EXPECT_EQ(result, SplitToVector("/a/b/a/", '/')); 49 | // Empty string between non-empty input. The test string is: 50 | // "a[space][space]b[space]c[space]" 51 | result = {"a", "", "b", "c", ""}; 52 | EXPECT_EQ(result, SplitToVector("a b c ", ' ')); 53 | } 54 | 55 | TEST(SetSplitTest, Correctness) { 56 | std::set result; 57 | // Empty string. 58 | result = {""}; 59 | EXPECT_EQ(result, SplitToSet("", '.')); 60 | // Delimiters only. 61 | result = {""}; 62 | EXPECT_EQ(result, SplitToSet(",,", ',')); 63 | result = {"a", "b"}; 64 | EXPECT_EQ(result, SplitToSet("a/b/a", '/')); 65 | // Empty string between non-empty input. 66 | result = {"a", "", "b"}; 67 | EXPECT_EQ(result, SplitToSet("a b a ", ' ')); 68 | } 69 | 70 | TEST(StrCatTest, Correctness) { 71 | // Empty strings. 72 | EXPECT_EQ("", StrCat("", "")); 73 | EXPECT_EQ("", StrCat("", "", "")); 74 | EXPECT_EQ("", StrCat("", "", "", "")); 75 | EXPECT_EQ("", StrCat("", "", "", "", "")); 76 | // Combination of non-empty and empty strings. 77 | EXPECT_EQ("a", StrCat("a", "")); 78 | EXPECT_EQ("a", StrCat("", "a")); 79 | EXPECT_EQ("a", StrCat("", "a", "")); 80 | EXPECT_EQ("ab", StrCat("", "a", "b", "")); 81 | EXPECT_EQ("abc", StrCat("a", "", "b", "", "c")); 82 | // Non-empty strings. 83 | EXPECT_EQ("ab", StrCat("a", "b")); 84 | EXPECT_EQ("abc", StrCat("a", "b", "c")); 85 | EXPECT_EQ("abcd", StrCat("a", "b", "c", "d")); 86 | EXPECT_EQ("abcde", StrCat("a", "b", "c", "d", "e")); 87 | EXPECT_EQ("abcdef", StrCat("a", "b", "c", "d", "e", "f")); 88 | } 89 | 90 | TEST(StrAppendTest, Correctness) { 91 | string out; 92 | // Appending single characters. 93 | StrAppend(&out, ""); 94 | EXPECT_EQ("", out); 95 | StrAppend(&out, "a"); 96 | EXPECT_EQ("a", out); 97 | StrAppend(&out, "b"); 98 | EXPECT_EQ("ab", out); 99 | // Exercise every variant of the StrAppend function. 100 | out.clear(); 101 | StrAppend(&out, "a", "b"); 102 | EXPECT_EQ("ab", out); 103 | out.clear(); 104 | StrAppend(&out, "a", "b", "c"); 105 | EXPECT_EQ("abc", out); 106 | out.clear(); 107 | StrAppend(&out, "a", "b", "c", "d"); 108 | EXPECT_EQ("abcd", out); 109 | out.clear(); 110 | StrAppend(&out, "a", "b", "c", "d", "e"); 111 | EXPECT_EQ("abcde", out); 112 | out.clear(); 113 | StrAppend(&out, "a", "b", "c", "d", "e", "f"); 114 | EXPECT_EQ("abcdef", out); 115 | // The next few tests demonstrate anti-patterns. Where the string being 116 | // appended to is passed as data as well. 117 | // This will work because 'out' occurs only once as an argument. 118 | out = "ab"; 119 | StrAppend(&out, out); 120 | EXPECT_EQ("abab", out); 121 | // These do not work because the string is modified as it is being read for 122 | // each argument. 123 | out.clear(); 124 | StrAppend(&out, out, "ab", out); 125 | EXPECT_NE("ab", out); 126 | } 127 | 128 | TEST(SetJoinTest, OutputProperties) { 129 | // Inputs that map to the empty string. 130 | // Sets that map to the empty string. 131 | // The empty set. 132 | EXPECT_EQ("", SetJoin(std::set{}, ",")); 133 | EXPECT_EQ("", SetJoin(std::set{}, ",")); 134 | // A set containing only the empty string. 135 | EXPECT_EQ("", SetJoin(std::set{""}, ",")); 136 | EXPECT_EQ("", SetJoin(std::set{"", ""}, "")); 137 | EXPECT_EQ("", SetJoin(std::set{"", ""}, ", ")); 138 | // Inputs mapping to output with no delimiters. 139 | EXPECT_EQ("a", SetJoin(std::set{"a"}, ",")); 140 | EXPECT_EQ("a", SetJoin(std::set{"a", "a"}, ",")); 141 | EXPECT_EQ("aa", SetJoin(std::set{"aa"}, ",")); 142 | EXPECT_EQ("1", SetJoin(std::set{1}, ",")); 143 | // Inputs mapping to outputs with delimiters. 144 | EXPECT_EQ("a,b", SetJoin(std::set{"a", "b"}, ",")); 145 | EXPECT_EQ("a, b, c", SetJoin(std::set{"a", "b", "c"}, ", ")); 146 | EXPECT_EQ(", a", SetJoin(std::set{"", "a"}, ", ")); 147 | // std::set is ordered, so the empty string appears first in the next output. 148 | EXPECT_EQ(", a", SetJoin(std::set{"a", ""}, ", ")); 149 | EXPECT_EQ("1, 2", SetJoin(std::set{1, 2}, ", ")); 150 | } 151 | 152 | } // anonymous namespace 153 | } // namespace util 154 | } // namespace morphie 155 | 156 | #endif // LOGLE_UTIL_STRING_UTILS_TEST_H_ 157 | -------------------------------------------------------------------------------- /util/time_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // There are different types for representing time in C++. 16 | // - std::time_t is a value type supporting arithmetic that represents a 17 | // duration since the start of an unspecified epoch. This type is inherited 18 | // from the C time libraries. The unit for time_t is unspecified in the 19 | // standard. It often represents seconds since the start of the Unix epoch. 20 | // - std::tm is a struct, also from the C time library, for representing civil 21 | // time. It contains fields for the year, month, day, hour, minute, second and 22 | // daylight savings time but does not have a timezone field. 23 | // - std::chrono:time_point is a C++11 type with explicit representation of the 24 | // underlying clock and duration. 25 | // The C++11 standard defines the functions std::put_time and std::get_time, 26 | // which allow for parsing and printing civil time and take into account 27 | // timezones and daylight savings time. However, these functions were only 28 | // implemented in GCC 5. To avoid requiring that this code be compiled with GCC 29 | // versions 5 and above, C time function are used to parse and print time. 30 | // These functions ignore timezone information when parsing and use an 31 | // inaccessible value for the local timezone when converting civil time to 32 | // absolute time. The code below attempts to infer the implicit timezone and 33 | // daylight savings information when possible. 34 | #include "util/time_utils.h" 35 | 36 | #include 37 | #include 38 | 39 | namespace morphie { 40 | namespace util { 41 | 42 | // Uses the function gmtime_r to convert time in seconds to civil time in UTC 43 | // and strftime to print this time. 44 | string UnixMicrosToRFC3339(int64_t unix_micros) { 45 | std::tm tm; 46 | std::time_t unix_secs = unix_micros / 1000000; 47 | if (unix_micros % 1000000 < 0) { 48 | unix_secs--; 49 | } 50 | std::tm* tmp = gmtime_r(&unix_secs, &tm); 51 | char buf[sizeof("9223372036854775807-12-31T23:59:59+00:00")]; 52 | std::strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%S+00:00", tmp); 53 | return buf; 54 | } 55 | 56 | // Uses strptime to parse time_str and populate the fields in tm. Explicitly 57 | // extracts the timezone offset, which is not parsed by strptime. 58 | bool RFC3339ToUnixMicros(const string& time_str, int64_t* unix_micros) { 59 | std::tm tm; 60 | char* ap = const_cast(time_str.c_str()); 61 | char* bp = strptime(ap, "%Y-%m-%dT%H:%M:%S", &tm); 62 | if (bp == nullptr) { 63 | return false; 64 | } 65 | if (*bp != '+' && *bp != '-') { 66 | return false; 67 | } 68 | int hour = 0; 69 | int minute = 0; 70 | int pos = 0; 71 | int nitems = sscanf(bp, "%d:%d%n", &hour, &minute, &pos); 72 | if (nitems != 2 && nitems != 3) { 73 | return false; 74 | } 75 | if (pos != 6 || bp[pos] != '\0') { 76 | return false; 77 | } 78 | int sign = -1; 79 | if (hour < 0) { 80 | hour = -hour; 81 | sign = 1; 82 | } 83 | if (hour > 23 || minute < 0 || minute > 59) { 84 | return false; 85 | } 86 | // Convert civil time to absolute time. 87 | int64_t seconds = tm.tm_sec; 88 | seconds += tm.tm_min * 60; 89 | seconds += tm.tm_hour * 3600; 90 | seconds += tm.tm_yday * 86400; 91 | seconds += (tm.tm_year - 70) * 31536000; 92 | seconds += ((tm.tm_year - 69) / 4) * 86400; 93 | seconds += sign * (((hour * 60) + minute) * 60); 94 | *unix_micros = seconds * 1000000; 95 | return true; 96 | } 97 | 98 | } // namespace util 99 | } // namespace morphie 100 | -------------------------------------------------------------------------------- /util/time_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | // This file contains functions for converting between a civil time format and 16 | // an absolute time representation. Civil time refers to a calendar date and 17 | // clock time possibly augmented with timezone and daylight savings time 18 | // information. Civil time is ambiguous because timezone and daylight savings 19 | // information may be missing and because it is influenced by historical 20 | // adjustments. Absolute time is the number of ticks of a clock of fixed 21 | // duration since a uniquely chosen timepoint called an epoch. The functions in 22 | // this file convert between absolute time measured as microseconds since the 23 | // Unix epoch and civil time in RFC3339 format. 24 | #ifndef LOGLE_UTIL_TIME_UTILS_H_ 25 | #define LOGLE_UTIL_TIME_UTILS_H_ 26 | 27 | #include 28 | 29 | #include "base/string.h" 30 | 31 | namespace morphie { 32 | namespace util { 33 | 34 | // Returns an RFC3339 string in UTC corresponding to the number of microseconds 35 | // since the Unix epoch. 36 | string UnixMicrosToRFC3339(int64_t unix_micros); 37 | 38 | // Returns true and stores the microseconds since the Unix epoch in *unix_micros 39 | // if time_str could be parsed as an RFC3339 string. Returns false otherwise. 40 | bool RFC3339ToUnixMicros(const string& time_str, int64_t* unix_micros); 41 | 42 | } // namespace util 43 | } // namespace morphie 44 | 45 | #endif // LOGLE_UTIL_TIME_UTILS_H_ 46 | -------------------------------------------------------------------------------- /util/time_utils_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | #include "util/time_utils.h" 16 | 17 | #include 18 | 19 | #include "base/string.h" 20 | #include "gtest.h" 21 | 22 | namespace morphie { 23 | namespace util { 24 | namespace { 25 | 26 | TEST(TimeUtilsTest, RFC3339ToMicros) { 27 | // One microsecond after the epoch is still part of the 0-th second, but one 28 | // microsecond before the epoch is part of the previous second. 29 | EXPECT_EQ("1970-01-01T00:00:00+00:00", UnixMicrosToRFC3339(0)); 30 | EXPECT_EQ("1970-01-01T00:00:00+00:00", UnixMicrosToRFC3339(1)); 31 | EXPECT_EQ("1970-01-01T00:00:01+00:00", UnixMicrosToRFC3339(1000000)); 32 | EXPECT_EQ("1969-12-31T23:59:59+00:00", UnixMicrosToRFC3339(-1)); 33 | EXPECT_EQ("1969-12-31T23:59:59+00:00", UnixMicrosToRFC3339(-999999)); 34 | EXPECT_EQ("2016-02-10T23:56:41+00:00", UnixMicrosToRFC3339(1455148601000000)); 35 | } 36 | 37 | TEST(TimeUtilsTest, MicrosToRFC3339) { 38 | // Incomplete civil times or non-time strings cannot be parsed. 39 | int64_t micros; 40 | EXPECT_FALSE(RFC3339ToUnixMicros("", µs)); 41 | EXPECT_FALSE(RFC3339ToUnixMicros("foo", µs)); 42 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01", µs)); 43 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01T01:23", µs)); 44 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01T00:00:00+24:00", µs)); 45 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01T00:00:00-24:00", µs)); 46 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01T00:00:00+00:60", µs)); 47 | EXPECT_FALSE(RFC3339ToUnixMicros("1970-01-01T00:00:00+00:-2", µs)); 48 | // The start of the Unix epoch is absolute time 0. 49 | EXPECT_TRUE(RFC3339ToUnixMicros("1970-01-01T00:00:00+00:00", µs)); 50 | EXPECT_EQ(0, micros); 51 | EXPECT_TRUE(RFC3339ToUnixMicros("1970-01-01T00:00:01+00:00", µs)); 52 | EXPECT_EQ(1000000, micros); 53 | EXPECT_TRUE(RFC3339ToUnixMicros("1969-12-31T23:59:59+00:00", µs)); 54 | EXPECT_EQ(-1000000, micros); 55 | // 5:00PM is 7 hours before midnight, so the date below should be the 56 | // corresponding number of microseconds before absolute time 0. 57 | int64_t kSecsBeforeMidnight = 7 * 60 * 60; 58 | EXPECT_TRUE(RFC3339ToUnixMicros("1969-12-31T17:00:00+00:00", µs)); 59 | EXPECT_EQ(-1000000 * kSecsBeforeMidnight, micros); 60 | } 61 | 62 | } // anonymous namespace 63 | } // namespace util 64 | } // namespace morphie 65 | --------------------------------------------------------------------------------