├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── ci.yml ├── .gitignore ├── .gitmodules ├── .readthedocs.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── MAINTAINERS.md ├── Makefile ├── README.md ├── docs ├── _static │ ├── SF_Collector_Exporter.png │ ├── SF_Collector_Processor.png │ ├── SF_Object_View.png │ ├── SF_Object_View_v2.png │ ├── SF_Object_View_v5.png │ └── experimental.png ├── api-utils.rst ├── api.rst ├── binary.rst ├── build.rst ├── coc.rst ├── collector.rst ├── conf.py ├── contributing.rst ├── deploy.rst ├── docker.rst ├── exporter.rst ├── helm.rst ├── index.rst ├── libs.rst ├── license.rst ├── processor.rst ├── publications.rst ├── python-api.rst ├── quick.rst ├── refs.bib ├── requirements.txt └── spec.rst ├── make.bat └── update /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Indicate project** 11 | i.e. collector, exporter, deployment, etc. 12 | Note please also indicate project with correct issue label 13 | 14 | **Describe the bug** 15 | A clear and concise description of what the bug is. 16 | 17 | **To reproduce** 18 | Steps to reproduce the behavior: 19 | 1. '...' 20 | 2. '...' 21 | 3. '...' 22 | 23 | **Expected behavior** 24 | A clear and concise description of what you expected to happen. 25 | 26 | **Screenshots** 27 | If applicable, add screenshots to help explain your problem. 28 | 29 | **Environment (please complete the following information):** 30 | - OS: [e.g. ubuntu 16.04, kernel version] 31 | - SysFlow version: [e.g. v0.1, specify branch if applicable] 32 | - Command line arguments and filters (if applicable): 33 | - Configurations (if applicable): 34 | - Container orchestration framework and version (if applicable): [e.g kubernetes, openshift] 35 | - Container runtime (if applicable): [e.g. Docker, CRI-O] etc. 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | 40 | **Files** 41 | Attach any additional log files, config files, scripts or filters, sample sysflow, etc. that will help diagnose the problem 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Indicate project** 11 | i.e. collector, exporter, deployment, etc. 12 | Note please also indicate project with correct issue label 13 | 14 | **Is your feature request related to a problem? Please describe.** 15 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 16 | 17 | **Describe the feature you'd like** 18 | A clear and concise description of what you want to happen. 19 | 20 | **Describe alternatives you've considered** 21 | A clear and concise description of any alternative solutions or features you've considered. 22 | 23 | **Additional context** 24 | Add any other context or screenshots about the feature request here. 25 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | sync: 8 | name: Sync Submodules 9 | runs-on: ubuntu-latest 10 | 11 | defaults: 12 | run: 13 | shell: bash 14 | 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v2 18 | with: 19 | submodules: true 20 | 21 | - name: Git Submodule Update 22 | run: | 23 | git submodule foreach "git fetch origin; git checkout master; git pull --no-rebase origin master" 24 | 25 | - name: Commit update 26 | run: | 27 | git config --global user.name 'SysFlow' 28 | git config --global user.email 'sysflow@noreply.github.com' 29 | git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} 30 | git commit -am "Auto updated submodule references" && git push || echo "No changes to commit" 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | .vscode 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "modules/sf-exporter"] 2 | path = modules/sf-exporter 3 | url = https://github.com/sysflow-telemetry/sf-exporter.git 4 | [submodule "modules/sf-apis"] 5 | path = modules/sf-apis 6 | url = https://github.com/sysflow-telemetry/sf-apis.git 7 | [submodule "modules/sf-collector"] 8 | path = modules/sf-collector 9 | url = https://github.com/sysflow-telemetry/sf-collector.git 10 | [submodule "modules/sf-deployments"] 11 | path = modules/sf-deployments 12 | url = https://github.com/sysflow-telemetry/sf-deployments.git 13 | [submodule "modules/sf-processor"] 14 | path = modules/sf-processor 15 | url = https://github.com/sysflow-telemetry/sf-processor.git 16 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: docs/conf.py 20 | builder: html 21 | fail_on_warning: false 22 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 23 | # builder: "dirhtml" 24 | # Fail on all warnings to avoid broken references 25 | # fail_on_warning: true 26 | 27 | # Optionally build your docs in additional formats such as PDF and ePub 28 | formats: 29 | - pdf 30 | - epub 31 | 32 | # Optional but recommended, declare the Python requirements required 33 | # to build your documentation 34 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 35 | python: 36 | install: 37 | - requirements: docs/requirements.txt 38 | 39 | submodules: 40 | include: all 41 | recursive: false 42 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [Slack channel](https://join.slack.com/t/sysflow-telemetry/shared_invite/enQtODA5OTA3NjE0MTAzLTlkMGJlZDQzYTc3MzhjMzUwNDExNmYyNWY0NWIwODNjYmRhYWEwNGU0ZmFkNGQ2NzVmYjYxMWFjYTM1MzA5YWQ) or via [email](mailto:sysflow@us.ibm.com). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the Qiskit project's [Code of Conduct](https://github.com/Qiskit/qiskit/blob/master/CODE_OF_CONDUCT.md) and has roots from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at [version](http://contributor-covenant.org/version/1/4). 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing In General 2 | Our project welcomes external contributions. 3 | 4 | To contribute code or documentation, please submit a pull request to the proper github repositories. 5 | 6 | A good way to familiarize yourself with the codebase and contribution process is 7 | to look for and tackle low-hanging fruit in the github issue trackers associated with projects. 8 | Before embarking on a more ambitious contribution, please quickly [get in touch](#communication) with us. 9 | 10 | **Note: We appreciate your effort, and want to avoid a situation where a contribution 11 | requires extensive rework (by you or by us), sits in backlog for a long time, or 12 | cannot be accepted at all!** 13 | 14 | ### Proposing new features 15 | 16 | If you would like to implement a new feature, please raise an issue in the appropriate repository 17 | before sending a pull request so the feature can be discussed. This is to avoid 18 | you wasting your valuable time working on a feature that the project developers 19 | are not interested in accepting into the code base. 20 | 21 | ### Fixing bugs 22 | 23 | If you would like to fix a bug, please raise an issue in the appropriate repository before sending a 24 | pull request so it can be tracked. 25 | 26 | ### Merge approval 27 | 28 | The project maintainers use LGTM (Looks Good To Me) in comments on the code 29 | review to indicate acceptance. A change requires LGTMs from two of the 30 | maintainers of each component affected. 31 | 32 | For a list of the maintainers, see the MAINTAINERS.md page in the appropriate repository. 33 | 34 | ## Legal 35 | 36 | Each source file must include a license header for the Apache 37 | Software License 2.0. Using the SPDX format is the simplest approach. 38 | e.g. 39 | 40 | ``` 41 | /* 42 | Copyright All Rights Reserved. 43 | 44 | SPDX-License-Identifier: Apache-2.0 45 | */ 46 | ``` 47 | 48 | We have tried to make it as easy as possible to make contributions. This 49 | applies to how we handle the legal aspects of contribution. We use the 50 | same approach - the [Developer's Certificate of Origin 1.1 (DCO)](https://github.com/hyperledger/fabric/blob/master/docs/source/DCO1.1.txt) - that the Linux® Kernel [community](https://elinux.org/Developer_Certificate_Of_Origin) 51 | uses to manage code contributions. 52 | 53 | We simply ask that when submitting a patch for review, the developer 54 | must include a sign-off statement in the commit message. 55 | 56 | Here is an example Signed-off-by line, which indicates that the 57 | submitter accepts the DCO: 58 | 59 | ``` 60 | Signed-off-by: John Doe 61 | ``` 62 | 63 | You can include this automatically when you commit a change to your 64 | local git repository using the following command: 65 | 66 | ``` 67 | git commit -s 68 | ``` 69 | 70 | ## Communication 71 | Please feel free to connect with us on our [Slack channel](https://join.slack.com/t/sysflow-telemetry/shared_invite/enQtODA5OTA3NjE0MTAzLTlkMGJlZDQzYTc3MzhjMzUwNDExNmYyNWY0NWIwODNjYmRhYWEwNGU0ZmFkNGQ2NzVmYjYxMWFjYTM1MzA5YWQ) or 72 | via [email](mailto:sysflow@us.ibm.com). Note that the projects in this repository are not formal products. As a result, the communication channels are to the maintainers and not to a support staff. 73 | 74 | ## Setup 75 | 76 | The documentation is a work in progress but should provide a good overview on how to get started with the project. The Dockerfile also provides a treasure trove of information 77 | on how to build the application, dependencies, and how to test the collector. 78 | 79 | ## Testing 80 | 81 | This project is in its infancy and with limited resources we haven't built many testers for the projects. For the sf-collector, we do have a set of unit tests that test the coverage of most of the events of interest in `sf-collector/tests`. 82 | These tests can be run using the [bats testing framework](https://github.com/bats-core/bats-core). Directions on how to install bats are in the accompanied link. To run the tests, run `bats -t tests.bat` from the tests directory. Note, 83 | that the tests also rely on python3. Before conducting a pull request, these unit tests should be run. Note, there is a version of the docker image with a `testing` tag that contains bats and the unit tests. This might be useful for testing. 84 | Also, conducting a load test and running the application under valgrind is desirable for pull requests. 85 | 86 | 87 | ## Coding style guidelines 88 | We follow the [LLVM Coding standards](https://llvm.org/docs/CodingStandards.html) where possible across the projects. There is a .clang-format file in the master repo [clang-format](https://github.com/sysflow-telemetry/sf-collector/blob/master/src/.clang-format) that can be used in conjunction with [ClangFormat Tool](https://clang.llvm.org/docs/ClangFormat.html) to automatically format code. For linting, 89 | we use [Clang Tidy Linter](https://clang.llvm.org/extra/clang-tidy/). This is referenced in the sf-collector Makefile. 90 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | ``` 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | ``` 179 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # MAINTAINERS 2 | 3 | Frederico Araujo 4 | 5 | Teryl Taylor 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = docs 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Documentation Status](https://readthedocs.org/projects/sysflow/badge/?version=latest)](https://sysflow.readthedocs.io/en/latest/?badge=latest) 2 | 3 | # SysFlow: Cloud-native system telemetry pipeline 4 | 5 | This repository hosts the [documentation](https://sysflow.readthedocs.io/) and [issue tracker](https://github.com/sysflow-telemetry/sf-docs/issues) for all SysFlow projects. 6 | 7 | # Quick reference 8 | 9 | - **Documentation**: 10 | [the SysFlow Documentation](https://sysflow.readthedocs.io) 11 | 12 | - **Where to get help**: 13 | [the SysFlow Community Slack](https://join.slack.com/t/sysflow-telemetry/shared_invite/enQtODA5OTA3NjE0MTAzLTlkMGJlZDQzYTc3MzhjMzUwNDExNmYyNWY0NWIwODNjYmRhYWEwNGU0ZmFkNGQ2NzVmYjYxMWFjYTM1MzA5YWQ) 14 | 15 | - **Where to file issues**: 16 | [the github issue tracker](https://github.com/sysflow-telemetry/sysflow/issues) 17 | 18 | - **Source of this description**: 19 | [repo's readme](https://github.com/sysflow-telemetry/sysflow/edit/master/README.md) ([history](https://github.com/sysflow-telemetry/sysflow/commits/master)) 20 | 21 | - **Docker images**: 22 | [docker hub](https://hub.docker.com/u/sysflowtelemetry) | [GHCR](https://github.com/orgs/sysflow-telemetry/packages) 23 | 24 | # What is SysFlow? 25 | 26 | The SysFlow Telemetry Pipeline is a framework for monitoring cloud workloads and for creating performance and security analytics. The goal of this project is to build all the plumbing required for system telemetry so that users can focus on writing and sharing analytics on a scalable, common open-source platform. The backbone of the telemetry pipeline is a new data format called SysFlow, which lifts raw system event information into an abstraction that describes process behaviors, and their relationships with containers, files, and network. This object-relational format is highly compact, yet it provides broad visibility into container clouds. We have also built several APIs that allow users to process SysFlow with their favorite toolkits. Learn more about SysFlow in the [SysFlow specification document](https://sysflow.readthedocs.io/en/latest/spec.html). 27 | 28 | The SysFlow framework consists of the following sub-projects: 29 | 30 | - [sf-apis](https://github.com/sysflow-telemetry/sf-apis) provides the SysFlow schema and programatic APIs in go, python, and C++. 31 | - [sf-collector](https://github.com/sysflow-telemetry/sf-collector) monitors and collects system call and event information from hosts and exports them in the SysFlow format using Apache Avro object serialization. 32 | - [sf-processor](https://github.com/sysflow-telemetry/sf-processor) provides a performance optimized policy engine for processing, enriching, filtering SysFlow events, generating alerts, and exporting the processed data to various targets. 33 | - [sf-exporter](https://github.com/sysflow-telemetry/sf-exporter) exports SysFlow traces to S3-compliant storage systems for archival purposes. 34 | - [sf-deployments](https://github.com/sysflow-telemetry/sf-deployments) contains deployment packages for SysFlow, including Docker, Helm, and OpenShift. 35 | - [sysflow](https://github.com/sysflow-telemetry/sysflow) is the documentation repository and issue tracker for the SysFlow framework. 36 | 37 | ## Online documentation 38 | SysFlow documentation is available at [sysflow.readthedocs.io](https://sysflow.readthedocs.io/). 39 | 40 | ## Offline build 41 | This documentation depends on Sphinx (http://www.sphinx-doc.org/en/master/), which must be installed to do builds. The project also requires the following Sphinx plugins: 42 | 43 | * http://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html 44 | * https://pypi.org/project/m2r/ 45 | 46 | To build the site as HTML go to the base directory and type: 47 | ``` 48 | make html 49 | ``` 50 | 51 | # License 52 | 53 | View [license information](https://github.com/sysflow-telemetry/sysflow/blob/master/LICENSE.md) for SysFlow. 54 | 55 | As with all software packages, these likely also contain other software which may be under other licenses (such as Bash, etc from the base distribution, along with any direct or indirect dependencies of the primary software being contained). 56 | 57 | As for any pre-built image usage, it is the image user's responsibility to ensure that any use of this image complies with any relevant licenses for all software contained within. 58 | -------------------------------------------------------------------------------- /docs/_static/SF_Collector_Exporter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/SF_Collector_Exporter.png -------------------------------------------------------------------------------- /docs/_static/SF_Collector_Processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/SF_Collector_Processor.png -------------------------------------------------------------------------------- /docs/_static/SF_Object_View.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/SF_Object_View.png -------------------------------------------------------------------------------- /docs/_static/SF_Object_View_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/SF_Object_View_v2.png -------------------------------------------------------------------------------- /docs/_static/SF_Object_View_v5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/SF_Object_View_v5.png -------------------------------------------------------------------------------- /docs/_static/experimental.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sysflow-telemetry/sysflow/03cb8a36743ec5c20ffab5852a6ed776a21150a6/docs/_static/experimental.png -------------------------------------------------------------------------------- /docs/api-utils.rst: -------------------------------------------------------------------------------- 1 | SysFlow APIs and Utilities (sf-apis repo) 2 | ========================================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | api 8 | python-api 9 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-apis/API.md 2 | -------------------------------------------------------------------------------- /docs/binary.rst: -------------------------------------------------------------------------------- 1 | .. binary documentation master file 2 | .. toctree:: 3 | :maxdepth: 2 4 | :caption: Contents: 5 | 6 | .. mdinclude:: ../modules/sf-deployments/binary/README.md 7 | -------------------------------------------------------------------------------- /docs/build.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-collector/docs/BUILD.md 2 | -------------------------------------------------------------------------------- /docs/coc.rst: -------------------------------------------------------------------------------- 1 | Code of Conduct 2 | =============== 3 | .. mdinclude:: ../CODE_OF_CONDUCT.md 4 | -------------------------------------------------------------------------------- /docs/collector.rst: -------------------------------------------------------------------------------- 1 | SysFlow Collector (sf-collector repo) 2 | ======================================== 3 | 4 | The SysFlow Collector monitors and collects system call and event information from hosts 5 | and exports them in the SysFlow format using Apache Avro object serialization. It's built atop 6 | `libSysFlow `_, a library 7 | that lifts system call information into SysFlow, a higher order object relational format that 8 | models how containers, processes and files interact with their environment through process 9 | control flow, file, and network operations. Learn more about SysFlow in the SysFlow Specification 10 | Document. 11 | 12 | The SysFlow Collector builds on the `CNCF Falco libs `_ to 13 | passively collect system events and turn them into SysFlow. As a result, the collector supports the 14 | libs' powerful filtering capabilities. Check the build and installation instructions for installing 15 | the collector. 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | build 21 | 22 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | import sphinx_rtd_theme 16 | sys.path.insert(0, os.path.abspath('../modules/sf-apis/py3/classes/sysflow/')) 17 | sys.path.insert(0, os.path.abspath('../modules/sf-apis/py3/classes/')) 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'SysFlow Telemetry Pipeline' 22 | copyright = '2024 - Maintained by the SysFlow team' 23 | author = 'The SysFlow team' 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = u'0.4' 27 | version = u'0.4' 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx_rtd_theme', 'sphinx.ext.autodoc', 'm2r2', 'sphinxcontrib.bibtex'] 35 | 36 | # Bibitex 37 | bibtex_bibfiles = ['refs.bib'] 38 | 39 | # Master file 40 | master_doc = 'index' 41 | 42 | # Add any paths that contain templates here, relative to this directory. 43 | templates_path = ['_templates'] 44 | 45 | # List of patterns, relative to source directory, that match files and 46 | # directories to ignore when looking for source files. 47 | # This pattern also affects html_static_path and html_extra_path. 48 | exclude_patterns = [] 49 | 50 | # source_suffix = '.rst' 51 | #app.add_source_parser('.md', M2RParser) 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | 55 | # The theme to use for HTML and HTML Help pages. See the documentation for 56 | # a list of builtin themes. 57 | # 58 | #html_theme = 'alabaster' 59 | html_theme = 'sphinx_rtd_theme' 60 | 61 | # Add any paths that contain custom static files (such as style sheets) here, 62 | # relative to this directory. They are copied after the builtin static files, 63 | # so a file named "default.css" will overwrite the builtin "default.css". 64 | html_static_path = ['_static'] 65 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============== 3 | .. mdinclude:: ../CONTRIBUTING.md 4 | -------------------------------------------------------------------------------- /docs/deploy.rst: -------------------------------------------------------------------------------- 1 | Deployments (sf-deployments repo) 2 | ======================================== 3 | 4 | SysFlow can be deployed using Docker Compose, Helm, and binary packages. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | docker 11 | helm 12 | binary 13 | 14 | -------------------------------------------------------------------------------- /docs/docker.rst: -------------------------------------------------------------------------------- 1 | .. docker documentation master file, created by 2 | sphinx-quickstart on Mon Aug 5 20:51:49 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Contents: 8 | 9 | .. mdinclude:: ../modules/sf-deployments/docker/README.md 10 | -------------------------------------------------------------------------------- /docs/exporter.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-exporter/BUILD.md 2 | -------------------------------------------------------------------------------- /docs/helm.rst: -------------------------------------------------------------------------------- 1 | .. sf-helm-charts documentation master file, created by 2 | sphinx-quickstart on Mon Aug 5 20:51:49 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Contents: 8 | 9 | .. mdinclude:: ../modules/sf-deployments/helm/README.md 10 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | SysFlow Telemetry Pipeline 2 | ====================================================== 3 | 4 | The SysFlow Telemetry Pipeline is a framework for monitoring cloud and enterprise workloads. The framework builds the plumbing required for system telemetry so that users can focus on writing and sharing analytics on a scalable, common open-source platform. 5 | 6 | .. note:: If in a hurry, skip to our `quick start `_ guide. 7 | 8 | The backbone of the telemetry pipeline is a new `data format `_ which lifts raw system event information into an abstraction that describes process behaviors, and their relationships with containers, files, and network activity. This object-relational format is highly compact, yet it provides broad visibility into legacy endpoints and container clouds. 9 | 10 | The platform is designed as a pluggable edge processing architecture which includes a policy engine that accepts declarative policies that support edge filtering, tagging, and alerting on SysFlow streams. It also offers several APIs that allow users to process SysFlow with their favorite toolkits. 11 | 12 | The pipeline can be `deployed `_ using Docker, Kubernetes, OpenShift, and bare metal/VMs. The `SysFlow agent `_ can be configured as an edge analytics pipeline to stream SysFlow records through rsyslog, or as a batch exporter of raw SysFlow traces to S3-compatible object stores. 13 | 14 | An integrated `Jupyter environment `_ makes it easy to perform log hunting on collected traces. There are also Apache Avro schema files for SysFlow so that users can generate APIs for other programming languages. C++, Python, and Golang `APIs `_ are available, allowing users to interact with SysFlow traces programmatically. 15 | 16 | To learn more about SysFlow, check the table of contents below. 17 | 18 | *We welcome feedback, bug reports, and contributions!* 19 | 20 | Keep in touch 21 | ------------- 22 | Please connect with us on our `Slack `_ community! 23 | 24 | Bugs & Feature requests 25 | ------------- 26 | For bugs and feature requests, please check our `issue tracker `_. 27 | 28 | License 29 | ------- 30 | SysFlow and all projects are released under the Apache v2.0 license. 31 | 32 | .. toctree:: 33 | :maxdepth: 2 34 | :caption: Contents: 35 | 36 | quick 37 | spec 38 | libs 39 | collector 40 | processor 41 | exporter 42 | api-utils 43 | deploy 44 | license 45 | contributing 46 | coc 47 | publications 48 | 49 | Indices and tables 50 | ================== 51 | 52 | * :ref:`genindex` 53 | * :ref:`modindex` 54 | * :ref:`search` 55 | -------------------------------------------------------------------------------- /docs/libs.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-collector/docs/LIBS.md 2 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ============== 3 | .. mdinclude:: ../LICENSE.md 4 | -------------------------------------------------------------------------------- /docs/processor.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-processor/docs/BUILD.md 2 | 3 | .. mdinclude:: ../modules/sf-processor/docs/CONFIG.md 4 | 5 | .. mdinclude:: ../modules/sf-processor/docs/POLICIES.md 6 | 7 | .. mdinclude:: ../modules/sf-processor/docs/PLUGINS.md 8 | 9 | .. mdinclude:: ../modules/sf-processor/docs/DOCKER.md 10 | -------------------------------------------------------------------------------- /docs/publications.rst: -------------------------------------------------------------------------------- 1 | Talks & Publications 2 | ============== 3 | 4 | If citing SysFlow, please use :cite:`bigdata2020`. 5 | 6 | Below you can find a complete list of talks and papers associated with SysFlow. 7 | 8 | .. note:: Please `reach out to us `_ if you have an entry to add to this list. 9 | 10 | .. bibliography:: 11 | :all: 12 | :style: unsrtalpha -------------------------------------------------------------------------------- /docs/python-api.rst: -------------------------------------------------------------------------------- 1 | SysFlow Python API Reference 2 | ============================ 3 | 4 | SysFlow Reader API 5 | ------------------ 6 | .. automodule:: sysflow.reader 7 | :members: 8 | 9 | SysFlow Formatter API 10 | --------------------- 11 | .. automodule:: sysflow.formatter 12 | :members: 13 | 14 | SysFlow Object Types 15 | -------------------- 16 | .. automodule:: sysflow.objtypes 17 | :members: 18 | 19 | .. automodule:: sysflow.opflags 20 | :members: 21 | :noindex: 22 | 23 | SysFlow Utils API 24 | ----------------- 25 | .. automodule:: sysflow.utils 26 | :members: 27 | 28 | SysFlow Graphlet API 29 | -------------------- 30 | .. automodule:: sysflow.graphlet 31 | :members: 32 | 33 | SysFlow QL API 34 | -------------------- 35 | .. automodule:: sysflow.sfql 36 | :members: 37 | -------------------------------------------------------------------------------- /docs/quick.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../modules/sf-deployments/docker/QUICK.md 2 | -------------------------------------------------------------------------------- /docs/refs.bib: -------------------------------------------------------------------------------- 1 | @misc {lss2024, 2 | author={Frederico Araujo and Teryl Taylor}, 3 | booktitle={Linux Foundation Linux Security Summit North America}, 4 | title={Provenance-Aware Integrity Monitoring with Linux Security Identifiers}, 5 | year={2024}, 6 | url={https://sched.co/1aIeC} 7 | } 8 | 9 | @INPROCEEDINGS {oakland2024, 10 | author={William Blair and Frederico Araujo and Teryl Taylor and Jiyong Jang}, 11 | booktitle={2024 IEEE Symposium on Security and Privacy (SP)}, 12 | title={Automated Synthesis of Effect Graph Policies for Microservice-Aware Stateful System Call Specialization}, 13 | year={2024}, 14 | pages={64-64}, 15 | url={https://doi.ieeecomputersociety.org/10.1109/SP54263.2024.00064} 16 | } 17 | 18 | @misc {ossna2023, 19 | author={Frederico Araujo and Teryl Taylor}, 20 | booktitle={Linux Foundation Open Source Summit North America}, 21 | title={Relational Observability for Cloud-Native Security and Data Science}, 22 | year={2023}, 23 | url={https://sched.co/1K5IT} 24 | } 25 | 26 | @misc {avengercon2022, 27 | author={Trent Jaeger and Frederico Araujo and Teryl Taylor}, 28 | booktitle={AvengerCon}, 29 | title={Provenance Tracking With Attack Graphs Using SysFlow}, 30 | year={2022}, 31 | url={https://avengercon.org/workshop/Provenance-Tracking-With-Attack-Graphs-Using-SysFlow/} 32 | } 33 | 34 | @misc {flocon2022, 35 | author={Frederico Araujo and Teryl Taylor}, 36 | booktitle={FloCon}, 37 | title={Self-Modulating Endpoint Observability}, 38 | year={2022}, 39 | url={https://sched.co/lDbn} 40 | } 41 | 42 | @misc {bheurope2021, 43 | author={Xiaokui Shu and Frederico Araujo and Teryl Taylor and Jiyong Jang}, 44 | booktitle={Black Hat Europe - Arsenal}, 45 | title={An Open Stack for Threat Hunting in Hybrid Cloud with Connected Observability}, 46 | year={2021}, 47 | url={https://europe-arsenal-cfp.blackhat.com/} 48 | } 49 | 50 | @misc {flocon2021, 51 | author={Frederico Araujo and Teryl Taylor}, 52 | booktitle={FloCon}, 53 | title={A Pluggable Edge-Processing Pipeline for {SysFlow}}, 54 | year={2021}, 55 | url={https://sched.co/ePsl} 56 | } 57 | 58 | @misc {flocon2021b, 59 | author={William Blair and Frederico Araujo and Teryl Taylor and Jiyong Jang}, 60 | booktitle={FloCon}, 61 | title={Microservice-Aware Reference Monitoring through Hybrid Program Analysis}, 62 | year={2021}, 63 | url={https://sched.co/ePs3} 64 | } 65 | 66 | @INPROCEEDINGS {bigdata2020, 67 | author={Teryl Taylor and Frederico Araujo and Xiaokui Shu}, 68 | booktitle={IEEE International Conference on Big Data (Big Data)}, 69 | title={Towards an Open Format for Scalable System Telemetry}, 70 | year={2020}, 71 | pages={1031-1040}, 72 | url={https://arxiv.org/abs/2101.10474} 73 | } 74 | 75 | @misc {flocon2020, 76 | author={Frederico Araujo and Teryl Taylor}, 77 | booktitle={FloCon}, 78 | title={{SysFlow}: Scalable System Telemetry for Improved Security Analytics}, 79 | year={2020}, 80 | url={https://sched.co/VPW3} 81 | } 82 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | fastavro>=1.4.2 2 | frozendict>=2.0.3 3 | tabulate>=0.8.9 4 | minio>=7.0.3 5 | m2r2>=0.3.2 6 | sphinx-rtd-theme>=0.4.3 7 | antlr4-python3-runtime==4.9.2 8 | dotty-dict>=1.3.0 9 | pandas>=1.3.0 10 | sphinx>=6.2.1 11 | sphinxcontrib-bibtex>=2.1.4 12 | graphviz>=0.17 13 | matplotlib>=3.4.3 14 | importlib.metadata>=4.12.0 15 | mitreattack-python>=1.5.7 16 | -------------------------------------------------------------------------------- /docs/spec.rst: -------------------------------------------------------------------------------- 1 | SysFlow Specification 2 | ======================================== 3 | 4 | The SysFlow format lifts raw system event information into an abstraction that describes process behaviors, and their relationships with containers, files, and network. This object-relational format is highly compact, yet it provides broad visibility into container clouds. The framework includes several APIs that allow users to process SysFlow with their favorite toolkits. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | .. mdinclude:: ../modules/sf-collector/docs/SPEC.md 10 | -------------------------------------------------------------------------------- /make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /update: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | git submodule foreach "(git checkout dev; git pull origin dev)" 4 | 5 | --------------------------------------------------------------------------------