├── .circleci └── config.yml ├── .github ├── ISSUE_TEMPLATE │ ├── reports-enhancement.md │ └── reports-issue.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .readthedocs.yaml ├── CHANGES.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── assets ├── .DS_Store ├── css │ ├── .DS_Store │ └── theme-1.css └── js │ ├── .DS_Store │ └── main.js ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── _static │ └── pyreports.svg │ ├── conf.py │ ├── datatools.rst │ ├── dev │ ├── cli.rst │ ├── core.rst │ └── io.rst │ ├── example.rst │ ├── executors.rst │ ├── index.rst │ ├── install.rst │ ├── managers.rst │ ├── package.rst │ ├── pyreports.rst │ └── report.rst ├── favicon.ico ├── index.html ├── pyproject.toml ├── pyreports ├── __init__.py ├── cli.py ├── core.py ├── datatools.py ├── exception.py └── io.py ├── setup.py └── tests ├── __init__.py ├── test_core.py ├── test_data.py ├── test_db.py └── test_file.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | jobs: 4 | build-and-test: 5 | docker: 6 | - image: circleci/python 7 | steps: 8 | - checkout 9 | - run: sudo apt update && sudo apt install -y freetds-dev libssl-dev python-dev freetds-bin 10 | - run: sudo pip install --upgrade pip 11 | - run: sudo pip install "Cython<3" 12 | - run: sudo pip install numpy 13 | - run: sudo pip install pandas 14 | - run: sudo python setup.py install 15 | - run: sudo chmod -R 777 /tmp 16 | - run: sudo python -m unittest discover tests 17 | 18 | workflows: 19 | main: 20 | jobs: 21 | - build-and-test -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/reports-enhancement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: pyreports enhancement 3 | about: pyreports enhancement template 4 | title: pyreports enhancement 5 | labels: enhancement 6 | assignees: MatteoGuadrini 7 | --- 8 | 9 | ## Description 10 | 11 | Description of the proposal 12 | 13 | 14 | ## Proposed names of the parameters (short and long) 15 | 16 | * name parameter 17 | * possible argument(s) 18 | 19 | Additional context 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/reports-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: pyreports bug 3 | about: pyreports bug template 4 | title: pyreports bug 5 | labels: bug 6 | assignees: MatteoGuadrini 7 | --- 8 | 9 | ## Description 10 | 11 | Description of problem 12 | 13 | ## Steps to Reproduce 14 | 15 | Line of code 16 | 17 | ## Expected Behaviour 18 | 19 | Description of what is expected 20 | 21 | ## Your Environment 22 | 23 | * pyreports version used: 24 | * Operating System and version: 25 | 26 | Additional context 27 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Title of Pull Request 2 | 3 | ## List of changes 4 | 5 | - This pull request has a changes in the format: `changes is...`. 6 | - ✅ `Add function on fontpreview` 7 | - ✅ `Add parameter -p/--parameter` 8 | - ✅ `Update readme.md` 9 | - ❌ `Delete part of documentation` 10 | 11 | ### Description of the proposed features 12 | 13 | Descriptive text of the features. 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### PyCharm+all ### 2 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 3 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 4 | 5 | # User-specific stuff 6 | .idea/**/workspace.xml 7 | .idea/**/tasks.xml 8 | .idea/**/usage.statistics.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | .idea/**/dbnavigator.xml 20 | 21 | # Gradle 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # Gradle and Maven with auto-import 26 | # When using Gradle or Maven with auto-import, you should exclude module files, 27 | # since they will be recreated, and may cause churn. Uncomment if using 28 | # auto-import. 29 | # .idea/modules.xml 30 | # .idea/*.iml 31 | # .idea/modules 32 | 33 | # CMake 34 | cmake-build-*/ 35 | 36 | # Mongo Explorer plugin 37 | .idea/**/mongoSettings.xml 38 | 39 | # File-based project format 40 | *.iws 41 | 42 | # IntelliJ 43 | out/ 44 | 45 | # mpeltonen/sbt-idea plugin 46 | .idea_modules/ 47 | 48 | # JIRA plugin 49 | atlassian-ide-plugin.xml 50 | 51 | # Cursive Clojure plugin 52 | .idea/replstate.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | fabric.properties 59 | 60 | # Editor-based Rest Client 61 | .idea/httpRequests 62 | 63 | ### PyCharm+all Patch ### 64 | # Ignores the whole .idea folder and all .iml files 65 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 66 | 67 | .idea/ 68 | 69 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 70 | 71 | *.iml 72 | modules.xml 73 | .idea/misc.xml 74 | *.ipr 75 | 76 | ### Python ### 77 | # Byte-compiled / optimized / DLL files 78 | __pycache__/ 79 | *.py[cod] 80 | *$py.class 81 | 82 | # C extensions 83 | *.so 84 | 85 | # Distribution / packaging 86 | .Python 87 | build/ 88 | develop-eggs/ 89 | dist/ 90 | downloads/ 91 | eggs/ 92 | .eggs/ 93 | lib/ 94 | lib64/ 95 | parts/ 96 | sdist/ 97 | var/ 98 | wheels/ 99 | *.egg-info/ 100 | .installed.cfg 101 | *.egg 102 | MANIFEST 103 | 104 | # PyInstaller 105 | # Usually these files are written by a python script from a template 106 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 107 | *.manifest 108 | *.spec 109 | 110 | # Installer logs 111 | pip-log.txt 112 | pip-delete-this-directory.txt 113 | 114 | # Unit test / coverage pyreports 115 | htmlcov/ 116 | .tox/ 117 | .coverage 118 | .coverage.* 119 | .cache 120 | nosetests.xml 121 | coverage.xml 122 | *.cover 123 | .hypothesis/ 124 | .pytest_cache/ 125 | 126 | # Translations 127 | *.mo 128 | *.pot 129 | 130 | # Django stuff: 131 | *.log 132 | local_settings.py 133 | db.sqlite3 134 | 135 | # Flask stuff: 136 | instance/ 137 | .webassets-cache 138 | 139 | # Scrapy stuff: 140 | .scrapy 141 | 142 | # Sphinx documentation 143 | docs/_build/ 144 | 145 | # PyBuilder 146 | target/ 147 | 148 | # Jupyter Notebook 149 | .ipynb_checkpoints 150 | 151 | # pyenv 152 | .python-version 153 | 154 | # celery beat schedule file 155 | celerybeat-schedule 156 | 157 | # SageMath parsed files 158 | *.sage.py 159 | 160 | # Environments 161 | .env 162 | .venv 163 | env/ 164 | venv/ 165 | ENV/ 166 | env.bak/ 167 | venv.bak/ 168 | 169 | # Spyder project settings 170 | .spyderproject 171 | .spyproject 172 | 173 | # Rope project settings 174 | .ropeproject 175 | 176 | # mkdocs documentation 177 | /site 178 | 179 | # mypy 180 | .mypy_cache/ 181 | 182 | ### Python Patch ### 183 | .venv/ 184 | 185 | ### Python.VirtualEnv Stack ### 186 | # Virtualenv 187 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 188 | [Ii]nclude 189 | [Ll]ib 190 | [Ll]ib64 191 | [Ll]ocal 192 | [Ss]cripts 193 | pyvenv.cfg 194 | pip-selfcheck.json 195 | 196 | ### VisualStudioCode ### 197 | .vscode/* 198 | !.vscode/settings.json 199 | !.vscode/tasks.json 200 | !.vscode/launch.json 201 | !.vscode/extensions.json 202 | 203 | 204 | # End of https://www.gitignore.io/api/python,pycharm+all,visualstudiocode 205 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.11" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: docs/source/conf.py 20 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 21 | # builder: "dirhtml" 22 | # Fail on all warnings to avoid broken references 23 | # fail_on_warning: true 24 | 25 | # Optionally build your docs in additional formats such as PDF and ePub 26 | # formats: 27 | # - pdf 28 | # - epub 29 | 30 | # Optional but recommended, declare the Python requirements required 31 | # to build your documentation 32 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 33 | python: 34 | install: 35 | - requirements: docs/requirements.txt -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Release notes 2 | 3 | ## 1.8.0 4 | May 31, 2025 5 | 6 | - Add **add** special method into _Executor_ class 7 | - Add **type** property on _Manager_ class 8 | - Add **ExecutorDataError** class 9 | - Add **reset** method into _Report_ class 10 | - Add **clone** method into _DataObject_ class 11 | - Add **clone** method into _Report_ class 12 | - Add **column** method on _DataObject_ class 13 | - Add **subset** method on _DataAdapters_ class 14 | - Add **subset** function 15 | - Add _try/except_ into **send** method on _ReportBook_ class 16 | - Add **sort** method into _DataAdapters_ class 17 | - Add **sort** function 18 | - Add **sort**, **subset** and **subset** data tools 19 | - Make _report_ attribute into property on _Report_ class 20 | - Use _remove_duplicates_ method on **deduplicate** functions 21 | - Change behavior of _column_ parameter into **filter** function on _Executor_ class 22 | 23 | ## 1.7.0 24 | Apr 19, 2024 25 | 26 | - Remove **pymssql** support 27 | - Add **DataObject** class 28 | - Add **DataAdapters** class 29 | - Add **DataPrinters** class 30 | - Add **aggregate** into _DataAdapters_ class 31 | - Add **merge** method into _DataAdapters_ class 32 | - Add **counter** method into _DataAdapters_ class 33 | - Add **chunks** method into _DataAdapters_ class 34 | - Add **deduplicate** method into _DataAdapters_ class 35 | - Add **iter** method into _DataAdapters_ class 36 | - Add **getitem** method into _DataAdapters_ class 37 | - Add **repr** and **str** function into _DataPrinters_ class 38 | - Add **average** method into _DataPrinters_ class 39 | - Add **most_common** method into _DataPrinters_ class 40 | - Add **percentage** method into _DataPrinters_ class 41 | - Add **len** method into _DataPrinters_ class 42 | - Add **DataAdapters** and **DataPrinters** classes into _Report_ class 43 | - Add **DataObjectError** exception class 44 | 45 | ## 1.6.0 46 | Jul 14, 2023 47 | 48 | - Add **deduplicate** function on datatools 49 | - Add **Manager** abstract class 50 | - Add **READABLE_MANAGER** and **WRITABLE_MANAGER** tuple 51 | - Add _pyproject.toml_ file 52 | - Add **negation** to filter method on _Executor_ class 53 | - Fix _max_len_ into **aggregate** function, refs #2 54 | - Fix _sendmail_ method addresses, refs #3 55 | - Rename **SQLiteConnection** class 56 | - Reformat code with ruff code analysis 57 | 58 | ## 1.5.0 59 | Aug 4, 2022 60 | 61 | - Added **cli** module 62 | - Added **reports** cli 63 | - Added **\__getitem\__** method on _Report_ class 64 | - Added **\__delitem\__** method on _Report_ class 65 | - Added **\__getitem\__** method on _ReportBook_ class 66 | - Added **\__delitem\__** method on _ReportBook_ class 67 | - Added **\__contains\__** on _Executor_ class 68 | - Fix **NoSQLManager** creation into _manager_ function 69 | - Fix **print_data** on _Report_ class 70 | 71 | ## 1.4.0 72 | Jun 27, 2022 73 | 74 | - Added **\__bool\__** method on _Report_ class 75 | - Added **\__iter\__** method on _Report_ class 76 | - Added **\__bool\__** method on _ReportBook_ class 77 | - Added **\__iter\__** method on _Connection_ and _File_ classes 78 | - Added **\__iter\__** method on _FileManager_ class 79 | - Added **\__iter\__** method on _DatabaseManager_ class 80 | - Added **\__getitem\__** on _Executor_ class 81 | - Added **\__delitem\__** on _Executor_ class 82 | - Fix name of attachment on **send** method of _Report_ class 83 | - Fix **write** method on _LogFile_ class 84 | 85 | ## 1.3.0 86 | Apr 15, 2022 87 | 88 | - Added **NoSQLManager** class; this class extend _Manager_ class on the [nosqlapi](https://github.com/MatteoGuadrini/nosqlapi) package 89 | - Added **LogFile** class; this class load a log file and _read_ method accept regular expression 90 | - Added **\__bool\__** and **\__repr\__** method on _File_ and _Connection_ abstract classes 91 | - Fix documentation API section 92 | - Fix tests package 93 | - Fix CircleCi docker image 94 | 95 | ## 1.2.0 96 | Aug 5, 2021 97 | 98 | - Added _fill_value_ argument on **aggregate** function; this value also is callable without arguments 99 | - Added _send_ method on **Report** class; with this method you send report via email 100 | - Added _send_ method on **ReportBook** class; with this method you send report via email 101 | - Fix \*__str__* method on **Report** class 102 | 103 | ## 1.1.0 104 | Jun 5, 2021 105 | 106 | - Created abstract **File** class 107 | - Created **TextFile** class 108 | - Added *\__str__* method for pretty representation of **Executor** class 109 | - Added *\__repr__* method for representation of **DatabaseManager** class 110 | - Added *\__repr__* method for representation of **FileManager** class 111 | - Added *\__repr__* method for representation of **LdapManager** class 112 | - Fix documentation for new abstract **File** class 113 | 114 | ## 1.0.0 115 | May 26, 2021 116 | 117 | - Created abstract **Connection** class 118 | - Created **\*Connection** classes 119 | - Created **\*File** classes 120 | - Created **FileManager**, **DatabaseManager** and **LdapManager** classes 121 | - Created **Executor** class 122 | - Created **Report** class 123 | - Created **ReportBook** class 124 | - Created **average**, **most_common**, **percentage**, **counter**, **aggregate**, **merge**, **chunks**, functions -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of 4 | fostering an open and welcoming community, we pledge to respect all people who 5 | contribute through reporting issues, posting feature requests, updating 6 | documentation, submitting pull requests or patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free 9 | experience for everyone, regardless of level of experience, gender, gender 10 | identity and expression, sexual orientation, disability, personal appearance, 11 | body size, race, ethnicity, age, religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic 20 | addresses, without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or 24 | reject comments, commits, code, wiki edits, issues, and other contributions 25 | that are not aligned to this Code of Conduct, or to ban temporarily or 26 | permanently any contributor for other behaviors that they deem inappropriate, 27 | threatening, offensive, or harmful. 28 | 29 | By adopting this Code of Conduct, project maintainers commit themselves to 30 | fairly and consistently applying these principles to every aspect of managing 31 | this project. Project maintainers who do not follow or enforce the Code of 32 | Conduct may be permanently removed from the project team. 33 | 34 | This Code of Conduct applies both within project spaces and in public spaces 35 | when an individual is representing the project or its community. 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 38 | reported by contacting the project maintainers at msfdev@metasploit.com. If 39 | the incident involves a committer, you may report directly to 40 | egypt@metasploit.com or todb@metasploit.com. 41 | 42 | All complaints will be reviewed and investigated and will result in a 43 | response that is deemed necessary and appropriate to the circumstances. 44 | Maintainers are obligated to maintain confidentiality with regard to the 45 | reporter of an incident. 46 | 47 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 48 | version 1.3.0, available at 49 | [http://contributor-covenant.org/version/1/3/0/][version] 50 | 51 | [homepage]: http://contributor-covenant.org 52 | [version]: http://contributor-covenant.org/version/1/3/0/ -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with the owners of this repository before making a change. 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | ## Pull Request Process 9 | 10 | 1. Be sure to use the Pull Request templates and its guidelines contained therein. 11 | 2. Update the README.md with details of changes. 12 | 3. Increase the version numbers in variable and manual to the new version that this 13 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 14 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 15 | do not have permission to do that, you may request the second reviewer to merge it for you. 16 | 17 | ## Code of Conduct 18 | 19 | ### Our Pledge 20 | 21 | In the interest of fostering an open and welcoming environment, we as 22 | contributors and maintainers pledge to making participation in our project and 23 | our community a harassment-free experience for everyone, regardless of age, body 24 | size, disability, ethnicity, gender identity and expression, level of experience, 25 | nationality, personal appearance, race, religion, or sexual identity and 26 | orientation. 27 | 28 | ### Our Standards 29 | 30 | Examples of behavior that contributes to creating a positive environment 31 | include: 32 | 33 | * Using welcoming and inclusive language 34 | * Being respectful of differing viewpoints and experiences 35 | * Gracefully accepting constructive criticism 36 | * Focusing on what is best for the community 37 | * Showing empathy towards other community members 38 | 39 | Examples of unacceptable behavior by participants include: 40 | 41 | * The use of sexualized language or imagery and unwelcome sexual attention or 42 | advances 43 | * Trolling, insulting/derogatory comments, and personal or political attacks 44 | * Public or private harassment 45 | * Publishing others' private information, such as a physical or electronic 46 | address, without explicit permission 47 | * Other conduct which could reasonably be considered inappropriate in a 48 | professional setting 49 | 50 | ### Our Responsibilities 51 | 52 | Project maintainers are responsible for clarifying the standards of acceptable 53 | behavior and are expected to take appropriate and fair corrective action in 54 | response to any instances of unacceptable behavior. 55 | 56 | Project maintainers have the right and responsibility to remove, edit, or 57 | reject comments, commits, code, wiki edits, issues, and other contributions 58 | that are not aligned to this Code of Conduct, or to ban temporarily or 59 | permanently any contributor for other behaviors that they deem inappropriate, 60 | threatening, offensive, or harmful. 61 | 62 | ### Scope 63 | 64 | This Code of Conduct applies both within project spaces and in public spaces 65 | when an individual is representing the project or its community. Examples of 66 | representing a project or community include using an official project e-mail 67 | address, posting via an official social media account, or acting as an appointed 68 | representative at an online or offline event. Representation of a project may be 69 | further defined and clarified by project maintainers. 70 | 71 | ### Enforcement 72 | 73 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 74 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 75 | complaints will be reviewed and investigated and will result in a response that 76 | is deemed necessary and appropriate to the circumstances. The project team is 77 | obligated to maintain confidentiality with regard to the reporter of an incident. 78 | Further details of specific enforcement policies may be posted separately. 79 | 80 | Project maintainers who do not follow or enforce the Code of Conduct in good 81 | faith may face temporary or permanent repercussions as determined by other 82 | members of the project's leadership. 83 | 84 | ### Attribution 85 | 86 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 87 | available at [http://contributor-covenant.org/version/1/4][version] 88 | 89 | [homepage]: http://contributor-covenant.org 90 | [version]: http://contributor-covenant.org/version/1/4/ -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include pyproject.toml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pyreports 2 | 3 | pyreports 4 | 5 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/2bad30d308414c83836f22f012c98649)](https://www.codacy.com/gh/MatteoGuadrini/pyreports/dashboard?utm_source=github.com&utm_medium=referral&utm_content=MatteoGuadrini/pyreports&utm_campaign=Badge_Grade) 6 | [![CircleCI](https://circleci.com/gh/MatteoGuadrini/pyreports.svg?style=svg)](https://circleci.com/gh/MatteoGuadrini/pyreports) 7 | 8 | _pyreports_ is a python library that allows you to create complex reports from various sources such as databases, 9 | text files, ldap, etc. and perform processing, filters, counters, etc. 10 | and then export or write them in various formats or in databases. 11 | 12 | ## Test package 13 | 14 | To test the package, follow these instructions: 15 | 16 | ```console 17 | $ git clone https://github.com/MatteoGuadrini/pyreports.git 18 | $ cd pyreports 19 | $ python -m unittest discover tests 20 | ``` 21 | 22 | ## Install package 23 | 24 | To install package, follow these instructions: 25 | 26 | ```console 27 | $ pip install pyreports #from pypi 28 | 29 | $ git clone https://github.com/MatteoGuadrini/pyreports.git #from official repo 30 | $ cd pyreports 31 | $ pip install . # or python setup.py install 32 | ``` 33 | 34 | ## Why choose this library? 35 | 36 | _pyreports_ wants to be a library that simplifies the collection of data from multiple sources such as databases, 37 | files and directory servers (through LDAP), the processing of them through built-in and customized functions, 38 | and the saving in various formats (or, by inserting the data in a database). 39 | 40 | ## How does it work 41 | 42 | _pyreports_ uses the [**tablib**](https://tablib.readthedocs.io/en/stable/) library to organize the data into _Dataset_ object. 43 | 44 | ### Simple report 45 | 46 | I take the data from a database table, filter the data I need and save it in a csv file 47 | 48 | ```python 49 | import pyreports 50 | 51 | # Select source: this is a DatabaseManager object 52 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 53 | 54 | # Get data 55 | mydb.execute('SELECT * FROM site_login') 56 | site_login = mydb.fetchall() 57 | 58 | # Filter data 59 | error_login = pyreports.Executor(site_login) 60 | error_login.filter([400, 401, 403, 404, 500]) 61 | 62 | # Save report: this is a FileManager object 63 | output = pyreports.manager('csv', '/home/report/error_login.csv') 64 | output.write(error_login.get_data()) 65 | 66 | ``` 67 | 68 | ### Combine source 69 | 70 | I take the data from a database table, and a log file, and save the report in json format 71 | 72 | ```python 73 | import pyreports 74 | 75 | # Select source: this is a DatabaseManager object 76 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 77 | # Select another source: this is a FileManager object 78 | mylog = pyreports.manager('file', '/var/log/httpd/error.log') 79 | 80 | # Get data 81 | mydb.execute('SELECT * FROM site_login') 82 | site_login = mydb.fetchall() 83 | error_log = mylog.read() 84 | 85 | # Filter database 86 | error_login = pyreports.Executor(site_login) 87 | error_login.filter([400, 401, 403, 404, 500]) 88 | users_in_error = set(error_login.select_column('users')) 89 | 90 | # Prepare log 91 | myreport = dict() 92 | log_user_error = pyreports.Executor(error_log) 93 | log_user_error.filter(list(users_in_error)) 94 | for line in log_user_error: 95 | for user in users_in_error: 96 | myreport.setdefault(user, []) 97 | myreport[user].append(line) 98 | 99 | # Save report: this is a FileManager object 100 | output = pyreports.manager('json', '/home/report/error_login.json') 101 | output.write(myreport) 102 | 103 | ``` 104 | 105 | ### Report object 106 | 107 | ```python 108 | import pyreports 109 | 110 | # Select source: this is a DatabaseManager object 111 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 112 | output = pyreports.manager('xlsx', '/home/report/error_login.xlsx', mode='w') 113 | 114 | # Get data 115 | mydb.execute('SELECT * FROM site_login') 116 | site_login = mydb.fetchall() 117 | 118 | # Create report data 119 | report = pyreports.Report(site_login, title='Site login failed', filters=[400, 401, 403, 404, 500], output=output) 120 | # Filter data 121 | report.exec() 122 | # Save data on file 123 | report.export() 124 | 125 | ``` 126 | 127 | ### ReportBook collection object 128 | 129 | ```python 130 | import pyreports 131 | 132 | # Select source: this is a DatabaseManager object 133 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 134 | 135 | # Get data 136 | mydb.execute('SELECT * FROM site_login') 137 | site_login = mydb.fetchall() 138 | 139 | # Create report data 140 | report_failed = pyreports.Report(site_login, title='Site login failed', filters=[400, 401, 403, 404, 500]) 141 | report_success = pyreports.Report(site_login, title='Site login success', filters=[200, 201, 202, 'OK']) 142 | # Filter data 143 | report_failed.exec() 144 | report_success.exec() 145 | # Create my ReportBook object 146 | my_report = pyreports.ReportBook([report_failed, report_success]) 147 | # Save data on Excel file, with two worksheet ('Site login failed' and 'Site login success') 148 | my_report.export(output='/home/report/site_login.xlsx') 149 | 150 | ``` 151 | 152 | ## Tools for dataset 153 | 154 | This library includes many tools for handling data received from databases and files. 155 | Here are some practical examples of data manipulation. 156 | 157 | ```python 158 | import pyreports 159 | 160 | # Select source: this is a DatabaseManager object 161 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 162 | 163 | # Get data 164 | mydb.execute('SELECT * FROM site_login') 165 | site_login = mydb.fetchall() 166 | 167 | # Most common error 168 | most_common_error_code = pyreports.most_common(site_login, 'code') # args: Dataset, column name 169 | print(most_common_error_code) # 200 170 | 171 | # Percentage of error 404 172 | percentage_error_404 = pyreports.percentage(site_login, 404) # args: Dataset, filter 173 | print(percentage_error_404) # 16.088264794 (percent) 174 | 175 | # Count every error code 176 | count_error_code = pyreports.counter(site_login, 'code') # args: Dataset, column name 177 | print(count_error_code) # Counter({200: 4032, 201: 42, 202: 1, 400: 40, 401: 38, 403: 27, 404: 802, 500: 3}) 178 | ``` 179 | 180 | ### Command line 181 | 182 | ```console 183 | $ cat car.yml 184 | reports: 185 | - report: 186 | title: 'Red ford machine' 187 | input: 188 | manager: 'mysql' 189 | source: 190 | # Connection parameters of my mysql database 191 | host: 'mysql1.local' 192 | database: 'cars' 193 | user: 'admin' 194 | password: 'dba0000' 195 | params: 196 | query: 'SELECT * FROM cars WHERE brand = %s AND color = %s' 197 | params: ['ford', 'red'] 198 | # Filter km 199 | filters: [40000, 45000] 200 | output: 201 | manager: 'csv' 202 | filename: '/tmp/car_csv.csv' 203 | 204 | $ report car.yaml 205 | ``` 206 | 207 | ## Official docs 208 | 209 | In the following links there is the [official documentation](https://pyreports.readthedocs.io/en/latest/), for the use and development of the library. 210 | 211 | * Managers: [doc](https://pyreports.readthedocs.io/en/latest/managers.html) 212 | * Executor: [doc](https://pyreports.readthedocs.io/en/latest/executors.html) 213 | * Report: [doc](https://pyreports.readthedocs.io/en/latest/report.html) 214 | * data tools: [doc](https://pyreports.readthedocs.io/en/latest/datatools.html) 215 | * examples: [doc](https://pyreports.readthedocs.io/en/latest/example.html) 216 | * API: [io](https://pyreports.readthedocs.io/en/latest/dev/io.html), [core](https://pyreports.readthedocs.io/en/latest/dev/core.html) 217 | * CLI: [cli](https://pyreports.readthedocs.io/en/latest/dev/cli.html) 218 | 219 | ## Open source 220 | _pyreports_ is an open source project. Any contribute, It's welcome. 221 | 222 | **A great thanks**. 223 | 224 | For donations, press this 225 | 226 | For me 227 | 228 | [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/guos) 229 | 230 | For [Telethon](http://www.telethon.it/) 231 | 232 | The Telethon Foundation is a non-profit organization recognized by the Ministry of University and Scientific and Technological Research. 233 | They were born in 1990 to respond to the appeal of patients suffering from rare diseases. 234 | Come today, we are organized to dare to listen to them and answers, every day of the year. 235 | 236 | Telethon 237 | 238 | [Adopt the future](https://www.ioadottoilfuturo.it/) 239 | 240 | 241 | ## Acknowledgments 242 | 243 | Thanks to Mark Lutz for writing the _Learning Python_ and _Programming Python_ books that make up my python foundation. 244 | 245 | Thanks to Kenneth Reitz and Tanya Schlusser for writing the _The Hitchhiker’s Guide to Python_ books. 246 | 247 | Thanks to Dane Hillard for writing the _Practices of the Python Pro_ books. 248 | 249 | Special thanks go to my wife, who understood the hours of absence for this development. 250 | Thanks to my children, for the daily inspiration they give me and to make me realize, that life must be simple. 251 | 252 | Thanks Python! -------------------------------------------------------------------------------- /assets/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MatteoGuadrini/pyreports/727825204dd7c63dbcccde00743e6d8199eec438/assets/.DS_Store -------------------------------------------------------------------------------- /assets/css/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MatteoGuadrini/pyreports/727825204dd7c63dbcccde00743e6d8199eec438/assets/css/.DS_Store -------------------------------------------------------------------------------- /assets/css/theme-1.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Template Name: devAid - Bootstrpa 4 Theme for developers 3 | * Version: 2.2 4 | * Author: Xiaoying Riley 5 | * Copyright: 3rd Wave Media 6 | * Twitter: @3rdwave_themes 7 | * Website: https://themes.3rdwavemedia.com/ 8 | */ 9 | /* ======= Base ======= */ 10 | body { 11 | font-family: 'Lato', arial, sans-serif; 12 | color: #444; 13 | font-size: 16px; 14 | -webkit-font-smoothing: antialiased; 15 | -moz-osx-font-smoothing: grayscale; 16 | } 17 | 18 | h1, h2, h3, h4, h5, h6 { 19 | font-family: 'Montserrat', sans-serif; 20 | font-weight: 700; 21 | color: #17baef; 22 | } 23 | 24 | a { 25 | color: #17baef; 26 | -webkit-transition: all 0.4s ease-in-out; 27 | -moz-transition: all 0.4s ease-in-out; 28 | -ms-transition: all 0.4s ease-in-out; 29 | -o-transition: all 0.4s ease-in-out; 30 | } 31 | 32 | a:hover { 33 | text-decoration: underline; 34 | color: #0e98c5; 35 | } 36 | 37 | .btn, a.btn { 38 | -webkit-transition: all 0.4s ease-in-out; 39 | -moz-transition: all 0.4s ease-in-out; 40 | -ms-transition: all 0.4s ease-in-out; 41 | -o-transition: all 0.4s ease-in-out; 42 | font-family: 'Montserrat', arial, sans-serif; 43 | padding: 8px 16px; 44 | font-weight: bold; 45 | } 46 | 47 | .btn .svg-inline--fa, a.btn .svg-inline--fa { 48 | margin-right: 5px; 49 | } 50 | 51 | .btn:focus, a.btn:focus { 52 | color: #fff; 53 | -webkit-box-shadow: none; 54 | -moz-box-shadow: none; 55 | box-shadow: none; 56 | } 57 | 58 | a.btn-cta-primary, .btn-cta-primary { 59 | background: #074f66; 60 | border: 1px solid #074f66; 61 | color: #fff; 62 | font-weight: 600; 63 | text-transform: uppercase; 64 | } 65 | 66 | a.btn-cta-primary:hover, .btn-cta-primary:hover { 67 | background: #053c4e; 68 | border: 1px solid #053c4e; 69 | color: #fff; 70 | } 71 | 72 | a.btn-cta-secondary, .btn-cta-secondary { 73 | background: #eebf3f; 74 | border: 1px solid #eebf3f; 75 | color: #fff; 76 | font-weight: 600; 77 | text-transform: uppercase; 78 | } 79 | 80 | a.btn-cta-secondary:hover, .btn-cta-secondary:hover { 81 | background: #ecb728; 82 | border: 1px solid #ecb728; 83 | color: #fff; 84 | } 85 | 86 | .text-highlight { 87 | color: #074f66; 88 | } 89 | 90 | .offset-header { 91 | padding-top: 90px; 92 | } 93 | 94 | pre code { 95 | font-size: 16px; 96 | } 97 | 98 | /* ======= Header ======= */ 99 | .header { 100 | padding: 10px 0; 101 | background: #17baef; 102 | color: #fff; 103 | position: fixed; 104 | width: 100%; 105 | } 106 | 107 | .header.navbar-fixed-top { 108 | background: #fff; 109 | z-index: 9999; 110 | -webkit-box-shadow: 0 0 4px rgba(0, 0, 0, 0.4); 111 | -moz-box-shadow: 0 0 4px rgba(0, 0, 0, 0.4); 112 | box-shadow: 0 0 4px rgba(0, 0, 0, 0.4); 113 | } 114 | 115 | .header.navbar-fixed-top .logo a { 116 | color: #17baef; 117 | } 118 | 119 | .header .logo { 120 | margin: 0; 121 | font-size: 26px; 122 | padding-top: 10px; 123 | } 124 | 125 | .header .logo a { 126 | color: #fff; 127 | } 128 | 129 | .header .logo a:hover { 130 | text-decoration: none; 131 | } 132 | 133 | .header .main-nav .navbar-collapse { 134 | padding: 0; 135 | } 136 | 137 | .header .main-nav .navbar-toggler { 138 | margin-right: 0; 139 | margin-top: 0; 140 | background: none; 141 | float: right; 142 | margin-top: 8px; 143 | margin-bottom: 8px; 144 | padding: 8px 8px; 145 | right: 10px; 146 | top: 10px; 147 | background: #074f66; 148 | } 149 | 150 | .header .main-nav .navbar-toggler:focus { 151 | outline: none; 152 | } 153 | 154 | .header .main-nav .navbar-toggler .icon-bar { 155 | display: block; 156 | background-color: #fff; 157 | height: 2px; 158 | width: 22px; 159 | -webkit-border-radius: 1px; 160 | -moz-border-radius: 1px; 161 | -ms-border-radius: 1px; 162 | -o-border-radius: 1px; 163 | border-radius: 1px; 164 | -moz-background-clip: padding; 165 | -webkit-background-clip: padding-box; 166 | background-clip: padding-box; 167 | } 168 | 169 | .header .main-nav .navbar-toggler .icon-bar + .icon-bar { 170 | margin-top: 4px; 171 | } 172 | 173 | .header .main-nav .navbar-toggler:hover .icon-bar { 174 | background-color: #fff; 175 | } 176 | 177 | .header .main-nav .nav .nav-item { 178 | font-weight: bold; 179 | margin-right: 30px; 180 | font-family: 'Montserrat', sans-serif; 181 | } 182 | 183 | .header .main-nav .nav .nav-item .nav-link { 184 | color: #09617e; 185 | -webkit-transition: none; 186 | -moz-transition: none; 187 | -ms-transition: none; 188 | -o-transition: none; 189 | font-size: 14px; 190 | padding: 15px 10px; 191 | } 192 | 193 | .header .main-nav .nav .nav-item .nav-link.active { 194 | color: #17baef; 195 | background: none; 196 | } 197 | 198 | .header .main-nav .nav .nav-item .nav-link:hover { 199 | color: #074f66; 200 | background: none; 201 | } 202 | 203 | .header .main-nav .nav .nav-item .nav-link:focus { 204 | outline: none; 205 | background: none; 206 | } 207 | 208 | .header .main-nav .nav .nav-item .nav-link:active { 209 | outline: none; 210 | background: none; 211 | } 212 | 213 | .header .main-nav .nav .nav-item.last { 214 | margin-right: 0; 215 | } 216 | 217 | /* ======= Promo Section ======= */ 218 | .promo { 219 | background: #17baef; 220 | color: #fff; 221 | padding-top: 150px; 222 | } 223 | 224 | .promo .title { 225 | font-size: 98px; 226 | color: #074f66; 227 | margin-top: 0; 228 | } 229 | 230 | .promo .title .highlight { 231 | color: #eebf3f; 232 | } 233 | 234 | .promo .intro { 235 | font-size: 28px; 236 | max-width: 680px; 237 | margin: 0 auto; 238 | margin-bottom: 30px; 239 | } 240 | 241 | .promo .btns .btn { 242 | margin-right: 15px; 243 | font-size: 18px; 244 | padding: 8px 30px; 245 | } 246 | 247 | .promo .meta { 248 | margin-top: 120px; 249 | margin-bottom: 30px; 250 | color: #0a7396; 251 | } 252 | 253 | .promo .meta li { 254 | margin-right: 15px; 255 | } 256 | 257 | .promo .meta a { 258 | color: #0a7396; 259 | } 260 | 261 | .promo .meta a:hover { 262 | color: #074f66; 263 | } 264 | 265 | .promo .social-media { 266 | background: #0c86ae; 267 | padding: 10px 0; 268 | margin: 0 auto; 269 | } 270 | 271 | .promo .social-media li { 272 | margin-top: 15px; 273 | } 274 | 275 | .promo .social-media li.facebook-like { 276 | margin-top: 0; 277 | position: relative; 278 | top: 2px; 279 | } 280 | 281 | /* ======= About Section ======= */ 282 | .about { 283 | padding: 80px 0; 284 | background: #f5f5f5; 285 | } 286 | 287 | .about .title { 288 | color: #074f66; 289 | margin-top: 0; 290 | margin-bottom: 60px; 291 | } 292 | 293 | .about .intro { 294 | max-width: 800px; 295 | margin: 0 auto; 296 | margin-bottom: 60px; 297 | } 298 | 299 | .about .item { 300 | position: relative; 301 | margin-bottom: 30px; 302 | } 303 | 304 | .about .item .icon-holder { 305 | position: absolute; 306 | left: 30px; 307 | top: 0; 308 | } 309 | 310 | .about .item .icon-holder .svg-inline--fa { 311 | font-size: 24px; 312 | color: #074f66; 313 | } 314 | 315 | .about .item .content { 316 | padding-left: 60px; 317 | } 318 | 319 | .about .item .content .sub-title { 320 | margin-top: 0; 321 | color: #074f66; 322 | font-size: 18px; 323 | } 324 | 325 | /* ======= Features Section ======= */ 326 | .features { 327 | padding: 80px 0; 328 | background: #17baef; 329 | color: #fff; 330 | } 331 | 332 | .features .title { 333 | color: #074f66; 334 | margin-top: 0; 335 | margin-bottom: 30px; 336 | } 337 | 338 | .features a { 339 | color: #074f66; 340 | } 341 | 342 | .features a:hover { 343 | color: #042a36; 344 | } 345 | 346 | .features .feature-list li { 347 | margin-bottom: 10px; 348 | color: #074f66; 349 | } 350 | 351 | .features .feature-list li .svg-inline--fa { 352 | margin-right: 5px; 353 | color: #fff; 354 | } 355 | 356 | /* ======= Docs Section ======= */ 357 | .docs { 358 | padding: 80px 0; 359 | background: #f5f5f5; 360 | } 361 | 362 | .docs .title { 363 | color: #074f66; 364 | margin-top: 0; 365 | margin-bottom: 30px; 366 | } 367 | 368 | .docs .docs-inner { 369 | max-width: 800px; 370 | background: #fff; 371 | padding: 30px; 372 | -webkit-border-radius: 4px; 373 | -moz-border-radius: 4px; 374 | -ms-border-radius: 4px; 375 | -o-border-radius: 4px; 376 | border-radius: 4px; 377 | -moz-background-clip: padding; 378 | -webkit-background-clip: padding-box; 379 | background-clip: padding-box; 380 | margin: 0 auto; 381 | } 382 | 383 | .docs .block { 384 | margin-bottom: 60px; 385 | } 386 | 387 | .docs .code-block { 388 | margin: 30px inherit; 389 | } 390 | 391 | .docs .code-block pre[class*="language-"] { 392 | -webkit-border-radius: 4px; 393 | -moz-border-radius: 4px; 394 | -ms-border-radius: 4px; 395 | -o-border-radius: 4px; 396 | border-radius: 4px; 397 | -moz-background-clip: padding; 398 | -webkit-background-clip: padding-box; 399 | background-clip: padding-box; 400 | } 401 | 402 | /* ======= License Section ======= */ 403 | .license { 404 | padding: 80px 0; 405 | background: #f5f5f5; 406 | } 407 | 408 | .license .title { 409 | margin-top: 0; 410 | margin-bottom: 30px; 411 | color: #074f66; 412 | } 413 | 414 | .license .license-inner { 415 | max-width: 800px; 416 | background: #fff; 417 | padding: 30px; 418 | -webkit-border-radius: 4px; 419 | -moz-border-radius: 4px; 420 | -ms-border-radius: 4px; 421 | -o-border-radius: 4px; 422 | border-radius: 4px; 423 | -moz-background-clip: padding; 424 | -webkit-background-clip: padding-box; 425 | background-clip: padding-box; 426 | margin: 0 auto; 427 | } 428 | 429 | .license .info { 430 | max-width: 760px; 431 | margin: 0 auto; 432 | } 433 | 434 | .license .cta-container { 435 | max-width: 540px; 436 | margin: 0 auto; 437 | margin-top: 60px; 438 | -webkit-border-radius: 4px; 439 | -moz-border-radius: 4px; 440 | -ms-border-radius: 4px; 441 | -o-border-radius: 4px; 442 | border-radius: 4px; 443 | -moz-background-clip: padding; 444 | -webkit-background-clip: padding-box; 445 | background-clip: padding-box; 446 | } 447 | 448 | .license .cta-container .speech-bubble { 449 | background: #d6f3fc; 450 | color: #074f66; 451 | padding: 30px; 452 | margin-bottom: 30px; 453 | position: relative; 454 | -webkit-border-radius: 4px; 455 | -moz-border-radius: 4px; 456 | -ms-border-radius: 4px; 457 | -o-border-radius: 4px; 458 | border-radius: 4px; 459 | -moz-background-clip: padding; 460 | -webkit-background-clip: padding-box; 461 | background-clip: padding-box; 462 | } 463 | 464 | .license .cta-container .speech-bubble:after { 465 | position: absolute; 466 | left: 50%; 467 | bottom: -10px; 468 | margin-left: -10px; 469 | content: ""; 470 | display: inline-block; 471 | width: 0; 472 | height: 0; 473 | border-left: 10px solid transparent; 474 | border-right: 10px solid transparent; 475 | border-top: 10px solid #d6f3fc; 476 | } 477 | 478 | .license .cta-container .icon-holder { 479 | margin-bottom: 15px; 480 | } 481 | 482 | .license .cta-container .icon-holder .svg-inline--fa { 483 | font-size: 56px; 484 | } 485 | 486 | .license .cta-container .intro { 487 | margin-bottom: 30px; 488 | } 489 | 490 | /* ======= Contact Section ======= */ 491 | .contact { 492 | padding: 80px 0; 493 | background: #17baef; 494 | color: #fff; 495 | } 496 | 497 | .contact .contact-inner { 498 | max-width: 760px; 499 | margin: 0 auto; 500 | } 501 | 502 | .contact .title { 503 | color: #074f66; 504 | margin-top: 0; 505 | margin-bottom: 30px; 506 | } 507 | 508 | .contact .intro { 509 | margin-bottom: 60px; 510 | } 511 | 512 | .contact a { 513 | color: #074f66; 514 | } 515 | 516 | .contact a:hover { 517 | color: #042a36; 518 | } 519 | 520 | .contact .author-message { 521 | position: relative; 522 | margin-bottom: 60px; 523 | } 524 | 525 | .contact .author-message .profile { 526 | position: absolute; 527 | left: 30px; 528 | top: 15px; 529 | width: 100px; 530 | height: 100px; 531 | } 532 | 533 | .contact .author-message .profile img { 534 | -webkit-border-radius: 50%; 535 | -moz-border-radius: 50%; 536 | -ms-border-radius: 50%; 537 | -o-border-radius: 50%; 538 | border-radius: 50%; 539 | -moz-background-clip: padding; 540 | -webkit-background-clip: padding-box; 541 | background-clip: padding-box; 542 | } 543 | 544 | .contact .author-message .speech-bubble { 545 | margin-left: 155px; 546 | background: #10b2e7; 547 | color: #074f66; 548 | padding: 30px; 549 | -webkit-border-radius: 4px; 550 | -moz-border-radius: 4px; 551 | -ms-border-radius: 4px; 552 | -o-border-radius: 4px; 553 | border-radius: 4px; 554 | -moz-background-clip: padding; 555 | -webkit-background-clip: padding-box; 556 | background-clip: padding-box; 557 | position: relative; 558 | } 559 | 560 | .contact .author-message .speech-bubble .sub-title { 561 | color: #074f66; 562 | font-size: 16px; 563 | margin-top: 0; 564 | margin-bottom: 30px; 565 | } 566 | 567 | .contact .author-message .speech-bubble a { 568 | color: #fff; 569 | } 570 | 571 | .contact .author-message .speech-bubble:after { 572 | position: absolute; 573 | left: -10px; 574 | top: 60px; 575 | content: ""; 576 | display: inline-block; 577 | width: 0; 578 | height: 0; 579 | border-top: 10px solid transparent; 580 | border-bottom: 10px solid transparent; 581 | border-right: 10px solid #10b2e7; 582 | } 583 | 584 | .contact .author-message .speech-bubble .source { 585 | margin-top: 30px; 586 | } 587 | 588 | .contact .author-message .speech-bubble .source a { 589 | color: #074f66; 590 | } 591 | 592 | .contact .author-message .speech-bubble .source .title { 593 | color: #0c86ae; 594 | } 595 | 596 | .contact .info .sub-title { 597 | color: #0e98c5; 598 | margin-bottom: 30px; 599 | margin-top: 0; 600 | } 601 | 602 | .contact .social-icons { 603 | list-style: none; 604 | padding: 10px 0; 605 | margin-bottom: 0; 606 | display: inline-block; 607 | margin: 0 auto; 608 | } 609 | 610 | .contact .social-icons li { 611 | float: left; 612 | } 613 | 614 | .contact .social-icons li.last { 615 | margin-right: 0; 616 | } 617 | 618 | .contact .social-icons a { 619 | display: inline-block; 620 | background: #0c86ae; 621 | width: 48px; 622 | height: 48px; 623 | text-align: center; 624 | font-size: 24px; 625 | -webkit-border-radius: 50%; 626 | -moz-border-radius: 50%; 627 | -ms-border-radius: 50%; 628 | -o-border-radius: 50%; 629 | border-radius: 50%; 630 | -moz-background-clip: padding; 631 | -webkit-background-clip: padding-box; 632 | background-clip: padding-box; 633 | margin-right: 8px; 634 | float: left; 635 | } 636 | 637 | .contact .social-icons a:hover { 638 | background: #e6ad14; 639 | } 640 | 641 | .contact .social-icons a .svg-inline--fa { 642 | color: #fff; 643 | text-align: center; 644 | margin-top: 14px; 645 | font-size: 20px; 646 | } 647 | 648 | /* ======= Footer ======= */ 649 | .footer { 650 | padding: 15px 0; 651 | background: #042a36; 652 | color: #fff; 653 | } 654 | 655 | .footer .copyright { 656 | -webkit-opacity: 0.8; 657 | -moz-opacity: 0.8; 658 | opacity: 0.8; 659 | } 660 | 661 | .footer .fa-heart { 662 | color: #fb866a; 663 | } 664 | 665 | @media (max-width: 767px) { 666 | .header .main-nav .navbar-collapse { 667 | border-top: none; 668 | -webkit-box-shadow: none; 669 | -moz-box-shadow: none; 670 | box-shadow: none; 671 | width: 100%; 672 | left: 0; 673 | top: 60px; 674 | position: absolute; 675 | background: #fff; 676 | } 677 | .header .main-nav .navbar-collapse .navbar-nav { 678 | margin-left: 10px; 679 | } 680 | .header.navbar-fixed-top { 681 | height: 70px; 682 | } 683 | .promo .btns .btn { 684 | margin-right: 0; 685 | clear: both; 686 | display: block; 687 | margin-bottom: 30px; 688 | } 689 | .promo .title { 690 | font-size: 66px; 691 | } 692 | .promo .meta { 693 | margin-top: 60px; 694 | } 695 | .promo .meta li { 696 | float: none; 697 | display: block; 698 | margin-bottom: 5px; 699 | } 700 | .contact .author-message { 701 | text-align: center; 702 | } 703 | .contact .author-message .profile { 704 | position: static; 705 | margin: 0 auto; 706 | margin-bottom: 30px; 707 | } 708 | .contact .author-message .speech-bubble { 709 | margin-left: 0; 710 | } 711 | .contact .author-message .speech-bubble:after { 712 | display: none; 713 | } 714 | .contact .social-icons a { 715 | width: 36px; 716 | height: 36px; 717 | margin-right: 2px; 718 | font-size: 18px; 719 | } 720 | .contact .social-icons a .svg-inline--fa { 721 | margin-top: 7px; 722 | } 723 | } 724 | -------------------------------------------------------------------------------- /assets/js/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MatteoGuadrini/pyreports/727825204dd7c63dbcccde00743e6d8199eec438/assets/js/.DS_Store -------------------------------------------------------------------------------- /assets/js/main.js: -------------------------------------------------------------------------------- 1 | jQuery(document).ready(function($) { 2 | 3 | /* ======= Scrollspy ======= */ 4 | $('body').scrollspy({ target: '#header', offset: 400}); 5 | 6 | /* ======= Fixed header when scrolled ======= */ 7 | 8 | $(window).bind('scroll', function() { 9 | if ($(window).scrollTop() > 50) { 10 | $('#header').addClass('navbar-fixed-top'); 11 | } 12 | else { 13 | $('#header').removeClass('navbar-fixed-top'); 14 | } 15 | }); 16 | 17 | /* ======= ScrollTo ======= */ 18 | $('a.scrollto').on('click', function(e){ 19 | 20 | //store hash 21 | var target = this.hash; 22 | 23 | e.preventDefault(); 24 | 25 | $('body').scrollTo(target, 800, {offset: -70, 'axis':'y', easing:'easeOutQuad'}); 26 | //Collapse mobile menu after clicking 27 | if ($('.navbar-collapse').hasClass('show')){ 28 | $('.navbar-collapse').removeClass('show'); 29 | } 30 | 31 | }); 32 | 33 | }); -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | ldap3 2 | mysql-connector-python 3 | psycopg2-binary 4 | tablib 5 | tablib[all] 6 | nosqlapi 7 | pydata-sphinx-theme -------------------------------------------------------------------------------- /docs/source/_static/pyreports.svg: -------------------------------------------------------------------------------- 1 | 2 | 18 | 20 | 21 | 23 | image/svg+xml 24 | 26 | 27 | 28 | 29 | 30 | 32 | 52 | 54 | 57 | 60 | 63 | 66 | 69 | 72 | 75 | 78 | 81 | 84 | 87 | 90 | 93 | 97 | 98 | 102 | 106 | 110 | 114 | 118 | 122 | 126 | 130 | 134 | 135 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | import os 17 | import sys 18 | 19 | sys.path.insert(0, os.path.abspath("../..")) 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = "pyreports" 24 | copyright = "2025, Matteo Guadrini" 25 | author = "Matteo Guadrini" 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = "1.8.0" 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ["_templates"] 39 | 40 | # List of patterns, relative to source directory, that match files and 41 | # directories to ignore when looking for source files. 42 | # This pattern also affects html_static_path and html_extra_path. 43 | exclude_patterns = [] 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | 47 | # The theme to use for HTML and HTML Help pages. See the documentation for 48 | # a list of builtin themes. 49 | # 50 | html_theme = "pydata_sphinx_theme" 51 | 52 | # Add any paths that contain custom static files (such as style sheets) here, 53 | # relative to this directory. They are copied after the builtin static files, 54 | # so a file named "default.css" will overwrite the builtin "default.css". 55 | html_static_path = ["_static"] 56 | 57 | html_logo = "_static/pyreports.svg" 58 | 59 | master_doc = "index" 60 | -------------------------------------------------------------------------------- /docs/source/datatools.rst: -------------------------------------------------------------------------------- 1 | Data tools 2 | ########## 3 | 4 | The package comes with utility functions to work directly with *Datasets*. 5 | In this section we will see all these functions contained in the **datatools** module. 6 | 7 | 8 | .. toctree:: 9 | 10 | 11 | DataObject 12 | ---------- 13 | 14 | **DataObject** class represents a pure *Dataset*. 15 | 16 | .. autoclass:: pyreports.DataObject 17 | :members: 18 | 19 | .. code-block:: python 20 | 21 | import pyreports, tablib 22 | 23 | data = pyreports.DataObject(tablib.Dataset(*[("Arthur", "Dent", 42)])) 24 | assert isinstance(data.data, tablib.Dataset) == True 25 | 26 | # Clone data 27 | new_data = data.clone() 28 | assert isinstance(new_data.data, tablib.Dataset) == True 29 | 30 | # Select column 31 | new_data.column("name") 32 | new_data.column(0) 33 | 34 | 35 | 36 | DataAdapters 37 | ------------ 38 | 39 | **DataAdapters** class is an object that contains methods that modifying *Dataset*. 40 | 41 | .. code-block:: python 42 | 43 | import pyreports, tablib 44 | 45 | data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)])) 46 | assert isinstance(data.data, tablib.Dataset) == True 47 | 48 | 49 | # Aggregate 50 | planets = tablib.Dataset(*[("Heart",)]) 51 | data.aggregate(planets) 52 | 53 | # Merge 54 | others = tablib.Dataset(*[("Betelgeuse", "Ford", "Prefect", 42)]) 55 | data.merge(others) 56 | 57 | # Counter 58 | data = pyreports.DataAdapters(Dataset(*[("Heart", "Arthur", "Dent", 42)])) 59 | data.merge(self.data) 60 | counter = data.counter() 61 | assert counter["Arthur"] == 2 62 | 63 | # Chunks 64 | data.data.headers = ["planet", "name", "surname", "age"] 65 | assert list(data.chunks(4))[0][0] == ("Heart", "Arthur", "Dent", 42) 66 | 67 | # Deduplicate 68 | data.deduplicate() 69 | assert len(data.data) == 2 70 | 71 | # Subsets 72 | new_data = data.subset("planet", "age") 73 | assert len(data.data[0]) == 2 74 | 75 | # Sort 76 | new_data = data.sort("age") 77 | reverse_data = data.sort("age", reverse=True) 78 | 79 | # Get items 80 | assert data[1] == ("Betelgeuse", "Ford", "Prefect", 42) 81 | 82 | # Iter items 83 | for item in data: 84 | print(item) 85 | 86 | 87 | .. autoclass:: pyreports.DataAdapters 88 | :members: 89 | 90 | DataPrinters 91 | ------------ 92 | 93 | **DataPrinters** class is an object that contains methods that printing *Dataset*'s information. 94 | 95 | .. code-block:: python 96 | 97 | import pyreports, tablib 98 | 99 | data = pyreports.DataPrinters(tablib.Dataset(*[("Arthur", "Dent", 42), ("Ford", "Prefect", 42)], headers=["name", "surname", "age"])) 100 | assert isinstance(data.data, tablib.Dataset) == True 101 | 102 | # Print 103 | data.print() 104 | 105 | # Average 106 | assert data.average(2) == 42 107 | assert data.average("age") == 42 108 | 109 | # Most common 110 | data.data.append(("Ford", "Prefect", 42)) 111 | assert data.most_common(0) == "Ford" 112 | assert data.most_common("name") == "Ford" 113 | 114 | # Percentage 115 | assert data.percentage("Ford") == 66.66666666666666 116 | 117 | # Representation 118 | assert repr(data) == "" 119 | 120 | # String 121 | assert str(data) == 'name |surname|age\n------|-------|---\nArthur|Dent |42 \nFord |Prefect|42 \nFord |Prefect|42 ' 122 | 123 | # Length 124 | assert len(data) == 3 125 | 126 | .. autoclass:: pyreports.DataPrinters 127 | :members: 128 | 129 | 130 | Average 131 | ------- 132 | 133 | **average** function calculates the average of the numbers within a column. 134 | 135 | .. code-block:: python 136 | 137 | import pyreports 138 | 139 | # Build a dataset 140 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 141 | 142 | # Calculate average 143 | print(pyreports.average(mydata, 'salary')) # Column by name 144 | print(pyreports.average(mydata, 2)) # Column by index 145 | 146 | .. attention:: 147 | All values in the column must be ``float`` or ``int``, otherwise a ``ReportDataError`` exception will be raised. 148 | 149 | Most common 150 | ----------- 151 | 152 | The **most_common** function will return the value of a specific column that is most recurring. 153 | 154 | .. code-block:: python 155 | 156 | import pyreports 157 | 158 | # Build a dataset 159 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 160 | mydata.append(('Ford', 'Prefect', 65000)) 161 | 162 | # Get most common 163 | print(pyreports.most_common(mydata, 'name')) # Ford 164 | 165 | Percentage 166 | ---------- 167 | 168 | The **percentage** function will calculate the percentage based on a filter (Any) on the whole *Dataset*. 169 | 170 | .. code-block:: python 171 | 172 | import pyreports 173 | 174 | # Build a dataset 175 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 176 | mydata.append(('Ford', 'Prefect', 65000)) 177 | 178 | # Calculate percentage 179 | print(pyreports.percentage(mydata, 65000)) # 66.66666666666666 (percent) 180 | 181 | Counter 182 | ------- 183 | 184 | The **counter** function will return a `Counter `_ object, with inside it the count of each element of a specific column. 185 | 186 | .. code-block:: python 187 | 188 | import pyreports 189 | 190 | # Build a dataset 191 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 192 | mydata.append(('Ford', 'Prefect', 65000)) 193 | 194 | # Create Counter object 195 | print(pyreports.counter(mydata, 'name')) # Counter({'Arthur': 1, 'Ford': 2}) 196 | 197 | Aggregate 198 | --------- 199 | 200 | The **aggregate** function aggregates multiple columns of some *Dataset* into a single *Dataset*. 201 | 202 | .. warning:: 203 | The number of elements in the columns must be the same. If you want to aggregate columns with a different number of elements, 204 | you need to specify the argument ``fill_empty=True``. Otherwise, an ``InvalidDimension`` exception will be raised. 205 | 206 | .. code-block:: python 207 | 208 | import pyreports 209 | 210 | # Build a datasets 211 | employee = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 212 | places = tablib.Dataset([('London', 'Green palace', 1), ('Helsinky', 'Red palace', 2)], headers=['city', 'place', 'floor']) 213 | 214 | # Aggregate column for create a new Dataset 215 | new_data = pyreports.aggregate(employee['name'], employee['surname'], employee['salary'], places['city'], places['place'])) 216 | new_data.headers = ['name', 'surname', 'salary', 'city', 'place'] 217 | print(new_data) # ['name', 'surname', 'salary', 'city', 'place'] 218 | 219 | Merge 220 | ----- 221 | 222 | The **merge** function combines multiple *Dataset* objects into one. 223 | 224 | .. warning:: 225 | The datasets must have the same number of columns otherwise an ``InvalidDimension`` exception will be raised. 226 | 227 | .. code-block:: python 228 | 229 | import pyreports 230 | 231 | # Build a datasets 232 | employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 233 | employee2 = tablib.Dataset([('Tricia', 'McMillian', 55000), ('Zaphod', 'Beeblebrox', 65000)], headers=['name', 'surname', 'salary']) 234 | 235 | # Merge two Dataset object into only one 236 | employee = pyreports.merge(employee1, employee2) 237 | print(len(employee)) # 4 238 | 239 | Chunks 240 | ------ 241 | 242 | The **chunks** function divides a *Dataset* into pieces from *N* (``int``). This function returns a generator object. 243 | 244 | .. code-block:: python 245 | 246 | import pyreports 247 | 248 | # Build a datasets 249 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 250 | mydata.append(*[('Tricia', 'McMillian', 55000), ('Zaphod', 'Beeblebrox', 65000)]) 251 | 252 | # Divide data into 2 chunks 253 | new_data = pyreports.chunks(mydata, 2) # Generator object 254 | print(list(new_data)) # [[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], [('Tricia', 'McMillian', 55000), ('Zaphod', 'Beeblebrox', 65000)]] 255 | 256 | .. note:: 257 | If the division does not result zero, the last tuple of elements will be a smaller number. 258 | 259 | Deduplicate 260 | ----------- 261 | 262 | The **deduplicate** function remove duplicated rows into *Dataset* objects. 263 | 264 | .. code-block:: python 265 | 266 | import pyreports 267 | 268 | # Build a datasets 269 | employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 270 | 271 | # Remove duplicated rows (removed the last ('Ford', 'Prefect', 65000)) 272 | print(len(pyreports.deduplicate(employee1))) # 2 273 | 274 | Subset 275 | ------ 276 | 277 | The **subset** function make a new *Dataset* with only selected columns. 278 | 279 | .. code-block:: python 280 | 281 | import pyreports 282 | 283 | # Build a datasets 284 | employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 285 | 286 | # Select only a two columns 287 | print(len(pyreports.subset(employee1, 'name', 'surname')[0])) # 2 288 | 289 | Sort 290 | ---- 291 | 292 | The **sort** function sort the *Dataset* by column, also in reversed mode. 293 | 294 | .. code-block:: python 295 | 296 | import pyreports 297 | 298 | # Build a datasets 299 | employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 300 | 301 | # Sort and sort reversed 302 | print(pyreports.sort(employee1, 'salary')) 303 | print(pyreports.sort(employee1, 'salary', reverse=True)) -------------------------------------------------------------------------------- /docs/source/dev/cli.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | :maxdepth: 2 3 | :caption: Contents: 4 | 5 | Command Line Interface 6 | ###################### 7 | 8 | *pyreports* has a command line interface which takes a configuration file in `YAML `_ format as an argument. 9 | 10 | 11 | Command arguments 12 | ***************** 13 | 14 | The only mandatory argument is the `YAML `_ language configuration file. 15 | 16 | Optional arguments 17 | ------------------ 18 | 19 | Here are all the optional flags that the command line interface has. 20 | 21 | +---------------+----------------------+ 22 | | flags | description | 23 | +===============+======================+ 24 | | -v/--verbose | Enable verbose mode | 25 | +---------------+----------------------+ 26 | | -e/--exclude | Exclude reports | 27 | +---------------+----------------------+ 28 | | -V/--version | Print version | 29 | +---------------+----------------------+ 30 | | -h/--help | Print help | 31 | +---------------+----------------------+ 32 | 33 | Report configuration 34 | ******************** 35 | 36 | The *YAML* file representing your reports begins with a **reports** key. 37 | 38 | .. code-block:: yaml 39 | 40 | reports: 41 | # ... 42 | 43 | 44 | Each report you want to define is a **report** key inside *reports*. 45 | 46 | .. code-block:: yaml 47 | 48 | # My reports collection 49 | reports: 50 | # My single report 51 | - report: 52 | 53 | input section 54 | ------------- 55 | 56 | The report section must have a data **input**, which can be file, sql database or LDAP. 57 | 58 | .. code-block:: yaml 59 | :caption: FileManager 60 | 61 | reports: 62 | - report: 63 | # My input 64 | input: 65 | manager: 'log' 66 | filename: '/tmp/test_log.log' 67 | # Apache http log format 68 | params: 69 | pattern: '([(\d\.)]+) (.*) \[(.*?)\] (.*?) (\d+) (\d+) (.*?) (.*?) (\(.*?\))' 70 | headers: ['ip', 'user', 'date', 'req', 'ret', 'size', 'url', 'browser', 'host'] 71 | 72 | .. note:: 73 | Only *log* type has a ``pattern`` params. 74 | 75 | 76 | .. code-block:: yaml 77 | :caption: DatabaseManager 78 | 79 | reports: 80 | - report: 81 | # My input 82 | input: 83 | manager: 'mysql' 84 | source: 85 | # Connection parameters of my mysql database 86 | host: 'mysql1.local' 87 | database: 'cars' 88 | user: 'admin' 89 | password: 'dba0000' 90 | params: 91 | query: 'SELECT * FROM cars WHERE brand = %s AND color = %s' 92 | params: ['ford', 'red'] 93 | 94 | .. attention:: 95 | For complete list of *source* parameters see the various python package for the providers databases. 96 | 97 | .. code-block:: yaml 98 | :caption: LdapManager 99 | 100 | reports: 101 | - report: 102 | # My input 103 | input: 104 | manager: 'ldap' 105 | source: 106 | # Connection parameters of my ldap server 107 | server: 'ldap.local' 108 | username: 'user' 109 | password: 'password' 110 | ssl: False 111 | tls: True 112 | params: 113 | base_search: 'DC=test,DC=local' 114 | search_filter: '(&(objectClass=user)(objectCategory=person))' 115 | attributes: ['name', 'mail', 'phone'] 116 | 117 | output section 118 | -------------- 119 | 120 | **output** is a *Manager* object where save your report data, is same of input data. 121 | 122 | .. attention:: 123 | If *output* is null or absent, the output of data is *stdout*. 124 | 125 | .. code-block:: yaml 126 | :caption: FileManager 127 | 128 | reports: 129 | - report: 130 | # My input 131 | input: 132 | # ... 133 | output: 134 | manager: 'csv' 135 | filename: '/tmp/test_csv.csv' 136 | 137 | 138 | .. code-block:: yaml 139 | :caption: DatabaseManager 140 | 141 | reports: 142 | - report: 143 | # My input 144 | input: 145 | # ... 146 | output: 147 | manager: 'mysql' 148 | source: 149 | # Connection parameters of my mysql database 150 | host: 'mysql1.local' 151 | database: 'cars' 152 | user: 'admin' 153 | password: 'dba0000' 154 | 155 | other section 156 | ------------- 157 | 158 | *report* section has multiple key/value. 159 | 160 | .. code-block:: yaml 161 | 162 | reports: 163 | - report: 164 | # My input 165 | input: 166 | # ... 167 | output: 168 | # ... 169 | title: "One report" 170 | filters: ['string_filter', 42] 171 | map: | 172 | def map_func(integer): 173 | if isinstance(integer, int): 174 | return str(integer) 175 | negation: true 176 | column: "column_name" 177 | count: True 178 | 179 | .. warning:: 180 | **map** section accept any python code. Specify only a function that accept only one argument and with name ``map_func``. 181 | 182 | .. note:: 183 | **filters** could accept also a function that accept only one argument and return a ``bool`` value. 184 | 185 | data tools 186 | ---------- 187 | 188 | *report* section has also some datatools. 189 | 190 | .. code-block:: yaml 191 | 192 | reports: 193 | - report: 194 | # My input 195 | input: 196 | # ... 197 | output: 198 | # ... 199 | sort: 200 | column: age 201 | reverse: true 202 | deduplicate: true 203 | subset: 204 | - name 205 | - surname 206 | 207 | mail settings 208 | ------------- 209 | 210 | Reports can also be sent by email. Just specify the **mail** section. 211 | 212 | .. code-block:: yaml 213 | 214 | reports: 215 | - report: 216 | # My input 217 | input: 218 | # ... 219 | output: 220 | # ... 221 | # Other sections 222 | mail: 223 | server: 'smtp.local' 224 | from: 'ARTHUR DENT ' 225 | to: 'ford.prefect@hitchhikers.com' 226 | cc: 'startiblast@hitchhikers.com' 227 | bcc: 'allmouse@hitchhikers.com' 228 | subject: 'New report mail' 229 | body: 'Report in attachment' 230 | auth: ['user', 'password'] 231 | ssl: true 232 | headers: ['key', 'value'] 233 | 234 | .. warning:: 235 | **mail** settings required **output** settings. 236 | 237 | Report examples 238 | *************** 239 | 240 | Here are some report configurations ranging from the case of reading from a database and writing to a file up to an LDAP server. 241 | 242 | Database example 243 | ---------------- 244 | 245 | Below is an example of a report with data taken from a *mysql* database and save it into *csv* file. 246 | 247 | .. code-block:: yaml 248 | 249 | reports: 250 | - report: 251 | title: 'Red ford machine' 252 | input: 253 | manager: 'mysql' 254 | source: 255 | # Connection parameters of my mysql database 256 | host: 'mysql1.local' 257 | database: 'cars' 258 | user: 'admin' 259 | password: 'dba0000' 260 | params: 261 | query: 'SELECT * FROM cars WHERE brand = %s AND color = %s' 262 | params: ['ford', 'red'] 263 | # Filter km 264 | filters: [40000, 45000] 265 | output: 266 | manager: 'csv' 267 | filename: '/tmp/car_csv.csv' 268 | 269 | LDAP example 270 | ------------ 271 | 272 | Reports of users who have passwords without expiration by saving it in an *excel* file and sending it by email. 273 | 274 | .. code-block:: yaml 275 | 276 | reports: 277 | - report: 278 | title: 'Users who have passwords without expiration' 279 | input: 280 | manager: 'ldap' 281 | source: 282 | # Connection parameters of my ldap server 283 | server: 'ldap.local' 284 | username: 'user' 285 | password: 'password' 286 | ssl: False 287 | tls: True 288 | params: 289 | base_search: 'DC=test,DC=local' 290 | search_filter: '(&(objectCategory=person)(objectClass=user)(!userAccountControl:1.2.840.113556.1.4.803:=65536))' 291 | attributes: ['cn', 'mail', 'phone'] 292 | # Append prefix number on phone number 293 | map: | 294 | def map_func(phone): 295 | if phone.startswith('33'): 296 | return '+39' + phone 297 | output: 298 | manager: 'xlsx' 299 | filename: '/tmp/users.xlsx' 300 | mail: 301 | server: 'smtp.local' 302 | from: 'ARTHUR DENT [SAVE ORIGIN] -> [PROCESS] -> [OUTPUT]`` 92 | 93 | .. code-block:: python 94 | 95 | import pyreports 96 | import tablib 97 | import os 98 | 99 | # Define my Executor class 100 | class MyReport(pyreports.Report): 101 | 102 | def save_origin(self): 103 | # Save origin in origin file 104 | if self.output: 105 | self.output.write(self.data) 106 | os.rename(self.output.file, 'origin_' + self.output.file) 107 | # Process report 108 | self.export() 109 | 110 | # Test MyReport 111 | salary55k = pyreports.manager('csv', '/tmp/salary55k.csv') 112 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 113 | report_only_55k = MyReport(mydata, filters=[55000], title='Report salary 55k', output=salary55k) 114 | 115 | # My workflow report: [INPUT] -> [SAVE ORIGIN] -> [PROCESS] -> [OUTPUT] 116 | report_only_55k.save_origin() 117 | 118 | 119 | Always print 120 | ------------ 121 | 122 | Another highly requested feature is to save and print at the same time. Much like the Unix ``tee`` shell command, 123 | we will implement the new functionality in our custom *Report* object. 124 | 125 | .. code-block:: python 126 | 127 | import pyreports 128 | import tablib 129 | 130 | # Define my Executor class 131 | class MyReport(pyreports.Report): 132 | 133 | def tee(self): 134 | # Print data... 135 | print(self) 136 | # ...and save! 137 | self.export() 138 | 139 | # Test MyReport 140 | salary55k = pyreports.manager('csv', '/tmp/salary55k.csv') 141 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 142 | report_only_55k = MyReport(mydata, filters=[55000], title='Report salary 55k', output=salary55k) 143 | 144 | # Print and export 145 | report_only_55k.tee() 146 | 147 | Extend ReportBook 148 | ***************** 149 | 150 | The ``ReportBook`` object is a collection of ``Report`` type objects. 151 | When you iterate over an object of this type, you get a generator that returns the *Report* objects it contains one at a time. 152 | 153 | .. note:: 154 | Nothing prevents that you can also insert the ``MyReport`` classes created previously. They are also subclasses of ``Reports``. 155 | 156 | Book to dict 157 | ------------ 158 | 159 | One of the features that might interest you is to export a *ReportBook* as if it were a dictionary. 160 | 161 | .. code-block:: python 162 | 163 | import pyreports 164 | import tablib 165 | 166 | 167 | # Instantiate the Report objects 168 | mydata = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 169 | report_only_55k = pyreports.Report(mydata, filters=[55000], title='Report salary 55k') 170 | report_only_65k = pyreports.Report(mydata, filters=[65000], title='Report salary 65k') 171 | 172 | class MyReportBook(pyreports.ReportBook): 173 | 174 | def to_dict(self): 175 | return {report.title: report for report in self if report.title} 176 | 177 | # Test my book 178 | salary = MyReportBook([report_only_55k, report_only_65k]) 179 | salary.to_dict() # {'Report salary 55k': , 'Report salary 65k': } 180 | 181 | -------------------------------------------------------------------------------- /docs/source/dev/io.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | 3 | io 4 | ## 5 | 6 | In this section, you will find information on how to add new types of ``*Connection`` objects, ``*File`` objects, or ``*Manager`` objects. 7 | 8 | Connection 9 | ********** 10 | 11 | Each ``*Connection`` object inherits from the abstract ``Connection`` class, which forces each type of connection object to 12 | accept these arguments when creating the object: 13 | 14 | - ``args``, various positional arguments 15 | - ``kwargs``, various keyword arguments 16 | 17 | Besides this, the class must have a ``connect`` and a ``close`` method, respectively to connect to the database and one to close the connection, 18 | respectively. 19 | 20 | 21 | .. literalinclude:: ../../../pyreports/io.py 22 | :language: python 23 | :pyobject: Connection 24 | 25 | 26 | Example ``Connection`` based class: 27 | 28 | 29 | .. literalinclude:: ../../../pyreports/io.py 30 | :language: python 31 | :pyobject: SQLiteConnection 32 | 33 | .. warning:: 34 | All connections are `DBAPI 2.0 `_ compliant. If you need to create your own, it must adhere to these APIs. 35 | 36 | File 37 | **** 38 | 39 | The ``File`` is the abstract class that the other ``*File`` classes are based on. 40 | It contains only the ``file`` attribute, where the path of the file is saved during the creation of the object and two methods: 41 | ``read`` to read the contents of the file (must return a Dataset object) and ``write`` (accept a Dataset) and writes to the destination file. 42 | 43 | 44 | 45 | .. literalinclude:: ../../../pyreports/io.py 46 | :language: python 47 | :pyobject: File 48 | 49 | 50 | 51 | 52 | Example ``File`` based class: 53 | 54 | .. literalinclude:: ../../../pyreports/io.py 55 | :language: python 56 | :pyobject: CsvFile 57 | 58 | Alias 59 | ***** 60 | 61 | When creating a ``Connection`` or ``File`` class, if you want to use the ``manager`` function to create the returning ``*Manager`` object, 62 | you need to create an alias. There are two dicts in the ``io`` module, which represent the aliases of these objects. 63 | If you have created a new ``Connection`` class, you will need to enter your alias in the ``DBTYPE`` *dict* while for File-type classes, 64 | enter it in the ``FILETYPE`` *dict*. Here is an example: ``'ods': ODSFile`` 65 | 66 | 67 | 68 | Manager 69 | ******* 70 | 71 | Managers are classes that represent an input and output manager. For example, the ``DatabaseManager`` class accepts a 72 | ``Connection`` object and implements methods on these types of objects representing database connections. 73 | 74 | 75 | 76 | .. literalinclude:: ../../../pyreports/io.py 77 | :language: python 78 | :pyobject: DatabaseManager 79 | 80 | 81 | Manager function 82 | ---------------- 83 | 84 | Each ``*Manager`` class has associated a function of type ``create__manager(*args, **kwargs)``. 85 | This function will then be used by the ``manager`` function to create the corresponding ``*Manager`` object based on its alias. 86 | 87 | For example, the ``DatabaseManager`` class has associated the ``create_database_manager`` function which will be called by the 88 | ``manager`` function to create the object based on the type of alias passed. 89 | 90 | 91 | 92 | .. literalinclude:: ../../../pyreports/io.py 93 | :language: python 94 | :pyobject: manager 95 | 96 | 97 | 98 | .. literalinclude:: ../../../pyreports/io.py 99 | :language: python 100 | :pyobject: create_database_manager 101 | 102 | Example 103 | ******* 104 | 105 | Here we will see how to create your own ``*Connection`` class to access a specific database. 106 | 107 | .. code-block:: python 108 | 109 | import pyreports 110 | import DB2 111 | 112 | # class for connect DB2 database 113 | class DB2Connection(pyreports.io.Connection): 114 | 115 | def connect(self): 116 | self.connection = DB2.connect(*self.args, **self.kwargs) 117 | self.cursor = self.connection 118 | 119 | def close(self): 120 | self.connection.close() 121 | self.cursor.close() 122 | 123 | # Create an alias for DB2Connection object 124 | pyreports.io.DBTYPE['db2'] = DB2Connection 125 | 126 | # Create my DatabaseManager object 127 | mydb2 = pyreports.manager('db2', dsn='sample', uid='db2inst1', pwd='ibmdb2') 128 | -------------------------------------------------------------------------------- /docs/source/example.rst: -------------------------------------------------------------------------------- 1 | pyreports example 2 | ################# 3 | 4 | Example scripts using ``pyreports`` module. 5 | 6 | .. toctree:: 7 | 8 | 9 | 10 | 11 | Basic usage 12 | *********** 13 | 14 | In this section you will find examples that represent the entire reporting workflow, relying on the *\*Manager* objects as input and output, and the *Executor* object for the process part. 15 | 16 | Database to file 17 | ---------------- 18 | 19 | In this example, we extract the data from a mysql database, filter it by error code and finally export it to a csv. 20 | 21 | .. code-block:: python 22 | 23 | import pyreports 24 | 25 | # INPUT 26 | 27 | # Select source: this is a DatabaseManager object 28 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000') 29 | 30 | # Get data 31 | mydb.execute('SELECT * FROM site_login') 32 | site_login = mydb.fetchall() # return Dataset object 33 | 34 | # PROCESS 35 | 36 | # Filter data 37 | error_login = pyreports.Executor(site_login) # accept Dataset object 38 | error_login.filter([400, 401, 403, 404, 500]) 39 | 40 | # OUTPUT 41 | 42 | # Save report: this is a FileManager object 43 | output = pyreports.manager('csv', '/home/report/error_login.csv') 44 | output.write(error_login.get_data()) 45 | 46 | 47 | .. note:: 48 | A reflection on this example could be: "Why don't I apply the filter directly in the SQL syntax?" 49 | The answer is simple. The advantage of using an *Executor* object is that from general data I can filter or modify 50 | (*map* function or with my custom function) without affecting the original Dataset. So much so that I could do several 51 | different Executors, process them and then re-merge them into a single Executor, which would be difficult to do with SQL syntax. 52 | 53 | File to Database 54 | ---------------- 55 | 56 | In this example I have a json file as input, received from a web server, I process it and write to the database. 57 | 58 | .. code-block:: python 59 | 60 | import pyreports 61 | 62 | # INPUT 63 | 64 | # Return json from GET request on web server: this is a FileManager object 65 | web_server_result = pyreports.manager('json', '/home/report/users.json') 66 | # Get data 67 | users = web_server_result.read() # return Dataset object 68 | 69 | # PROCESS 70 | 71 | # Filter data 72 | user_int = pyreports.Executor(users) # accept Dataset object 73 | user_int.filter(key=lambda record: if record == 'INTERNAL') # My filter is a function 74 | user_ext = pyreports.Executor(users) 75 | user_ext.filter(key=lambda record: if record == 'EXTERNAL') 76 | 77 | # OUTPUT 78 | 79 | # Save report: this is a DatabaseManager object 80 | mydb = pyreports.manager('mysql', host='mysql1.local', database='users', user='dba', password='dba0000') 81 | 82 | # Write to database 83 | mydb.executemany("INSERT INTO internal_users(name, surname, employeeType) VALUES(%s, %s, %s)", list(user_int)) 84 | mydb.executemany("INSERT INTO external_users(name, surname, employeeType) VALUES(%s, %s, %s)", list(user_ext)) 85 | mydb.commit() 86 | 87 | 88 | Combine inputs 89 | -------------- 90 | 91 | In this example, we will take two different inputs, and combine them to export an excel file containing the data processing of the two sources. 92 | 93 | .. code-block:: python 94 | 95 | import pyreports 96 | 97 | # INPUT 98 | 99 | # Config Unix application file: this is a FileManager object 100 | config_file = pyreports.manager('yaml', '/home/myapp.yml') 101 | # Console admin: this is a DatabaseManager object 102 | mydb = pyreports.manager('mysql', server='mysql1.local', database='admins', user='sa', password='sa0000') 103 | # Get data 104 | admin_app = config_file.read() # return Dataset object: three column (name, shell, login) 105 | mydb.execute('SELECT * FROM console_admins') 106 | admins = mydb.fetchall() # return Dataset object: three column (name, shell, login) 107 | 108 | # PROCESS 109 | 110 | # Filter data 111 | all_console_admins = pyreports.Executor(admins) # accept Dataset object 112 | all_console_admins.filter(config_file['shell']) # filter by shells 113 | 114 | # OUTPUT 115 | 116 | # Save report: this is a FileManager object 117 | output = pyreports.manager('xlsx', '/home/report/all_admins.xlsx') 118 | output.write(all_console_admins.get_data()) 119 | 120 | Simple report 121 | ------------- 122 | 123 | In this example, we use a Report type object to create and filter the data through a function and save it in a csv file, printing the number of lines in total. 124 | 125 | .. code-block:: python 126 | 127 | import pyreports 128 | 129 | OFFICE_FILTER = 'Customer' 130 | 131 | # Function: filter by office 132 | def filter_by_office(value): 133 | if value == OFFICE_FILTER: 134 | return True 135 | 136 | # Connect to database 137 | mydb = pyreports.manager('postgresql', host='pssql1.local', database='users', user='admin', password='pwd0000') 138 | mydb.execute('SELECT * FROM employees') 139 | all_employees = mydb.fetchall() 140 | # Output to csv 141 | output = pyreports.manager('csv', f'/home/report/office_{OFFICE_FILTER}.csv') 142 | # All customer employees: Report object 143 | one_office = pyreports.Report(all_employees, 144 | filters=filter_by_office, 145 | title=f'All employees in {OFFICE_FILTER}', 146 | count=True, 147 | output=output) 148 | # Run and save report 149 | one_office.export() 150 | print(one_office.count) # Row count 151 | 152 | 153 | Advanced usage 154 | ************** 155 | 156 | From here on, the examples will be a bit more complex; we will process the data in order to modify it, filter it, 157 | combine it and merge it before exporting or parsing it in another object. 158 | 159 | Report apache log 160 | ----------------- 161 | 162 | In this example we will analyze and capture parts of a web server log. For each error code present in the log, we will 163 | create a report that will be inserted in a book, where each sheet will contain the details of the error code. 164 | In the last sheet, there will be an element counter for every single error present in the report. 165 | 166 | .. code-block:: python 167 | 168 | import pyreports 169 | import tablib 170 | import re 171 | 172 | # Get apache log data: this is a FileManager object 173 | apache_log = pyreports.manager('file', '/var/log/httpd/error.log').read() 174 | # apache log format: regex 175 | regex = '([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) - "(.*?)" "(.*?)"' 176 | 177 | # Function than receive Dataset and return a new Dataset 178 | def format_dataset_log(data_input): 179 | data = tablib.Dataset(headers=['ip', 'date', 'operation', 'code', 'client']) 180 | for row in data_input: 181 | log_parts = re.match(regex, row[0]).groups() 182 | new_row = list(log_parts[:4]) 183 | new_row.append(log_parts[5]) 184 | data.append(new_row) 185 | return data 186 | 187 | # Create a collection of Report objects 188 | all_apache_error = pyreports.ReportBook(title='Apache error on my site') 189 | 190 | # Create a Report object based on error code 191 | apache_error_log = format_dataset_log(apache_log) 192 | all_error = set(apache_error_log['code']) 193 | for code in all_error: 194 | all_apache_error.add(pyreports.Report(apache_error_log, filters=[code], title=f'Error {code}')) 195 | 196 | # Count all error code 197 | counter = pyreports.counter(apache_error_log, 'code') 198 | # Append new Report on ReportBook with error code counters 199 | error_counter = tablib.Dataset(counter.values(), headers=counter) 200 | all_apache_error.add(pyreports.Report(error_counter)) 201 | 202 | # Save ReportBook on Excel 203 | all_apache_error.export('/home/report/apache_log_error_code.xlsx') 204 | 205 | We now have a script that parses and breaks an apache httpd log file by error code. 206 | 207 | Report e-commerce data 208 | ---------------------- 209 | 210 | In this example, we combine data from different e-commerce databases. 211 | In addition, we will create two reports: one for the sales, the other for the warehouse. 212 | Then once saved, we will create an additional report that combines both of the previous ones. 213 | 214 | .. code-block:: python 215 | 216 | import pyreports 217 | 218 | # Get data from database: a DatabaseManager object 219 | mydb = pyreports.manager('postgresql', host='pssql1.local', database='ecommerce', user='reader', password='pwd0000') 220 | mydb.execute('SELECT * FROM sales') 221 | sales = mydb.fetchall() 222 | mydb.execute('SELECT * FROM warehouse') 223 | warehouse = mydb.fetchall() 224 | 225 | # filters 226 | household = ['plates', 'glass', 'fork'] 227 | clothes = ['shorts', 'tshirt', 'socks'] 228 | 229 | # Create sales Report objects 230 | sales_by_household= pyreports.Report(sales, filter=household, title='household sold items') 231 | sales_by_clothes = pyreports.Report(sales, filter=clothes, title='clothes sold items') 232 | 233 | # Create warehouse Report objects 234 | warehouse_by_household= pyreports.Report(warehouse, filter=household, title='household items in warehouse') 235 | warehouse_by_clothes = pyreports.Report(warehouse, filter=clothes, title='clothes items in warehouse') 236 | 237 | # Create a ReportBook objects 238 | sales_book = pyreports.ReportBook([sales_by_household, sales_by_clothes], filter='Total sold') 239 | warehouse_book = pyreports.ReportBook([warehouse_by_household, warehouse_by_clothes], filter='Total remained') 240 | 241 | # Save reports 242 | sales_book.export('/home/report/sales.xlsx') 243 | warehouse_book.export('/home/report/warehouse.xlsx') 244 | 245 | # Other report: combine two book 246 | all = sales_book + warehouse_book 247 | all.export('/home/report/all.xlsx') 248 | 249 | # Now print to stdout all data 250 | all.export() 251 | 252 | Command line report 253 | ------------------- 254 | 255 | In this example, we're going to create a script that doesn't save any files. We will read from a database, modify the data 256 | so that it is more readable and print it in standard output. We will also see how to use our script with other command line tools. 257 | 258 | .. code-block:: python 259 | 260 | import pyreports 261 | 262 | # Get data from database: a DatabaseManager object 263 | mydb = pyreports.manager('sqllite', database='/var/myapp/myapp.db') 264 | mydb.execute('SELECT * FROM performance') 265 | performance = mydb.fetchall() 266 | 267 | # Transform data for command line reader 268 | cmd = pyreports.Executor(performance) 269 | 270 | def number_to_second(seconds): 271 | if isinstance(seconds, int): 272 | ret = float(int) 273 | return f'{ret:.2f} s' 274 | else: 275 | return seconds 276 | 277 | cmd.map(number_to_second) 278 | 279 | # Print data 280 | print(cmd.get_data()) 281 | 282 | Now we can read the db directly from the command line. 283 | 284 | .. code-block:: console 285 | 286 | $ python performance.py 287 | $ python performance.py | grep -G "12.*" 288 | 289 | .. note:: 290 | The examples we can give are almost endless. This library has such flexible python objects that we can adapt them to any use case. 291 | You can also use it as a simple database data reader. 292 | 293 | Use cases 294 | ********* 295 | 296 | As you may have noticed, there are many use cases for this library. The ``manager`` objects are so flexible that you 297 | can read and write data from any source. 298 | Furthermore, thanks to the ``Executor`` objects you can filter and modify the data on-demand when you want and restore 299 | it at a later time, and then channel it into the ``Report`` objects and then into the ``ReportBook`` collection objects. 300 | 301 | Below, I'll list other use cases common to both package users and developers: 302 | 303 | - Export LDAP users and insert them into a database 304 | - Read a log file and write it into a database 305 | - Find out which LDAP users are present in a web server log file 306 | - Backup configuration files by exporting them in yaml format (passwd, httpd.conf, etc) 307 | - Calculate access rates of a database 308 | - Count how many times an ip address is present in a log file 309 | 310 | I could go on indefinitely; anything you can think of about a file, a database and an LDAP server and you need to 311 | manipulate or verify the data, this is the library for you. -------------------------------------------------------------------------------- /docs/source/executors.rst: -------------------------------------------------------------------------------- 1 | Executors 2 | ######### 3 | 4 | The **Executor** object is the one who analyzes and processes the data that is instantiated with it. 5 | This type of object is the first core object we will see and the basis for all the others. 6 | 7 | 8 | .. toctree:: 9 | 10 | 11 | 12 | 13 | Executor at work 14 | **************** 15 | 16 | To instantiate an *Executor* object, you need two things: the mandatory one, a **Dataset** and the other optional is a **header**, 17 | which represents the data. Let's see how to instantiate an *Executor* object. 18 | 19 | .. code-block:: python 20 | 21 | import pyreports 22 | 23 | # Create a data source 24 | mydb = pyreports.manager('mysql', host='mysql1.local', database='test', user='dba', password='dba0000') 25 | 26 | # Get data 27 | mydb.execute('SELECT * FROM salary') 28 | employees = mydb.fetchall() # return Dataset object 29 | 30 | # Create Executor object 31 | myex = pyreports.Executor(employees) # The employees object already has a header, as it was created by a database manager 32 | 33 | .. note:: 34 | If I wanted to apply a header different from the name of the table columns, perhaps because they are not very speaking or full of underscores, I would have to instantiate the object as follows: 35 | ``myex = pyreports.Executor(employees, header=['name', 'surname', 'salary'])``. 36 | If you wanted to remove the header instead, just set it as ``None``: ``myex = pyreports.Executor(employees, header=None)`` 37 | 38 | The *Executor* is a flexible object. It is not related to the *pyreports* library. An *Executor* can also be instantiated via 39 | its own Dataset or from a list of tuples (Python primitives used to instantiate a Dataset object. It is also equal to 40 | the return value of a database object) 41 | 42 | .. code-block:: python 43 | 44 | import pyreports 45 | import tablib 46 | 47 | # Create my Dataset object 48 | mydata = tablib.Dataset() 49 | mydata.append(['Arthur', 'Dent', 55000]) 50 | mydata.append(['Ford', 'Prefect', 65000]) 51 | 52 | # Create Executor object: same result for both 53 | myex = pyreports.Executor(mydata, header=['name', 'surname', 'salary']) 54 | myex = pyreports.Executor([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], header=['name', 'surname', 'salary']) 55 | 56 | # Set header after creation 57 | myex.headers(['name', 'surname', 'salary']) 58 | 59 | Filter data 60 | ----------- 61 | 62 | One of the main functions of working with data is to filter it. The *Executor* object has a filter method for doing this. 63 | This method accepts a list of values that must correspond to one of the values of a row in the Executor's Dataset. 64 | 65 | Another way to filter the data of an *Executor* object is to pass a callable that takes a single argument and returns something. 66 | The return value will be called by the ``bool`` class to see if it is ``True`` or ``False``. 67 | This callable will be called to every single value of the row of the Executor's Dataset. 68 | 69 | Finally, it is possible to declare the name of a single return column, if not all columns are needed. 70 | 71 | .. note:: 72 | You can pass both a list of values and a function to filter the data. 73 | 74 | .. code-block:: python 75 | 76 | # Filter data by list 77 | myex.filter([55000, 65000, 75000]) # Filter data only for specified salaries 78 | 79 | # Filter data by callable 80 | myex.filter(key=str.istitle) # Filter data only for string contains Title case 81 | 82 | def big_salary(salary): 83 | if not isinstance(salary, int): 84 | return False 85 | return True if salary >= 65000 else False # My custom function 86 | 87 | myex.filter(key=big_salary) # Filter data with a salary greater than or equal to 65000 88 | 89 | # Filter data by column 90 | myex.filter([55000, 65000, 75000], column='salary') # Filter by column: name 91 | myex.filter([55000, 65000, 75000], column=2) # Filter by column: index 92 | 93 | # Filter data by list, callable and column 94 | myex.filter([55000, 65000, 75000], str.istitle, 'salary') # Filter for all three methods 95 | 96 | .. warning:: 97 | If the filters are not applied, the result will be an empty Executor object. 98 | If you want to reapply a filter, you will have to reset the object, using the ``reset()`` method. See below. 99 | 100 | Map (modify) data 101 | ----------------- 102 | 103 | The *Executor* object is provided with a method to modify the data in real time. 104 | The ``map`` method accepts a mandatory argument, i.e. a callable that accepts a single argument and an optional one 105 | that accepts the name of the column or the number of its index. 106 | 107 | .. code-block:: python 108 | 109 | # Define my function for increase salary; isn't that amazing! 110 | def salary_increase(salary): 111 | if isinstance(salary, int): 112 | if salary <= 65000: 113 | return salary + 10000 114 | return salary 115 | 116 | # Let's go! Increase salary today! 117 | myex.map(salary_increase) 118 | 119 | # Apply only salary columns 120 | myex.map(salary_increase, column='salary') 121 | 122 | .. warning:: 123 | If the function you are passing to the *map* method returns nothing, ``None`` will be substituted for the original value. 124 | If you are using special conditions make sure your function always returns to its original value. 125 | 126 | Get data 127 | -------- 128 | 129 | An *Executor* is not a data object. It is an object that contains data for processing, filters and etc. 130 | Once an instance of an *Executor* object is created, the original data is saved so that it can be retrieved. 131 | 132 | So there is a way to retrieve and print the current and original data. 133 | 134 | .. code-block:: python 135 | 136 | # Get data 137 | myex.get_data() # Return current Dataset object 138 | myex.origin # Return original Dataset object 139 | print(myex.get_data()) # Print Dataset with current data 140 | 141 | # Assign result to variable 142 | my_dataset = myex.get_data() # Return Dataset object 143 | 144 | # Create a new executor 145 | new_ex = pyreports.Executor(myex.get_data()) # New Executor object with current data 146 | new_ex = myex.clone() # New Executor object with original data 147 | 148 | .. note:: 149 | If you want to clone the original data contained in an Executor object, use the ``clone`` method. 150 | 151 | It is possible through this object, to restore the data source after the modification or the applied filter. 152 | 153 | .. code-block:: python 154 | 155 | # Restore data 156 | myex.reset() # Reset data to origin 157 | print(myex.get_data()) 158 | 159 | .. attention:: 160 | Once the object is reset, any changes made will be lost, unless the object has been cloned. 161 | 162 | Work with columns 163 | ----------------- 164 | 165 | Since the *Executor* object is based on a Dataset object, it is possible to work not only with rows but also with columns. 166 | Let's see how to select a single column. 167 | 168 | .. code-block:: python 169 | 170 | # Select column 171 | myex.select_column(1) # Select column by index number (surname) 172 | myex.select_column('surname') # Select column by name (surname) 173 | 174 | We can also add columns as long as they are the same length as the others, otherwise, we will receive an ``InvalidDimension`` exception. 175 | 176 | .. code-block:: python 177 | 178 | # Add column with values 179 | myex.add_column('floor', [1, 2]) 180 | 181 | # Add column with function values 182 | def stringify_salary(row): 183 | return f'$ {row[2]}' 184 | 185 | myex.add_column('str_salary', stringify_salary) 186 | 187 | .. note:: 188 | The function passed to the ``add_column`` method must have a single argument representing the row (the name *"row"* is a convention). 189 | You can use this argument to access data from other columns. 190 | 191 | It is also possible to delete a column. 192 | 193 | .. code-block:: python 194 | 195 | # Delete column 196 | myex.del_column('floor') 197 | 198 | Count 199 | ----- 200 | 201 | The *Executor* object contains data. You may need to count rows and columns. 202 | The object supports the protocol for counting through the built-in ``len`` function, which will return the current number of rows. 203 | 204 | .. code-block:: python 205 | 206 | # Count columns 207 | myex.count_columns() # Return number of columns 208 | 209 | # Count rows 210 | myex.count_rows() # Return number of rows 211 | len(myex) # Return number of rows 212 | 213 | Iteration 214 | --------- 215 | 216 | The *Executor* object supports the python iteration protocol (return of generator object). 217 | This means that you can use it in a for loop or in a list comprehension. 218 | 219 | .. code-block:: python 220 | 221 | # For each row in Executor 222 | for row in myex: 223 | print(row) 224 | 225 | # List comprehension 226 | my_list_of_rows = [row for row in myex] -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. pyreports documentation master file, created by 2 | sphinx-quickstart on Mon May 3 12:34:29 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pyreports's documentation! 7 | ===================================== 8 | 9 | *pyreports* is a python library that allows you to create complex reports from various sources such as databases, 10 | text files, ldap, etc. and perform processing, filters, counters, etc. and then export or write them in various formats or in databases. 11 | 12 | You can use this library for complex reports, or to simply filter data into datasets divided by topic. Furthermore, 13 | it is possible to export in various formats, such as csv, excel files or write directly to the database (*mysql*, *mssql*, *postgresql* and more). 14 | 15 | .. _workflow: 16 | 17 | Report workflow 18 | *************** 19 | 20 | This package provides tools for receiving, processing and exporting data. Mostly, it follows this workflow. 21 | 22 | .. code-block:: 23 | 24 | +-----------------+ +-----------------+ +-----------------+ 25 | | | | | | | 26 | | | | | | | 27 | | INPUT +----->| PROCESS +----->| OUTPUT | 28 | | | | | | | 29 | | | | | | | 30 | +-----------------+ +-----------------+ +-----------------+ 31 | 32 | 33 | Features 34 | ******** 35 | 36 | - Capture any type of data 37 | - Export data in many formats 38 | - Data analysis 39 | - Process data with filters and maps 40 | - Some functions will help you to process averages, percentages and much more 41 | 42 | .. toctree:: 43 | :maxdepth: 2 44 | :caption: Contents: 45 | 46 | install 47 | managers 48 | executors 49 | report 50 | datatools 51 | example 52 | package 53 | 54 | .. toctree:: 55 | :maxdepth: 2 56 | :caption: API: 57 | 58 | dev/io 59 | dev/core 60 | 61 | .. toctree:: 62 | :maxdepth: 2 63 | :caption: CLI: 64 | 65 | dev/cli 66 | 67 | 68 | Indices and tables 69 | ================== 70 | 71 | * :ref:`genindex` 72 | * :ref:`modindex` 73 | * :ref:`search` 74 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ############ 3 | 4 | Here are the installation instructions 5 | 6 | .. toctree:: 7 | 8 | 9 | 10 | 11 | Requirements 12 | ************ 13 | 14 | *pyreports* is written in python3 (3.8 and higher). 15 | 16 | Here are all the external libraries necessary for the proper functioning of the library: 17 | 18 | - `tablib `_ 19 | - `ldap3 `_ 20 | - `nosqlapi `_ 21 | - `pyYaml `_ 22 | - `mysql-connector-python `_ 23 | - `psycopg2-binary `_ 24 | 25 | 26 | Installation 27 | ************ 28 | 29 | .. code-block:: console 30 | 31 | $ pip install pyreports # on PyPi 32 | $ git clone https://github.com/MatteoGuadrini/pyreports.git #from official repo 33 | $ cd pyreports 34 | $ pip install . --upgrade 35 | -------------------------------------------------------------------------------- /docs/source/managers.rst: -------------------------------------------------------------------------------- 1 | Managers 2 | ######## 3 | 4 | The manager objects are responsible for managing inputs or outputs. We can have three macro types of managers: *database*, *file* and *ldap*. 5 | 6 | 7 | .. toctree:: 8 | 9 | 10 | 11 | 12 | Type of managers 13 | **************** 14 | 15 | Each type of manager is managed by micro types; Below is the complete list: 16 | 17 | #. Database 18 | #. sqlite (SQLite) 19 | #. mysql (MySQL or MariaDB) 20 | #. postgresql (PostgreSQL or EnterpriseDB) 21 | #. File 22 | #. file (standard text file) 23 | #. log (log file) 24 | #. csv (Comma Separated Value file) 25 | #. json (JSON file) 26 | #. yaml (YAML file) 27 | #. xlsx (Microsoft Excel file) 28 | #. LDAP 29 | #. ldap (Active Directory Server, OpenLDAP, FreeIPA, etc.) 30 | #. NoSQL 31 | #. nosql (MongoDB, CouchDB, RavenDB, Redis, Neo4j, Cassandra, etc.) 32 | 33 | .. note:: 34 | The connection arguments of a ``DatabaseManager`` vary according to the type of database being accessed. 35 | Look at the manuals and documentation of each type of database to find out more. 36 | 37 | .. code-block:: python 38 | 39 | import pyreports 40 | 41 | # DatabaseManager object 42 | sqlite_db = pyreports.manager('sqlite', database='/tmp/mydb.db') 43 | mysql_db = pyreports.manager('mysql', host='mysql1.local', database='test', user='dba', password='dba0000') 44 | postgresql_db = pyreports.manager('postgresql', host='postgresql1.local', database='test', user='dba', password='dba0000') 45 | 46 | # FileManager object 47 | file = pyreports.manager('file', '/tmp/text.txt') 48 | log = pyreports.manager('log', '/tmp/log.log') 49 | csv = pyreports.manager('csv', '/tmp/csv.csv') 50 | json = pyreports.manager('json', '/tmp/json.json') 51 | yaml = pyreports.manager('yaml', '/tmp/yaml.yml') 52 | xlsx = pyreports.manager('xlsx', '/tmp/xlsx.xlsx') 53 | 54 | # LdapManager object 55 | ldap = pyreports.manager('ldap', server='ldap.local', username='user', password='password', ssl=False, tls=True) 56 | 57 | # NoSQLManager object (nosql api compliant https://nosqlapi.rtfd.io/) 58 | nosql = pyreports.manager('nosql', MongoDBConnection, host='mongo1.local', database='test', user='dba', password='dba0000') 59 | 60 | Managers at work 61 | **************** 62 | 63 | A Manager object corresponds to each type of manager. And each Manager object has its own methods for writing and reading data. 64 | 65 | DatabaseManager 66 | --------------- 67 | 68 | **Databasemanager** have eight methods that are used to reconnect, query, commit changes and much more. Let's see these methods in action below. 69 | 70 | .. note:: 71 | The following example will be done on a *mysql* type database, but it can be applied to any database because `DB-API 2.0 `_ is used. 72 | 73 | .. code-block:: python 74 | 75 | import pyreports 76 | 77 | # DatabaseManager object 78 | mysql_db = pyreports.manager('mysql', host='mysql1.local', database='test', user='dba', password='dba0000') 79 | 80 | # Reconnect to database 81 | mysql_db.reconnect() 82 | 83 | # Query: CREATE 84 | mysql_db.execute("CREATE TABLE cars(id SERIAL PRIMARY KEY, name VARCHAR(255), price INT)") 85 | # Query: INSERT 86 | mysql_db.execute("INSERT INTO cars(name, price) VALUES('Audi', 52642)") 87 | # Query: INSERT (many) 88 | new_cars = [ 89 | ('Alfa Romeo', 42123), 90 | ('Aston Martin', 78324), 91 | ('Ferrari', 129782), 92 | ] 93 | mysql_db.executemany("INSERT INTO cars(name, price) VALUES(%s, %s)", new_cars) 94 | # Commit changes 95 | mysql_db.commit() 96 | # Query: SELECT 97 | mysql_db.execute('SELECT * FROM cars') 98 | # View description and other info of last query 99 | print(mysql_db.description, mysql_db.lastrowid, mysql_db.rowcount) 100 | # Fetch all data 101 | print(mysql_db.fetchall()) # Dataset object 102 | # Fetch first row 103 | print(mysql_db.fetchone()) # Dataset object 104 | # Fetch select N row 105 | print(mysql_db.fetchmany(2)) # Dataset object 106 | print(mysql_db.fetchmany()) # This is same fetchone() method 107 | # Query: SHOW 108 | mysql_db.execute("SHOW TABLES") 109 | print(mysql_db.fetchall()) # Dataset object 110 | 111 | # Call store procedure 112 | mysql_db.callproc('select_cars') 113 | mysql_db.callproc('select_cars', ['Audi']) # Call with args 114 | print(mysql_db.fetchall()) # Dataset object 115 | 116 | .. note:: 117 | Whatever operation is done, the return value of the ``fetch*`` methods return `Dataset objects `_. 118 | 119 | FileManager 120 | ----------- 121 | 122 | **FileManager** has two simple methods: *read* and *write*. Let's see how to use this manager. 123 | 124 | .. code-block:: python 125 | 126 | import pyreports 127 | 128 | # FileManager object 129 | csv = pyreports.manager('csv', '/tmp/cars.csv') 130 | 131 | # Read data 132 | cars = csv.read() # Dataset object 133 | 134 | # Write data 135 | cars.append(['Audi', 52642]) 136 | csv.write(cars) 137 | 138 | LdapManager 139 | ----------- 140 | 141 | **LdapManager** is an object that allows you to interface and get data from a directory server via the ldap protocol. 142 | 143 | .. code-block:: python 144 | 145 | import pyreports 146 | 147 | # LdapManager object 148 | ldap = pyreports.manager('ldap', server='ldap.local', username='user', password='password', ssl=False, tls=True) 149 | 150 | # Rebind connection 151 | ldap.rebind() 152 | 153 | # Query: get data 154 | # This is Dataset object 155 | users = ldap.query('DC=test,DC=local', '(&(objectClass=user)(objectCategory=person))', ['name', 'mail', 'phone']) 156 | if users: 157 | print(users) 158 | 159 | # Close connection 160 | ldap.unbind() 161 | 162 | .. warning:: 163 | *LdapManager* should only be used for inputs. An ldap manager has no write methods. 164 | 165 | NoSQLManager 166 | ------------ 167 | 168 | **NoSQLManager** is an object that allows you to interface and get data from a NoSQL database server. 169 | 170 | .. code-block:: python 171 | 172 | import pyreports 173 | 174 | # LdapManager object 175 | nosql = pyreports.manager('nosql', MongoDBConnection, host='mongo1.local', database='test', user='dba', password='dba0000') 176 | 177 | # Get data 178 | nosql.get('doc1') # Dataset object 179 | 180 | # Find data 181 | nosql.find('{"name": "Matteo"}') # Dataset object 182 | 183 | .. note:: 184 | *NoSQLManager* object accept connection that must be compliant of `nosqlapi `_. -------------------------------------------------------------------------------- /docs/source/package.rst: -------------------------------------------------------------------------------- 1 | pyreports package 2 | ================= 3 | 4 | The package includes python modules for creating reports, from input to output to data processing. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Package: 9 | 10 | pyreports -------------------------------------------------------------------------------- /docs/source/pyreports.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | 3 | 4 | 5 | 6 | pyreports modules 7 | =================== 8 | 9 | io 10 | __ 11 | 12 | The *io* module contains all the classes and functions needed to interface with inputs and outputs. 13 | 14 | .. automodule:: pyreports.io 15 | :members: 16 | :special-members: 17 | :show-inheritance: 18 | 19 | core 20 | ____ 21 | 22 | The *core* module contains all the classes that refer to the creation and manipulation of data. 23 | 24 | .. automodule:: pyreports.core 25 | :members: 26 | :special-members: 27 | :show-inheritance: 28 | 29 | datatools 30 | _________ 31 | 32 | The *datatools* module contains all utility functions for data processing. 33 | 34 | .. automodule:: pyreports.datatools 35 | :members: 36 | :special-members: 37 | :show-inheritance: 38 | 39 | 40 | exception 41 | _________ 42 | 43 | The *exception* module contains all the classes that represent explicit package exceptions. 44 | 45 | .. automodule:: pyreports.exception 46 | :members: 47 | :special-members: 48 | :show-inheritance: 49 | -------------------------------------------------------------------------------- /docs/source/report.rst: -------------------------------------------------------------------------------- 1 | Reports 2 | ####### 3 | 4 | The package it is provided with a **Report** object and a **ReportBook** object. 5 | 6 | The *Report* object provides an interface for a complete workflow-based report (see :ref:`workflow`). 7 | 8 | The *ReportBook* object, on the other hand, is a list of *Report* objects. 9 | 10 | This will follow the workflows of each *Report* it contains, except for the output, which can be saved in a single Excel file. 11 | 12 | 13 | .. toctree:: 14 | 15 | 16 | 17 | 18 | Report at work 19 | ************** 20 | 21 | The *Report* object provides an interface to the entire workflow of a report: it accepts an input, processes the data and provides an output. 22 | To instantiate an object, you basically need three things: 23 | 24 | - **input**: a *Dataset* object, mandatory. 25 | - **filter**, **map function** or/and **column**: they are the same objects you would use in an *Executor* object, optional. 26 | - **output**: a *Manager* object, optional. 27 | 28 | .. code-block:: python 29 | 30 | import pyreports 31 | import tablib 32 | 33 | # Instantiate a simple Report object 34 | mydata = tablib.Dataset(*[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 35 | myrep = pyreports.Report(mydata) 36 | 37 | # View report 38 | myrep # repr(myrep) 39 | print(myrep) # str(myrep) 40 | 41 | Advanced Report instance 42 | ------------------------ 43 | 44 | The *Report* object is very complex. Instantiating it as above makes little sense, because the result will be identical to the input dataset. 45 | This object enables a series of features for data processing. 46 | 47 | .. code-block:: python 48 | 49 | import pyreports 50 | import tablib 51 | 52 | # Instantiate a Report object 53 | salary55k = pyreports.manager('csv', '/tmp/salary55k.csv') 54 | mydata = tablib.Dataset(*[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 55 | report_only_55k = pyreports.Report(mydata, filters=[55000], title='Report salary 55k', output=salary55k) 56 | 57 | # View report 58 | myrep # 59 | 60 | The example above, creates a *Report* object that filters input data only for employees with a salary of 55k. 61 | But we can also edit the data on-demand and then filter it, as follows in the next example. 62 | 63 | .. note:: 64 | You can also pass a function to the ``filters`` argument, as for an *Executor* object. 65 | 66 | .. code-block:: python 67 | 68 | import pyreports 69 | import tablib 70 | 71 | # My custom function for modifying salary data 72 | def stringify_salary(salary): 73 | if isinstance(salary, int): 74 | return f'$ {salary}' 75 | else: 76 | return salary 77 | 78 | # Instantiate a Report object 79 | salary55k = pyreports.manager('sqlite', '/tmp/mydb.db') # DatabaseManager 80 | mydata = tablib.Dataset(*[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 81 | report_only_55k = pyreports.Report(mydata, 82 | filters=['$ 55000'], 83 | map_func=stringify_salary, 84 | title='Report salary 55k', 85 | output=salary55k) 86 | 87 | # View report 88 | myrep # 89 | 90 | .. note:: 91 | It is also possible to declare a counter of the processed lines by setting ``count=True``. 92 | Moreover, as for an Executor object, you can specify a single return ``column`` using the column argument; ex. ``column='surname'``. 93 | 94 | Execute Report 95 | -------------- 96 | 97 | Once a *Report* object has been instantiated, you can execute the filters and editing functions (map) set during the creation of the object. 98 | 99 | .. code-block:: python 100 | 101 | # Apply filters and map function 102 | report_only_55k.exec() 103 | 104 | # Print result 105 | print(report_only_55k) 106 | 107 | # Adding count after creation 108 | report_only_55k.count = True 109 | report_only_55k.exec() 110 | print(report_only_55k) 111 | 112 | .. warning:: 113 | Once a filter or map function is applied, it will not be possible to go back. 114 | If you want to change filters after call the ``exec`` method, you need to re-instantiate the object. 115 | 116 | Export 117 | ------ 118 | 119 | Once the ``exec`` method is called, and then once the data is processed, we can export the data based on the output set when instantiating the object. 120 | 121 | .. note:: 122 | If the output has not been specified, calling the export method will print the data to stdout. 123 | 124 | .. code-block:: python 125 | 126 | # Save report on /tmp/salary55k.csv 127 | report_only_55k.export() 128 | 129 | # Unset output 130 | report_only_55k.output = None 131 | report_only_55k.export() # This print the data on stdout 132 | 133 | # Set output 134 | report_only_55k.output = salary55k 135 | report_only_55k.export() # Save report on /tmp/salary55k.csv 136 | 137 | 138 | ReportBook at work 139 | ****************** 140 | 141 | The *ReportBook* object is a collection (list) of *Report* objects. 142 | This basically allows you to collect multiple reports in a single container object. 143 | The main advantage is the ability to iterate over each *Report* and access its properties. 144 | 145 | .. code-block:: python 146 | 147 | import pyreports 148 | import tablib 149 | 150 | # Instantiate the Report objects 151 | mydata = tablib.Dataset(*[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) 152 | report_only_55k = pyreports.Report(mydata, filters=[55000], title='Report salary 55k') 153 | report_only_65k = pyreports.Report(mydata, filters=[65000], title='Report salary 65k') 154 | 155 | # Create a ReportBook 156 | salary = pyreports.ReportBook([report_only_55k, report_only_65k]) 157 | 158 | # View ReportBook 159 | salary # repr(salary) 160 | print(salary) # str(salary) 161 | 162 | .. note:: 163 | The ReportBook object supports the ``title`` property, as follows: ``pyreports.ReportBook(title='My report book')`` 164 | 165 | Export reports 166 | -------------- 167 | 168 | The *ReportBook* object has an ``export`` method. 169 | This method not only saves *Report* objects to its output, but first executes the ``exec`` method of each *Report* object it contains. 170 | 171 | .. warning:: 172 | As for *Report* objects, even a *ReportBook* object once the export method has been called, 173 | it will need to be instantiated again if you want to reset the data to the source, before applying the filters and map functions. 174 | 175 | .. code-block:: python 176 | 177 | # Export a ReportBook 178 | salary.export() # This run exec() and export() on each Report object 179 | 180 | # Export each Report on one file Excel (xlsx) 181 | salary.export('/tmp/salary_report.xlsx') 182 | 183 | Add and remove report 184 | --------------------- 185 | 186 | Being a container, the *ReportBook* object can be used to add and remove *Report* object. 187 | 188 | .. code-block:: python 189 | 190 | # Create an empty ReportBook 191 | salary = pyreports.ReportBook(title='Salary report') 192 | 193 | # Add a Report object 194 | salary.add(report_only_55k) 195 | salary.add(report_only_65k) 196 | 197 | # Remove last Report object added 198 | salary.remove() # Remove report_only_65k object 199 | salary.remove(0) # Remove report_only_55k object, via index 200 | 201 | Count reports 202 | ------------- 203 | 204 | The *ReportBook* object supports the protocol for the built-in ``len`` function, to count the *Report* objects it contains. 205 | 206 | .. code-block:: python 207 | 208 | # Count object 209 | len(salary) 210 | 211 | Iteration 212 | --------- 213 | 214 | The *ReportBook* object supports the python iteration protocol (return of generator object). 215 | This means that you can use it in a for loop or in a list comprehension. 216 | 217 | .. code-block:: python 218 | 219 | # For each report in ReportBook 220 | for report in salary: 221 | print(report) 222 | 223 | # List comprehension 224 | my_list_of_report = [report for report in salary] 225 | 226 | Merge 227 | ----- 228 | 229 | ReportBook objects can be joined together, using the `+` operator. 230 | 231 | .. code-block:: python 232 | 233 | # ReportBook 234 | book1 = pyreports.ReportBook([report1, report2]) 235 | book2 = pyreports.ReportBook([report3, report4]) 236 | # Merge ReportBook 237 | tot_book = book1 + book2 238 | tot_book = book1.__add__(book2) 239 | 240 | print(tot_book) 241 | 242 | # ReportBook None 243 | # Report1 244 | # Report2 245 | # Report3 246 | # Report4 -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MatteoGuadrini/pyreports/727825204dd7c63dbcccde00743e6d8199eec438/favicon.ico -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | pyreports 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 | 39 | 40 | 41 | 70 | 71 | 72 |
73 |
74 | pyreports 75 |

pyreports

76 |

pyreports is a python library that allows you to create complex report from various sources

77 |
78 | Usage 79 | Docs 80 |
81 | 85 |
86 | 97 |
98 | 99 | 100 |
101 |
102 |

What is pyreports?

103 |

104 | pyreports wants to be a library that simplifies the collection of data from multiple sources such as databases, 105 | files and directory servers (through LDAP), the processing of them through built-in and customized functions, 106 | and the saving in various formats (or, by inserting the data in a database). 107 |

108 |
109 |
110 |
111 | 112 |
113 |
114 |

Designed for a data scientist

115 |

116 | Simplifying the life of those who collect and process data 117 |

118 |
119 |
120 |
121 |
122 | 123 |
124 |
125 |

Time saver

126 |

It saves time, without worrying about creating connections or managing files manually.

127 |
128 |
129 |
130 |
131 | 132 |
133 |
134 |

PEP 8

135 |

Each row of code is PEP 8 compliant

136 |
137 |
138 |
139 |
140 |
141 |
142 | 143 | 144 |
145 |
146 |

Features

147 |
    148 |
  • Write for Python 3.6 and high
    • 149 |
  • Each database connection is DBAPI 2.0 compliant
    • 150 |
  • Each NoSQL database connection is nosqlapi compliant
    • 151 |
  • Work with Dataset objects
    • 152 |
  • All objects are extensible
    • 153 |
  • Functions that support data modification
    • 154 |
  • Data exports to Excel, CSV, YAML, Json and more.
    • 155 |
  • Send exported data to email
    • 156 |
157 |
158 |
159 | 160 | 161 |
162 |
163 |
164 |

Get Started

165 |
166 |

Installation

167 |

Install pyreports is easy with pip.

168 |
169 | 170 | 

171 |     $ pip install pyreports
172 |
173 |
174 |
175 | 176 |
177 |

Basic usage

178 |

I take the data from a database table, filter the data I need and save it in a csv file.

179 | 180 |
181 | 182 | 
183 |     
184 | import pyreports
185 | 
186 | # Select source: this is a DatabaseManager object
187 | mydb = pyreports.manager('mysql', host='mysql1.local', database='login_users', user='dba', password='dba0000')
188 | 
189 | # Get data
190 | mydb.execute('SELECT * FROM site_login')
191 | site_login = mydb.fetchall()
192 | 
193 | # Filter data
194 | error_login = pyreports.Executor(site_login)
195 | error_login.filter([400, 401, 403, 404, 500])
196 | 
197 | # Save report: this is a FileManager object
198 | output = pyreports.manager('csv', '/home/report/error_login.csv')
199 | output.write(error_login.get_data())
200 |
201 |
202 |
203 | 204 |
205 |

Advanced usage

206 |

In this example we will analyze and capture parts of a web server log. 207 | For each error code present in the log, we will create a report that will be inserted in a book, 208 | where each sheet will contain the details of the error code. In the last sheet, 209 | there will be an element counter for every single error present in the report.

210 |
211 | 

212 | import pyreports
213 | import tablib
214 | 
215 | # Get apache log data: this is a FileManager object
216 | apache_log = pyreports.manager('log', '/var/log/httpd/error.log')
217 | # Read log based on regexp
218 | data_log = apache_log.read('([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) - "(.*?)" "(.*?)"',
219 |                         headers=['ip', 'date', 'operation', 'url', 'code', 'client'])
220 | 
221 | # Create a collection of Report objects
222 | all_apache_error = pyreports.ReportBook(title='Apache error on my site')
223 | 
224 | # Create a Report object based on error code
225 | all_error = set(data_log['code'])
226 | for code in all_error:
227 |     all_apache_error.add(pyreports.Report(data_log, filters=[code], title=f'Error {code}'))
228 | 
229 | # Count all error code
230 | counter = pyreports.counter(data_log, 'code')
231 | # Append new Report on ReportBook with error code counters
232 | error_counter = tablib.Dataset(counter.values(), headers=counter)
233 | all_apache_error.add(pyreports.Report(error_counter))
234 | 
235 | # Save ReportBook on Excel
236 | all_apache_error.export('/home/report/apache_log_error_code.xlsx')
237 |
238 |
239 |
240 | 241 |
242 |

Shell CLI

243 |

pyreports has a command line interface which takes a configuration file in YAML format as an argument.

244 |
245 | 

246 | $ cat car.yml
247 | reports:
248 |   - report:
249 |     title: 'Red ford machine'
250 |     input:
251 |       manager: 'mysql'
252 |       source:
253 |       # Connection parameters of my mysql database
254 |         host: 'mysql1.local'
255 |         database: 'cars'
256 |         user: 'admin'
257 |         password: 'dba0000'
258 |       params:
259 |         query: 'SELECT * FROM cars WHERE brand = %s AND color = %s'
260 |         params: ['ford', 'red']
261 |     # Filter km
262 |     filters: [40000, 45000]
263 |     output:
264 |       manager: 'csv'
265 |       filename: '/tmp/car_csv.csv'
266 | 
267 | $ report car.yaml
268 |
269 |
270 |
271 | 272 |
273 |

Full Documentation

274 |

The complete documentation is much more comprehensive and can be found by clicking the button below.

275 |

276 | More on ReadTheDocs 277 |

278 |
279 | 280 |
281 |
282 |
283 | 284 | 285 |
286 |
287 |
288 |

License

289 |
290 |

pyreports is free and open source.

291 |

The license is GPLv3 and you can consult it directly here: Github License

292 |

Donate: PayPal

293 |
294 |
295 |
296 |

If you are interested in improvements or have innovation proposals, or simply want to help improve the library, you can fork the project from here! Thank you for your support!

297 |
298 |
299 |
300 | Fork 301 |
302 |
303 |
304 |
305 |
306 | 307 | 308 |
309 |
310 |
311 |

Contact

312 |

I hope you find this python library useful.
Feel free to get in touch if you have any questions or suggestions.

313 |
314 |
315 | Matteo Guadrini aka GU 316 |
317 |
318 |

Love pyreports?

319 |

If you are interested in other projects, follow me on Github or Linkedin. You can also email me if you like!

320 |
321 | Matteo Guadrini 322 |
323 | DevOps and Rust/Python developer 324 |
325 |
326 |
327 |
328 |
329 |

Get Connected

330 | 335 |
336 |
337 |
338 |
339 | 340 | 341 |
342 |
343 | 344 | Designed with and 345 |
346 |
347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "setuptools_scm[toml]", "wheel", "cython"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "pyreports" 7 | version = "1.8.0" 8 | readme = "README.md" 9 | license = { text = "GNU General Public License v3.0" } 10 | keywords = ['pyreports', 'reports', 'report', 'csv', 'yaml', 'export', 11 | 'excel', 'database', 'ldap', 'dataset', 'file', 'executor', 'book'] 12 | authors = [{ name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }] 13 | maintainers = [ 14 | { name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }, 15 | ] 16 | description = "pyreports is a python library that allows you to create complex report from various sources." 17 | requires-python = ">=3.8" 18 | classifiers = [ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 21 | "Operating System :: OS Independent", 22 | ] 23 | dependencies = ['ldap3', 'mysql-connector-python', 24 | 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] 25 | 26 | [project.scripts] 27 | reports = "pyreports.cli:main" 28 | 29 | [project.urls] 30 | homepage = "https://github.com/MatteoGuadrini/pyreports" 31 | documentation = "https://pyreports.readthedocs.io/en/latest/" 32 | repository = "https://github.com/MatteoGuadrini/pyreports.git" 33 | changelog = "https://github.com/MatteoGuadrini/pyreports/blob/master/CHANGES.md" 34 | -------------------------------------------------------------------------------- /pyreports/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # vim: se ts=4 et syn=python: 4 | 5 | # created by: matteo.guadrini 6 | # __init__ -- pyreports 7 | # 8 | # Copyright (C) 2025 Matteo Guadrini 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Build complex reports from/to various formats.""" 24 | 25 | __version__ = "1.8.0" 26 | 27 | from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 28 | from .core import Executor, Report, ReportBook # noqa: F401 29 | from .exception import ReportDataError, ReportManagerError, DataObjectError # noqa: F401 30 | from .datatools import ( 31 | average, # noqa: F401 32 | most_common, # noqa: F401 33 | percentage, # noqa: F401 34 | counter, # noqa: F401 35 | aggregate, # noqa: F401 36 | chunks, # noqa: F401 37 | merge, # noqa: F401 38 | deduplicate, # noqa: F401 39 | subset, # noqa: F401 40 | sort, # noqa: F401 41 | DataObject, # noqa: F401 42 | DataAdapters, # noqa: F401 43 | DataPrinters, # noqa: F401 44 | ) 45 | from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 46 | -------------------------------------------------------------------------------- /pyreports/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # vim: se ts=4 et syn=python: 4 | 5 | # created by: matteo.guadrini 6 | # cli -- pyreports 7 | # 8 | # Copyright (C) 2025 Matteo Guadrini 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Command line interface""" 24 | 25 | # region imports 26 | import sys 27 | import yaml 28 | import argparse 29 | import pyreports 30 | from pyreports import __version__ 31 | 32 | 33 | # endregion 34 | 35 | 36 | # region functions 37 | def get_args(): 38 | """Get command-line arguments""" 39 | 40 | parser = argparse.ArgumentParser( 41 | description="pyreports command line interface (CLI)", 42 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 43 | epilog="Full docs here: https://pyreports.readthedocs.io/en/latest/dev/cli.html", 44 | ) 45 | 46 | parser.add_argument( 47 | "config", 48 | metavar="CONFIG_FILE", 49 | default=sys.stdin, 50 | type=argparse.FileType("rt", encoding="utf-8"), 51 | help="YAML configuration file", 52 | ) 53 | parser.add_argument( 54 | "-e", 55 | "--exclude", 56 | help="Excluded title report list", 57 | nargs=argparse.ZERO_OR_MORE, 58 | default=[], 59 | metavar="TITLE", 60 | ) 61 | parser.add_argument( 62 | "-v", "--verbose", help="Enable verbose mode", action="store_true" 63 | ) 64 | parser.add_argument( 65 | "-V", "--version", help="Print version", action="version", version=__version__ 66 | ) 67 | 68 | args = parser.parse_args() 69 | filename = args.config.name 70 | 71 | # Check if file is a YAML file 72 | try: 73 | args.config = load_config(args.config) 74 | except yaml.YAMLError as err: 75 | parser.error(f"file {filename} is not a valid YAML file: \n{err}") 76 | 77 | # Validate config file 78 | try: 79 | validate_config(args.config) 80 | except yaml.YAMLError as err: 81 | parser.error(str(err)) 82 | 83 | print_verbose(f"parsed YAML file {filename}", verbose=args.verbose) 84 | 85 | return args 86 | 87 | 88 | def load_config(yaml_file): 89 | """Load configuration file 90 | 91 | :param yaml_file: opened yaml file 92 | :return: Any 93 | """ 94 | with yaml_file as file: 95 | return yaml.safe_load(file) 96 | 97 | 98 | def validate_config(config): 99 | """Validate config object 100 | 101 | :param config: YAML config object 102 | :return: None 103 | """ 104 | try: 105 | reports = config["reports"] 106 | if reports is None or not isinstance(reports, list): 107 | raise yaml.YAMLError('"reports" section have not "report" list sections') 108 | datas = all([bool(report.get("report").get("input")) for report in reports]) 109 | if not datas: 110 | raise yaml.YAMLError( 111 | 'one of "report" section does not have "input" section' 112 | ) 113 | except KeyError as err: 114 | raise yaml.YAMLError(f'there is no "{err}" section') 115 | except AttributeError: 116 | raise yaml.YAMLError( 117 | 'correctly indents the "report" section or "report" section not exists' 118 | ) 119 | 120 | 121 | def make_manager(input_config): 122 | """Make Manager object 123 | 124 | :param input_config: file, sql or nosql report configuration 125 | :return: manager 126 | """ 127 | 128 | type_ = input_config.get("manager") 129 | 130 | if type_ in pyreports.io.FILETYPE: 131 | manager = pyreports.manager( 132 | input_config.get("manager"), input_config.get("filename", ()) 133 | ) 134 | else: 135 | manager = pyreports.manager( 136 | input_config.get("manager"), **input_config.get("source", {}) 137 | ) 138 | 139 | return manager 140 | 141 | 142 | def get_data(manager, params=None): 143 | """Get Dataset from source 144 | 145 | :param manager: Manager object 146 | :param params: parameter used into call of method in Manager object 147 | :return: Dataset 148 | """ 149 | if params is None: 150 | params = () 151 | data = None 152 | # FileManager 153 | if manager.type == "file": 154 | if params and isinstance(params, (list, tuple)): 155 | data = manager.read(*params) 156 | elif params and isinstance(params, dict): 157 | data = manager.read(**params) 158 | else: 159 | data = manager.read() 160 | # DatabaseManager 161 | elif manager.type == "sql": 162 | if params and isinstance(params, (list, tuple)): 163 | manager.execute(*params) 164 | data = manager.fetchall() 165 | elif params and isinstance(params, dict): 166 | manager.execute(**params) 167 | data = manager.fetchall() 168 | # LdapManager 169 | elif manager.type == "ldap": 170 | if params and isinstance(params, (list, tuple)): 171 | data = manager.query(*params) 172 | elif params and isinstance(params, dict): 173 | data = manager.query(**params) 174 | # NosqlManager 175 | elif manager.type == "nosql": 176 | if params and isinstance(params, (list, tuple)): 177 | data = manager.find(*params) 178 | elif params and isinstance(params, dict): 179 | data = manager.find(**params) 180 | 181 | return data 182 | 183 | 184 | def print_verbose(*messages, verbose=False): 185 | """Print messages if verbose is True 186 | 187 | :param messages: some string messages 188 | :param verbose: enable or disable verbosity 189 | :return: None 190 | """ 191 | if verbose: 192 | print("debug:", *messages) 193 | 194 | 195 | def main(): 196 | """Main logic""" 197 | 198 | # Get command line args 199 | args = get_args() 200 | # Take reports 201 | config = args.config 202 | reports = config.get("reports", ()) 203 | 204 | print_verbose( 205 | f"found {len(config.get('reports', ()))} report(s)", verbose=args.verbose 206 | ) 207 | 208 | # Build the data and report 209 | for report in reports: 210 | # Check if report isn't in excluded list 211 | if args.exclude and report.get("report").get("title") in args.exclude: 212 | print_verbose( 213 | f'exclude report "{report.get("report").get("title")}"', 214 | verbose=args.verbose, 215 | ) 216 | continue 217 | # Make a manager object 218 | input_ = report.get("report").get("input") 219 | print_verbose( 220 | f"make an input manager of type {input_.get('manager')}", 221 | verbose=args.verbose, 222 | ) 223 | manager = make_manager(input_) 224 | # Get data 225 | print_verbose(f"get data from manager {manager}", verbose=args.verbose) 226 | try: 227 | # Make a report object 228 | data = get_data(manager, input_.get("params")) 229 | # Check if sort is specified 230 | if report.get("report").get("sort"): 231 | column = report.get("report").get("sort").get("column") 232 | reverse = report.get("report").get("sort").get("reverse") 233 | data = pyreports.sort(data, column, reverse=reverse) 234 | # Check if deduplicate is specified 235 | if report.get("report").get("deduplicate"): 236 | data = pyreports.deduplicate(data) 237 | # Check if subset is specified 238 | if report.get("report").get("subset"): 239 | data = pyreports.subset( 240 | data, *report.get("report").get("subset").get("columns") 241 | ) 242 | if "map" in report.get("report"): 243 | exec(report.get("report").get("map")) 244 | map_func = globals().get("map_func") 245 | report_ = pyreports.Report( 246 | input_data=data, 247 | title=report.get("report").get("title"), 248 | filters=report.get("report").get("filters"), 249 | map_func=map_func, 250 | negation=report.get("report").get("negation", False), 251 | column=report.get("report").get("column"), 252 | count=report.get("report").get("count", False), 253 | output=make_manager(report.get("report").get("output")) 254 | if "output" in report.get("report") 255 | else None, 256 | ) 257 | print_verbose(f'created report "{report_.title}"', verbose=args.verbose) 258 | except Exception as err: 259 | exit(f"error: {err}") 260 | # Check output 261 | if report_.output: 262 | # Check if export or send report 263 | if report.get("report").get("mail"): 264 | print_verbose( 265 | f"send report to {report.get('report').get('mail').get('to')}", 266 | verbose=args.verbose, 267 | ) 268 | mail_settings = report.get("report").get("mail") 269 | report_.send( 270 | server=mail_settings.get("server"), 271 | _from=mail_settings.get("from"), 272 | to=mail_settings.get("to"), 273 | cc=mail_settings.get("cc"), 274 | bcc=mail_settings.get("bcc"), 275 | subject=mail_settings.get("subject"), 276 | body=mail_settings.get("body"), 277 | auth=tuple(mail_settings.get("auth")) 278 | if "auth" in mail_settings 279 | else None, 280 | _ssl=bool(mail_settings.get("ssl")), 281 | headers=mail_settings.get("headers"), 282 | ) 283 | else: 284 | print_verbose( 285 | f"export report to {report_.output}", verbose=args.verbose 286 | ) 287 | report_.export() 288 | else: 289 | # Print report in stdout 290 | print_verbose("print report to stdout", verbose=args.verbose) 291 | title = report.get("report").get("title") 292 | report_.exec() 293 | print(f"{title}\n{'=' * len(title)}\n") 294 | print(report_) 295 | 296 | 297 | # endregion 298 | 299 | # region main 300 | if __name__ == "__main__": 301 | main() 302 | 303 | # endregion 304 | -------------------------------------------------------------------------------- /pyreports/datatools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # vim: se ts=4 et syn=python: 4 | 5 | # created by: matteo.guadrini 6 | # datatools -- pyreports 7 | # 8 | # Copyright (C) 2025 Matteo Guadrini 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Contains all functions for data processing.""" 24 | 25 | # region Imports 26 | from .exception import DataObjectError 27 | from collections import Counter 28 | from tablib import Dataset, InvalidDimensions 29 | 30 | 31 | # endregion 32 | 33 | 34 | # region Classes 35 | class DataObject: 36 | """Data object class""" 37 | 38 | def __init__(self, input_data: Dataset): 39 | # Discard all objects that are not Datasets 40 | if isinstance(input_data, Dataset): 41 | self._data = input_data 42 | else: 43 | raise DataObjectError("only Dataset object is allowed for input") 44 | 45 | @property 46 | def data(self): 47 | return self._data 48 | 49 | @data.setter 50 | def data(self, dataset: Dataset): 51 | if not isinstance(dataset, Dataset): 52 | raise DataObjectError(f"{dataset} is not a Dataset object") 53 | self._data = dataset 54 | 55 | def clone(self): 56 | """Clone itself 57 | 58 | :return: Dataset 59 | """ 60 | return DataObject(self.data) 61 | 62 | def column(self, index): 63 | _select_column(self.data, column=index) 64 | 65 | 66 | class DataAdapters(DataObject): 67 | """Data adapters class""" 68 | 69 | def aggregate(self, *columns, fill_value=None): 70 | """Aggregate in the current Dataset other columns 71 | 72 | :param columns: columns added 73 | :param fill_value: fill value for empty field 74 | :return: None 75 | """ 76 | if not self.data: 77 | raise DataObjectError("dataset is empty") 78 | local_columns = [self.data.get_col(col) for col in range(self.data.width)] 79 | local_columns.extend(columns) 80 | self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) 81 | 82 | def merge(self, *datasets): 83 | """Merge in the current Dataset other Dataset objects 84 | 85 | :param datasets: datasets that will merge 86 | :return: None 87 | """ 88 | datasets = list(datasets) 89 | datasets.append(self.data) 90 | # Check if all Datasets are not empties 91 | if not all([data for data in datasets]): 92 | raise DataObjectError("one or more Datasets are empties") 93 | self.data = merge(*datasets) 94 | 95 | def counter(self): 96 | """Count value into the rows 97 | 98 | :return: Counter 99 | """ 100 | return Counter((item for row in self.data for item in row)) 101 | 102 | def chunks(self, length): 103 | """ 104 | Yield successive n-sized chunks from Dataset 105 | 106 | :param length: n-sized chunks 107 | :return: generator 108 | """ 109 | for idx in range(0, len(self.data), length): 110 | yield self.data[idx : idx + length] 111 | 112 | def deduplicate(self): 113 | """Remove duplicated rows 114 | 115 | :return: None 116 | """ 117 | self.data.remove_duplicates() 118 | 119 | def subset(self, *columns): 120 | """New dataset with only columns added 121 | 122 | :param columns: select columns of new Dataset 123 | :return: Dataset 124 | """ 125 | return self.data.subset(cols=columns) 126 | 127 | def sort(self, column, reverse=False): 128 | """Sort a Dataset by a specific column 129 | 130 | :param column: column to sort 131 | :param reverse: reversed order 132 | :return: Dataset 133 | """ 134 | return self.data.sort(col=column, reverse=reverse) 135 | 136 | def __iter__(self): 137 | return (row for row in self.data) 138 | 139 | def __getitem__(self, item): 140 | return self.data[item] 141 | 142 | 143 | class DataPrinters(DataObject): 144 | """Data printers class""" 145 | 146 | def print(self): 147 | """Print data 148 | 149 | :return: None 150 | """ 151 | print(self) 152 | 153 | def average(self, column): 154 | """Average of list of integers or floats 155 | 156 | :param column: column name or index 157 | :return: float 158 | """ 159 | return average(self.data, column) 160 | 161 | def most_common(self, column): 162 | """The most common element in a column 163 | 164 | :param column: column name or index 165 | :return: Any 166 | """ 167 | return most_common(self.data, column) 168 | 169 | def percentage(self, filter_): 170 | """Calculating the percentage according to filter 171 | 172 | :param filter_: equality filter 173 | :return: float 174 | """ 175 | return percentage(self.data, filter_) 176 | 177 | def __repr__(self): 178 | """Representation of DataObject 179 | 180 | :return: string 181 | """ 182 | return f"" 183 | 184 | def __str__(self): 185 | """Pretty representation of DataObject 186 | 187 | :return: string 188 | """ 189 | return str(self.data) 190 | 191 | def __len__(self): 192 | """Measure length of DataSet 193 | 194 | :return: int 195 | """ 196 | return len(self.data) 197 | 198 | 199 | # endregion 200 | 201 | 202 | # region Functions 203 | def _select_column(data: Dataset, column): 204 | """Select Dataset column 205 | 206 | :param data: Dataset object 207 | :param column: column name or index 208 | :return: list 209 | """ 210 | if isinstance(column, int): 211 | return data.get_col(column) 212 | else: 213 | return data[column] 214 | 215 | 216 | def average(data: Dataset, column): 217 | """ 218 | Average of list of integers or floats 219 | 220 | :param data: Dataset object 221 | :param column: column name or index 222 | :return: float 223 | """ 224 | # Select column 225 | data = _select_column(data, column) 226 | # Check if all item is integer or float 227 | if not all(isinstance(item, (int, float)) for item in data): 228 | raise DataObjectError("the column contains only int or float") 229 | # Calculate average 230 | return float(sum(data) / len(data)) 231 | 232 | 233 | def most_common(data: Dataset, column): 234 | """ 235 | The most common element in a column 236 | 237 | :param data: Dataset object 238 | :param column: column name or index 239 | :return: Any 240 | """ 241 | # Select column 242 | data = _select_column(data, column) 243 | return max(data, key=data.count) 244 | 245 | 246 | def percentage(data: Dataset, filter_): 247 | """ 248 | Calculating the percentage according to filter 249 | 250 | :param data: Dataset object 251 | :param filter_: equality filter 252 | :return: float 253 | """ 254 | # Filtering data... 255 | data_filtered = [item for row in data for item in row if filter_ == item] 256 | quotient = len(data_filtered) / len(data) 257 | return quotient * 100 258 | 259 | 260 | def counter(data: Dataset, column): 261 | """ 262 | Count all row value 263 | 264 | :param data: Dataset object 265 | :param column: column name or index 266 | :return: Counter 267 | """ 268 | # Select column 269 | data = _select_column(data, column) 270 | # Return Counter object 271 | return Counter((item for item in data)) 272 | 273 | 274 | def aggregate(*columns, fill_empty: bool = False, fill_value=None): 275 | """ 276 | Aggregate in a new Dataset the columns 277 | 278 | :param columns: columns added 279 | :param fill_empty: fill the empty field of data with "fill_value" argument 280 | :param fill_value: fills value for empty field if "fill_empty" argument is specified 281 | :return: Dataset 282 | """ 283 | if len(columns) >= 2: 284 | new_data = Dataset() 285 | # Check max len of all columns 286 | max_len = max([len(column) for column in columns]) 287 | for list_ in columns: 288 | if fill_empty: 289 | while max_len != len(list_): 290 | list_.append(fill_value() if callable(fill_value) else fill_value) 291 | else: 292 | if max_len != len(list_): 293 | raise InvalidDimensions("the columns are not the same length") 294 | max_len = len(list_) 295 | # Aggregate columns 296 | for column in columns: 297 | new_data.append_col(column) 298 | return new_data 299 | else: 300 | raise DataObjectError("you can aggregate two or more columns") 301 | 302 | 303 | def merge(*datasets): 304 | """ 305 | Merge two or more dataset in only one 306 | 307 | :param datasets: Dataset object collection 308 | :return: Dataset 309 | """ 310 | if len(datasets) >= 2: 311 | new_data = Dataset() 312 | # Check len of row 313 | length_row = max([len(dataset[0]) for dataset in datasets]) 314 | for data in datasets: 315 | if length_row != len(data[0]): 316 | raise InvalidDimensions("the row are not the same length") 317 | new_data.extend(data) 318 | return new_data 319 | else: 320 | raise DataObjectError("you can merge two or more dataset object") 321 | 322 | 323 | def chunks(data: Dataset, length): 324 | """ 325 | Yield successive n-sized chunks from data 326 | 327 | :param data: Dataset object 328 | :param length: n-sized chunks 329 | :return: generator 330 | """ 331 | for idx in range(0, len(data), length): 332 | yield data[idx : idx + length] 333 | 334 | 335 | def deduplicate(data: Dataset): 336 | """Remove duplicated rows 337 | 338 | :param data: Dataset object 339 | :return: Dataset 340 | """ 341 | data.remove_duplicates() 342 | return data 343 | 344 | 345 | def subset(data: Dataset, *columns): 346 | """Create a new Dataset with only the given columns 347 | 348 | :param data: Dataset object 349 | :param columns: selected columns 350 | :return: Dataset 351 | """ 352 | return data.subset(cols=columns) 353 | 354 | 355 | def sort(data, column, reverse=False): 356 | """Sort a Dataset by a specific column 357 | 358 | :param data: Dataset object 359 | :param column: column to sort 360 | :param reverse: reversed order 361 | :return: Dataset 362 | """ 363 | return data.sort(col=column, reverse=reverse) 364 | 365 | 366 | # endregion 367 | -------------------------------------------------------------------------------- /pyreports/exception.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # vim: se ts=4 et syn=python: 4 | 5 | # created by: matteo.guadrini 6 | # exception.py -- pyreports 7 | # 8 | # Copyright (C) 2024 Matteo Guadrini 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | 23 | """Contains all custom exception.""" 24 | 25 | 26 | # ExecutorException hierarchy 27 | class ExecutorError(Exception): 28 | pass 29 | 30 | class ExecutorDataError(ExecutorError): 31 | pass 32 | 33 | # ReportException hierarchy 34 | class DataObjectError(Exception): 35 | pass 36 | 37 | 38 | class ReportException(DataObjectError): 39 | pass 40 | 41 | 42 | class ReportDataError(ReportException): 43 | pass 44 | 45 | 46 | class ReportManagerError(ReportException): 47 | pass 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | __version__ = "1.8.0" 5 | __author__ = "Matteo Guadrini" 6 | __email__ = "matteo.guadrini@hotmail.it" 7 | __homepage__ = "https://github.com/MatteoGuadrini/pyreports" 8 | 9 | with open("README.md") as rme, open("CHANGES.md") as ch: 10 | long_description = rme.read() + "\n" + ch.read() 11 | 12 | setup( 13 | name="pyreports", 14 | version=__version__, 15 | packages=["pyreports"], 16 | url=__homepage__, 17 | license="GNU General Public License v3.0", 18 | author=__author__, 19 | author_email=__email__, 20 | keywords="pyreports reports report csv yaml export excel database ldap dataset file executor book", 21 | maintainer="Matteo Guadrini", 22 | maintainer_email="matteo.guadrini@hotmail.it", 23 | install_requires=[ 24 | "ldap3", 25 | "mysql-connector-python", 26 | "psycopg2-binary", 27 | "tablib", 28 | "tablib[all]", 29 | "nosqlapi", 30 | "pyyaml", 31 | ], 32 | description="pyreports is a python library that allows you to create complex report from various sources", 33 | long_description=long_description, 34 | long_description_content_type="text/markdown", 35 | classifiers=[ 36 | "Programming Language :: Python :: 3", 37 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 38 | "Operating System :: OS Independent", 39 | ], 40 | entry_points={"console_scripts": ["reports = pyreports.cli:main"]}, 41 | python_requires=">=3.8", 42 | ) 43 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # vim: se ts=4 et syn=python: 4 | 5 | # created by: matteo.guadrini 6 | # __init__ -- pyreports 7 | # 8 | # Copyright (C) 2022 Matteo Guadrini 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see . 22 | -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | import pyreports 5 | from tablib import Dataset 6 | from tempfile import gettempdir 7 | 8 | tmp_folder = gettempdir() 9 | 10 | 11 | class TestExecutor(unittest.TestCase): 12 | data = pyreports.Executor(Dataset(["Matteo", "Guadrini", 35])) 13 | 14 | def test_executor_instance(self): 15 | self.assertIsInstance(self.data, pyreports.Executor) 16 | 17 | def test_get_data(self): 18 | self.assertIsInstance(self.data.get_data(), Dataset) 19 | self.assertEqual(str(self.data.get_data()), "Matteo|Guadrini|35") 20 | 21 | def test_set_headers(self): 22 | self.data.headers = ["name", "surname", "age"] 23 | self.assertEqual(self.data.data.headers, ["name", "surname", "age"]) 24 | 25 | def test_filter_by_list(self): 26 | self.data.data.append(["Arthur", "Dent", 42]) 27 | self.data.data.append(["Ford", "Prefect", 42]) 28 | self.data.filter([42]) 29 | self.assertEqual(self.data.get_data()[0], ("Arthur", "Dent", 42)) 30 | self.data.reset() 31 | 32 | def test_filter_by_list_negation(self): 33 | data = pyreports.Executor(Dataset()) 34 | data.data.append(["Arthur", "Dent", 42]) 35 | data.data.append(["Ford", "Prefect", 42]) 36 | data.filter(["Prefect"], negation=True) 37 | self.assertEqual(data.get_data()[0], ("Arthur", "Dent", 42)) 38 | 39 | def test_filter_by_key(self): 40 | def is_answer(number): 41 | if number == 42: 42 | return True 43 | 44 | self.data.data.append(["Arthur", "Dent", 42]) 45 | self.data.data.append(["Ford", "Prefect", 42]) 46 | self.data.filter(key=is_answer) 47 | self.assertEqual(self.data.get_data()[0], ("Arthur", "Dent", 42)) 48 | self.assertEqual(self.data.get_data()[1], ("Ford", "Prefect", 42)) 49 | self.data.reset() 50 | 51 | def test_filter_by_key_negation(self): 52 | def is_answer(number): 53 | if number == 42: 54 | return True 55 | 56 | data = pyreports.Executor(Dataset()) 57 | data.data.append(["Arthur", "Dent", 42]) 58 | data.data.append(["Ford", "Prefect", 43]) 59 | data.filter(key=is_answer, negation=True) 60 | self.assertEqual(data.get_data()[0], ("Ford", "Prefect", 43)) 61 | 62 | def test_filter_by_list_and_column(self): 63 | self.data.headers = ["name", "surname", "age"] 64 | self.data.data.append(["Arthur", "Dent", 42]) 65 | self.data.data.append(["Ford", "Prefect", 42]) 66 | self.data.filter([42], column="age") 67 | self.assertEqual(self.data.get_data()[0], ("Arthur", "Dent", 42)) 68 | self.data.reset() 69 | 70 | def test_map(self): 71 | def int_to_string(number): 72 | if isinstance(number, int): 73 | return str(number) 74 | else: 75 | return number 76 | 77 | self.data.data.append(["Arthur", "Dent", 42]) 78 | self.data.data.append(["Ford", "Prefect", 42]) 79 | self.data.map(int_to_string) 80 | self.assertEqual(self.data.get_data()[1], ("Arthur", "Dent", "42")) 81 | self.assertEqual(self.data.get_data()[2], ("Ford", "Prefect", "42")) 82 | self.data.reset() 83 | 84 | def test_select_column(self): 85 | self.data.headers = ["name", "surname", "age"] 86 | self.data.data.append(["Arthur", "Dent", 42]) 87 | self.data.data.append(["Ford", "Prefect", 42]) 88 | # By name 89 | self.assertEqual(self.data.select_column("age"), [35, 42, 42]) 90 | # By number 91 | self.assertEqual(self.data.select_column(2), [35, 42, 42]) 92 | self.data.reset() 93 | 94 | def test_count(self): 95 | self.assertEqual(len(self.data), 2) 96 | self.assertEqual(self.data.count_rows(), 2) 97 | self.data.headers = ["name", "surname", "age"] 98 | self.assertEqual(self.data.count_columns(), 3) 99 | 100 | def test_clone(self): 101 | new_data = self.data.clone() 102 | self.assertNotEqual(new_data, self.data) 103 | self.assertIsInstance(new_data, pyreports.Executor) 104 | self.assertEqual(type(new_data), type(self.data)) 105 | 106 | def test_add_row(self): 107 | fake_data = self.data.clone() 108 | self.assertRaises( 109 | pyreports.exception.ExecutorError, self.data.__add__, fake_data 110 | ) 111 | new_data = Dataset(["Matteo", "Guadrini", 35]) 112 | self.data += new_data 113 | 114 | 115 | class TestReport(unittest.TestCase): 116 | input_data = Dataset(*[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)]) 117 | output_data = pyreports.manager("csv", f"{tmp_folder}/test_csv.csv") 118 | title = "Test report" 119 | filters = ["42"] 120 | column = "age" 121 | count = True 122 | report = pyreports.Report( 123 | input_data=input_data, 124 | title=title, 125 | filters=filters, 126 | map_func=lambda item: str(item) if isinstance(item, int) else item, 127 | column=column, 128 | count=count, 129 | output=output_data, 130 | ) 131 | 132 | def test_report_object(self): 133 | self.assertIsInstance(self.report, pyreports.Report) 134 | 135 | def test_no_output_report_object(self): 136 | new_report = pyreports.Report(input_data=self.input_data) 137 | self.assertIsInstance(new_report, pyreports.Report) 138 | 139 | def test_exec(self): 140 | self.report.exec() 141 | self.assertEqual(self.report.report[0], ("Arthur", "Dent", "42")) 142 | self.assertEqual(self.report.count, 1) 143 | 144 | def test_exec_negation(self): 145 | self.report.negation = True 146 | self.report.exec() 147 | self.assertEqual(self.report.report[0], ("Matteo", "Guadrini", "35")) 148 | self.assertEqual(self.report.count, 1) 149 | self.report.negation = False 150 | 151 | def test_export(self): 152 | self.report.export() 153 | self.assertIsInstance(self.report.output.read(), Dataset) 154 | 155 | def test_reset(self): 156 | self.report.reset() 157 | self.assertEqual(self.report.report, None) 158 | 159 | def test_clone(self): 160 | new_report = self.report.clone() 161 | self.assertNotEqual(new_report, self.report) 162 | self.assertIsInstance(new_report, pyreports.Report) 163 | 164 | 165 | class TestReportDatabase(unittest.TestCase): 166 | input_data = Dataset( 167 | *[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)], 168 | headers=("name", "surname", "age"), 169 | ) 170 | output_data = pyreports.manager("sqlite", f"{tmp_folder}/mydb.db") 171 | title = "Test report" 172 | column = "age" 173 | count = True 174 | report = pyreports.Report( 175 | input_data=input_data, 176 | title=title, 177 | map_func=lambda item: str(item) if isinstance(item, int) else item, 178 | column=column, 179 | count=count, 180 | output=output_data, 181 | ) 182 | 183 | def test_report_object(self): 184 | self.assertIsInstance(self.report, pyreports.Report) 185 | 186 | def test_no_output_report_object(self): 187 | new_report = pyreports.Report(input_data=self.input_data) 188 | self.assertIsInstance(new_report, pyreports.Report) 189 | 190 | def test_exec(self): 191 | self.report.exec() 192 | self.assertEqual(self.report.report[0], ("Matteo", "Guadrini", "35")) 193 | self.assertEqual(self.report.count, 2) 194 | 195 | def test_exec_column(self): 196 | self.report.reset() 197 | self.report.exec(column="age") 198 | self.assertEqual(self.report.report[0], ("Matteo", "Guadrini", "35")) 199 | self.assertEqual(self.report.count, 2) 200 | 201 | def test_exec_column_index(self): 202 | self.report.reset() 203 | self.report.exec(column=2) 204 | self.assertEqual(self.report.report[0], ("Matteo", "Guadrini", "35")) 205 | self.assertEqual(self.report.count, 2) 206 | 207 | def test_export(self): 208 | self.report.export() 209 | self.report.output.execute("SELECT * from test_report") 210 | self.assertIsInstance(self.report.output.fetchall(), Dataset) 211 | 212 | 213 | class TestReportBook(unittest.TestCase): 214 | input_data = Dataset(*[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)]) 215 | output_data = pyreports.manager("csv", f"{tmp_folder}/test_csv.csv") 216 | output_data2 = pyreports.manager("csv", f"{tmp_folder}/test_csv2.csv") 217 | title = "Test report" 218 | filters = ["42"] 219 | column = "age" 220 | count = True 221 | report1 = pyreports.Report( 222 | input_data=input_data, 223 | title=title + "1", 224 | filters=filters, 225 | map_func=lambda item: str(item) if isinstance(item, int) else item, 226 | column=column, 227 | count=count, 228 | output=output_data, 229 | ) 230 | report2 = pyreports.Report( 231 | input_data=input_data, 232 | title=title + "2", 233 | filters=filters, 234 | map_func=lambda item: str(item) if isinstance(item, int) else item, 235 | column=column, 236 | count=count, 237 | output=output_data2, 238 | ) 239 | book = pyreports.ReportBook([report1]) 240 | 241 | def test_report_book_instance(self): 242 | self.assertIsInstance(self.book, pyreports.ReportBook) 243 | 244 | def test_add_report(self): 245 | self.book.add(self.report2) 246 | self.assertRaises( 247 | pyreports.exception.ReportDataError, self.book.add, [self.report2] 248 | ) 249 | self.assertEqual(len(self.book), 2) 250 | 251 | def test_remove_report(self): 252 | self.book.remove() 253 | self.book.remove(0) 254 | self.assertEqual(len(self.book), 0) 255 | 256 | def test_merge_report_books(self): 257 | book1 = pyreports.ReportBook([self.report1]) 258 | book2 = pyreports.ReportBook([self.report2]) 259 | final_book1 = book1 + book2 260 | final_book1 += book1 261 | self.assertEqual(book1, final_book1) 262 | self.assertEqual(len(final_book1), 4) 263 | self.assertRaises( 264 | pyreports.exception.ReportDataError, final_book1.__add__, [final_book1] 265 | ) 266 | 267 | def test_export_book(self): 268 | self.book.export() 269 | self.book.export(output=f"{tmp_folder}/test_export_book.xlsx") 270 | 271 | 272 | if __name__ == "__main__": 273 | unittest.main() 274 | -------------------------------------------------------------------------------- /tests/test_data.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import tablib 4 | 5 | import pyreports 6 | from tablib import Dataset 7 | 8 | 9 | class TestDataTools(unittest.TestCase): 10 | data = Dataset( 11 | *[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42), ("Ford", "Prefect", 42)] 12 | ) 13 | data.headers = ["name", "surname", "age"] 14 | 15 | def test_average(self): 16 | self.assertEqual(int(pyreports.average(self.data, "age")), 39) 17 | 18 | def test_most_common(self): 19 | self.assertEqual(pyreports.most_common(self.data, "age"), 42) 20 | 21 | def test_percentage(self): 22 | self.assertEqual(int(pyreports.percentage(self.data, 42)), 66) 23 | 24 | def test_counter(self): 25 | c = pyreports.counter(self.data, 2) 26 | self.assertEqual(list(c.keys()), [35, 42]) 27 | self.assertEqual(c.most_common(1), [(42, 2)]) 28 | 29 | def test_aggregate(self): 30 | names = self.data.get_col(0) 31 | surnames = self.data.get_col(1) 32 | ages = self.data.get_col(2) 33 | self.assertEqual( 34 | pyreports.aggregate(names, surnames, ages)[0], ("Matteo", "Guadrini", 35) 35 | ) 36 | ages = ["Name", "Surname"] 37 | self.assertRaises( 38 | tablib.InvalidDimensions, pyreports.aggregate, names, surnames, ages 39 | ) 40 | self.assertRaises(pyreports.DataObjectError, pyreports.aggregate, names) 41 | 42 | def test_aggregate_fill_empty(self): 43 | names = self.data.get_col(0) 44 | surnames = self.data.get_col(1) 45 | ages = ["Name", "Surname"] 46 | self.assertEqual( 47 | pyreports.aggregate(names, surnames, ages, fill_empty=True)[2], 48 | ("Ford", "Prefect", None), 49 | ) 50 | 51 | def test_chunks(self): 52 | data = Dataset( 53 | *[ 54 | ("Matteo", "Guadrini", 35), 55 | ("Arthur", "Dent", 42), 56 | ("Ford", "Prefect", 42), 57 | ] 58 | ) 59 | data.extend( 60 | [ 61 | ("Matteo", "Guadrini", 35), 62 | ("Arthur", "Dent", 42), 63 | ("Ford", "Prefect", 42), 64 | ] 65 | ) 66 | data.headers = ["name", "surname", "age"] 67 | self.assertEqual( 68 | list(pyreports.chunks(data, 4))[0][0], ("Matteo", "Guadrini", 35) 69 | ) 70 | 71 | def test_merge(self): 72 | self.assertEqual( 73 | pyreports.merge(self.data, self.data)[3], ("Matteo", "Guadrini", 35) 74 | ) 75 | 76 | def test_deduplication(self): 77 | data = Dataset( 78 | *[ 79 | ("Matteo", "Guadrini", 35), 80 | ("Arthur", "Dent", 42), 81 | ("Matteo", "Guadrini", 35), 82 | ] 83 | ) 84 | self.assertEqual(len(pyreports.deduplicate(data)), 2) 85 | 86 | def test_subset(self): 87 | data = Dataset( 88 | *[ 89 | ("Matteo", "Guadrini", 35), 90 | ("Arthur", "Dent", 42), 91 | ("Matteo", "Guadrini", 35), 92 | ], 93 | headers=("name", "surname", "age"), 94 | ) 95 | new_data = pyreports.subset(data, "age") 96 | self.assertEqual(new_data[0], (35,)) 97 | self.assertEqual(new_data[1], (42,)) 98 | self.assertEqual(new_data[2], (35,)) 99 | 100 | def test_sort(self): 101 | data = Dataset( 102 | *[ 103 | ("Matteo", "Guadrini", 35), 104 | ("Arthur", "Dent", 42), 105 | ("Matteo", "Guadrini", 35), 106 | ], 107 | headers=("name", "surname", "age"), 108 | ) 109 | new_data = pyreports.sort(data, "age") 110 | self.assertEqual(new_data[1], ("Matteo", "Guadrini", 35)) 111 | new_data_reversed = pyreports.sort(data, "age", reverse=True) 112 | self.assertEqual(new_data_reversed[0], ("Arthur", "Dent", 42)) 113 | 114 | def test_data_object(self): 115 | data = pyreports.DataObject(Dataset(*[("Matteo", "Guadrini", 35)])) 116 | self.assertIsInstance(data, pyreports.DataObject) 117 | self.assertIsInstance(data.data, tablib.Dataset) 118 | 119 | def test_data_object_clone(self): 120 | data = pyreports.DataObject(Dataset(*[("Matteo", "Guadrini", 35)])) 121 | new_data = data.clone() 122 | self.assertIsInstance(new_data, pyreports.DataObject) 123 | self.assertIsInstance(new_data.data, tablib.Dataset) 124 | 125 | def test_data_adapters(self): 126 | data = pyreports.DataAdapters(Dataset(*[("Matteo", "Guadrini", 35)])) 127 | self.assertIsInstance(data, pyreports.DataAdapters) 128 | 129 | def test_data_adapters_aggregate(self): 130 | names = self.data.get_col(0) 131 | surnames = self.data.get_col(1) 132 | ages = self.data.get_col(2) 133 | data = pyreports.DataAdapters(Dataset()) 134 | self.assertRaises( 135 | pyreports.DataObjectError, data.aggregate, names, surnames, ages 136 | ) 137 | data = pyreports.DataAdapters(Dataset(*[("Heart",)])) 138 | data.aggregate(names, surnames, ages) 139 | self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) 140 | 141 | def test_data_adapters_merge(self): 142 | data = pyreports.DataAdapters(Dataset()) 143 | self.assertRaises(pyreports.DataObjectError, data.merge, self.data) 144 | data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) 145 | data.merge(self.data) 146 | 147 | def test_data_adapters_counter(self): 148 | data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) 149 | data.merge(self.data) 150 | counter = data.counter() 151 | self.assertEqual(counter["Arthur"], 2) 152 | 153 | def test_adapters_chunks(self): 154 | data = pyreports.DataAdapters( 155 | Dataset( 156 | *[ 157 | ("Matteo", "Guadrini", 35), 158 | ("Arthur", "Dent", 42), 159 | ("Ford", "Prefect", 42), 160 | ] 161 | ) 162 | ) 163 | data.data.extend( 164 | [ 165 | ("Matteo", "Guadrini", 35), 166 | ("Arthur", "Dent", 42), 167 | ("Ford", "Prefect", 42), 168 | ] 169 | ) 170 | data.data.headers = ["name", "surname", "age"] 171 | self.assertEqual(list(data.chunks(4))[0][0], ("Matteo", "Guadrini", 35)) 172 | 173 | def test_data_adapters_deduplicate(self): 174 | data = pyreports.DataAdapters( 175 | Dataset( 176 | *[ 177 | ("Matteo", "Guadrini", 35), 178 | ("Arthur", "Dent", 42), 179 | ("Matteo", "Guadrini", 35), 180 | ] 181 | ) 182 | ) 183 | data.deduplicate() 184 | self.assertEqual(len(data.data), 2) 185 | 186 | def test_data_adapters_iter(self): 187 | data = pyreports.DataAdapters( 188 | Dataset( 189 | *[ 190 | ("Matteo", "Guadrini", 35), 191 | ("Arthur", "Dent", 42), 192 | ("Matteo", "Guadrini", 35), 193 | ] 194 | ) 195 | ) 196 | self.assertEqual(list(iter(data.data))[1], ("Arthur", "Dent", 42)) 197 | 198 | def test_data_adapters_get_items(self): 199 | data = pyreports.DataAdapters( 200 | Dataset( 201 | *[ 202 | ("Matteo", "Guadrini", 35), 203 | ("Arthur", "Dent", 42), 204 | ("Matteo", "Guadrini", 35), 205 | ], 206 | headers=("name", "surname", "age"), 207 | ) 208 | ) 209 | # Get row 210 | self.assertEqual(data[1], ("Arthur", "Dent", 42)) 211 | # Get column 212 | self.assertEqual(data["name"], ["Matteo", "Arthur", "Matteo"]) 213 | 214 | def test_data_adapters_subset(self): 215 | data = pyreports.DataAdapters( 216 | Dataset( 217 | *[ 218 | ("Matteo", "Guadrini", 35), 219 | ("Arthur", "Dent", 42), 220 | ("Matteo", "Guadrini", 35), 221 | ], 222 | headers=("name", "surname", "age"), 223 | ) 224 | ) 225 | new_data = data.subset("age") 226 | self.assertEqual(new_data[0], (35,)) 227 | self.assertEqual(new_data[1], (42,)) 228 | self.assertEqual(new_data[2], (35,)) 229 | 230 | def test_data_adapters_sort(self): 231 | data = pyreports.DataAdapters( 232 | Dataset( 233 | *[ 234 | ("Matteo", "Guadrini", 35), 235 | ("Arthur", "Dent", 42), 236 | ("Matteo", "Guadrini", 35), 237 | ], 238 | headers=("name", "surname", "age"), 239 | ) 240 | ) 241 | new_data = data.sort("age") 242 | self.assertEqual(new_data[1], ("Matteo", "Guadrini", 35)) 243 | new_data_reversed = data.sort("age", reverse=True) 244 | self.assertEqual(new_data_reversed[0], ("Arthur", "Dent", 42)) 245 | 246 | def test_data_printers(self): 247 | data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) 248 | self.assertIsInstance(data, pyreports.DataPrinters) 249 | self.assertIsInstance(data.data, tablib.Dataset) 250 | 251 | def test_data_printers_len(self): 252 | data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) 253 | self.assertEqual(1, len(data)) 254 | 255 | def test_data_printers_average(self): 256 | data = pyreports.DataPrinters( 257 | Dataset(*[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)]) 258 | ) 259 | data.data.headers = ["Name", "Surname", "Age"] 260 | self.assertEqual(data.average(2), 38.5) 261 | 262 | def test_data_printers_most_common(self): 263 | data = pyreports.DataPrinters( 264 | Dataset( 265 | *[ 266 | ("Matteo", "Guadrini", 35), 267 | ("Arthur", "Dent", 42), 268 | ("Ford", "Prefect", 42), 269 | ] 270 | ) 271 | ) 272 | data.data.headers = ["Name", "Surname", "Age"] 273 | self.assertEqual(data.most_common("Age"), 42) 274 | 275 | def test_data_printers_percentage(self): 276 | data = pyreports.DataPrinters( 277 | Dataset( 278 | *[ 279 | ("Matteo", "Guadrini", 35), 280 | ("Arthur", "Dent", 42), 281 | ("Ford", "Prefect", 42), 282 | ] 283 | ) 284 | ) 285 | data.data.headers = ["Name", "Surname", "Age"] 286 | self.assertEqual(data.percentage(42), 66.66666666666666) 287 | 288 | 289 | if __name__ == "__main__": 290 | unittest.main() 291 | -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import pyreports 3 | from tablib import Dataset 4 | from unittest.mock import MagicMock, patch 5 | 6 | 7 | class TestDBConnection(unittest.TestCase): 8 | def test_connection(self): 9 | # pyreports.io.Connection object 10 | self.assertRaises(TypeError, pyreports.io.Connection) 11 | 12 | def test_sqllite_connection(self): 13 | # Simulate pyreports.io.SQLliteConnection object 14 | conn = MagicMock() 15 | with patch(target="sqlite3.connect") as mock: 16 | # Test connect 17 | conn.connection = mock.return_value 18 | conn.cursor = conn.connection.cursor.return_value 19 | conn.connection.database = "mydb.db" 20 | self.assertEqual(conn.connection.database, "mydb.db") 21 | # Test close 22 | conn.cursor.close() 23 | 24 | def test_mysql_connection(self): 25 | # Simulate pyreports.io.MySQLConnection object 26 | conn = MagicMock() 27 | with patch(target="mysql.connector.connect") as mock: 28 | # Test connect 29 | conn.connection = mock.return_value 30 | conn.cursor = conn.connection.cursor.return_value 31 | conn.connection.host = "mysqldb.local" 32 | conn.connection.database = "mydb" 33 | conn.connection.username = "username" 34 | conn.connection.password = "password" 35 | conn.connection.port = 3306 36 | self.assertEqual(conn.connection.host, "mysqldb.local") 37 | self.assertEqual(conn.connection.database, "mydb") 38 | self.assertEqual(conn.connection.username, "username") 39 | self.assertEqual(conn.connection.password, "password") 40 | self.assertEqual(conn.connection.port, 3306) 41 | # Test close 42 | conn.cursor.close() 43 | 44 | def test_postgresqldb_connection(self): 45 | # Simulate pyreports.io.PostgreSQLConnection object 46 | conn = MagicMock() 47 | with patch(target="psycopg2.connect") as mock: 48 | # Test connect 49 | conn.connection = mock.return_value 50 | conn.cursor = conn.connection.cursor.return_value 51 | conn.connection.host = "postgresqldb.local" 52 | conn.connection.database = "mydb" 53 | conn.connection.username = "username" 54 | conn.connection.password = "password" 55 | conn.connection.port = 5432 56 | self.assertEqual(conn.connection.host, "postgresqldb.local") 57 | self.assertEqual(conn.connection.database, "mydb") 58 | self.assertEqual(conn.connection.username, "username") 59 | self.assertEqual(conn.connection.password, "password") 60 | self.assertEqual(conn.connection.port, 5432) 61 | # Test close 62 | conn.cursor.close() 63 | 64 | 65 | class TestDBManager(unittest.TestCase): 66 | conn = MagicMock() 67 | with patch(target="psycopg2.connect") as mock: 68 | conn.connection = mock.return_value 69 | conn.cursor = conn.connection.cursor.return_value 70 | conn.connection.host = "postgresqldb.local" 71 | conn.connection.database = "mydb" 72 | conn.connection.username = "username" 73 | conn.connection.password = "password" 74 | conn.connection.port = 5432 75 | 76 | def test_db_manager(self): 77 | # Test database manager 78 | db_manager = pyreports.io.DatabaseManager(connection=self.conn) 79 | self.assertIsInstance(db_manager, pyreports.io.DatabaseManager) 80 | # Test reconnect 81 | db_manager.reconnect() 82 | # Test SELECT query 83 | db_manager.execute("SELECT * from test") 84 | data = db_manager.fetchall() 85 | self.assertIsInstance(data, Dataset) 86 | # Test store procedure 87 | db_manager.callproc("myproc") 88 | data = db_manager.fetchone() 89 | self.assertIsInstance(data, Dataset) 90 | 91 | 92 | class TestNoSQLManager(unittest.TestCase): 93 | conn = MagicMock() 94 | with patch(target="nosqlapi.Connection") as mock: 95 | conn.connection = mock.return_value 96 | conn.session = conn.connection.connect() 97 | conn.connection.host = "mongodb.local" 98 | conn.connection.database = "mydb" 99 | conn.connection.username = "username" 100 | conn.connection.password = "password" 101 | conn.connection.port = 27017 102 | 103 | def test_nosql_manager(self): 104 | # Test nosql database manager 105 | nosql_manager = pyreports.io.NoSQLManager(connection=self.conn) 106 | self.assertIsInstance(nosql_manager, pyreports.io.NoSQLManager) 107 | # Test get data 108 | data = nosql_manager.get("doc1") 109 | self.assertIsInstance(data, Dataset) 110 | # Test find data 111 | data = nosql_manager.find({"name": "Arthur"}) 112 | self.assertIsInstance(data, Dataset) 113 | 114 | 115 | class TestLDAPManager(unittest.TestCase): 116 | conn = MagicMock() 117 | with patch(target="ldap3.Server") as mock: 118 | conn.connector = mock.return_value 119 | with patch(target="ldap3.Connection") as mock: 120 | conn.bind = mock.return_value 121 | 122 | def test_bind(self): 123 | self.conn.bind.bind() 124 | self.conn.bind.unbind() 125 | 126 | def test_query(self): 127 | self.conn.bind.search( 128 | "OU=test,DC=test,DC=local", "objectCategory=person", ["name", "sn", "phone"] 129 | ) 130 | 131 | 132 | if __name__ == "__main__": 133 | unittest.main() 134 | -------------------------------------------------------------------------------- /tests/test_file.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tempfile import gettempdir 3 | from unittest.mock import MagicMock, mock_open, patch 4 | 5 | from tablib import Dataset 6 | import pyreports 7 | 8 | tmp_folder = gettempdir() 9 | 10 | 11 | class TestFile(unittest.TestCase): 12 | def test_file(self): 13 | # Simulate pyreports.io.File object 14 | file = MagicMock() 15 | file.raw_data = ["Matteo\n", "Guadrini\n", "35"] 16 | data = Dataset() 17 | for line in file.raw_data: 18 | data.append([line]) 19 | # Read file data 20 | file.read = MagicMock(return_value=data) 21 | lines = file.read() 22 | self.assertIsInstance(lines, Dataset) 23 | # Write file data 24 | read_data = "".join(file.raw_data) 25 | with open(f"{tmp_folder}/test_file.txt", "w") as wf: 26 | wf.write(read_data) 27 | with patch("__main__.open", mock_open(read_data=read_data)): 28 | with open(f"{tmp_folder}/test_file.txt") as rf: 29 | result = rf.read() 30 | self.assertEqual(read_data, result) 31 | # Real pyreports.io.File object 32 | file_real = pyreports.io.TextFile(f"{tmp_folder}/test_file.txt") 33 | real_data = file_real.read() 34 | self.assertIsInstance(real_data, Dataset) 35 | file_real.write(real_data) 36 | self.assertEqual(file_real.read()[0][0], "Matteo") 37 | 38 | def test_log(self): 39 | log_real = pyreports.io.LogFile(f"{tmp_folder}/test_log.log") 40 | # Write data 41 | log_real.write( 42 | ( 43 | [ 44 | "111.222.333.123", 45 | "HOME", 46 | "- [01/Feb/1998:01:08:39 -0800]", 47 | "GET", 48 | "/bannerad/ad.htm", 49 | "HTTP/1.0", 50 | "200", 51 | "198", 52 | "http://www.referrer.com/bannerad/ba_intro.htm", 53 | "Mozilla/4.01", 54 | "(Macintosh; I; PPC)", 55 | ], 56 | [ 57 | "111.222.333.123", 58 | "AWAY", 59 | "- [01/Feb/1998:01:08:39 -0800]", 60 | "GET", 61 | "/bannerad/ad7.gif", 62 | "HTTP/1.0", 63 | "200", 64 | "9332", 65 | "http://www.referrer.com/bannerad/ba_intro.htm", 66 | "Mozilla/4.01", 67 | "(Macintosh; I; PPC)", 68 | ], 69 | [ 70 | "111.222.333.123", 71 | "AWAY", 72 | "- [01/Feb/1998:01:08:39 -0800]", 73 | "GET", 74 | "/bannerad/click.htm", 75 | "HTTP/1.0", 76 | "200", 77 | "28083", 78 | "http://www.referrer.com/bannerad/ba_intro.htm", 79 | "Mozilla/4.01", 80 | "(Macintosh; I; PPC)", 81 | ], 82 | ) 83 | ) 84 | # Read data 85 | real_data = log_real.read( 86 | r"([(\d\.)]+) (.*) \[(.*?)\] (.*?) (\d+) (\d+) (.*?) (.*?) (\(.*?\))", 87 | headers=( 88 | "ip", 89 | "user", 90 | "date", 91 | "req", 92 | "ret", 93 | "size", 94 | "url", 95 | "browser", 96 | "host", 97 | ), 98 | ) 99 | self.assertIsInstance(real_data, Dataset) 100 | 101 | def test_csv(self): 102 | csv_real = pyreports.io.CsvFile(f"{tmp_folder}/test_csv.csv") 103 | # Write data 104 | csv_real.write(["Matteo", "Guadrini", 35]) 105 | # Read data 106 | real_data = csv_real.read() 107 | self.assertIsInstance(real_data, Dataset) 108 | # Iterate csv 109 | for row in csv_real: 110 | self.assertIsInstance(row, str) 111 | 112 | def test_json(self): 113 | json_real = pyreports.io.JsonFile(f"{tmp_folder}/test_json.json") 114 | # Write data 115 | json_real.write(["Matteo", "Guadrini", 35]) 116 | # Read data 117 | real_data = json_real.read() 118 | self.assertIsInstance(real_data, Dataset) 119 | 120 | def test_yaml(self): 121 | yaml_real = pyreports.io.YamlFile(f"{tmp_folder}/test_yaml.yml") 122 | # Write data 123 | yaml_real.write(["Matteo", "Guadrini", 35]) 124 | # Read data 125 | real_data = yaml_real.read() 126 | self.assertIsInstance(real_data, Dataset) 127 | 128 | def test_excel(self): 129 | excel_real = pyreports.io.ExcelFile(f"{tmp_folder}/test_excel.xlsx") 130 | # Write data 131 | excel_real.write(["Matteo", "Guadrini", 35]) 132 | # Read data 133 | real_data = excel_real.read() 134 | self.assertIsInstance(real_data, Dataset) 135 | 136 | 137 | class TestFileManager(unittest.TestCase): 138 | def test_file_manager(self): 139 | # Test file manager 140 | file_manager = pyreports.io.create_file_manager( 141 | "file", f"{tmp_folder}/test_file.txt" 142 | ) 143 | # Write file 144 | file_manager.write(["Matteo", "Guadrini", 45]) 145 | # Read file 146 | self.assertIsInstance(file_manager.read(), Dataset) 147 | 148 | def test_log_manager(self): 149 | # Test log manager 150 | log_manager = pyreports.io.create_file_manager( 151 | "log", f"{tmp_folder}/test_log.txt" 152 | ) 153 | # Write file 154 | log_manager.write( 155 | ( 156 | [ 157 | "111.222.333.123", 158 | "HOME", 159 | "- [01/Feb/1998:01:08:39 -0800]", 160 | "GET", 161 | "/bannerad/ad.htm", 162 | "HTTP/1.0", 163 | "200", 164 | "198", 165 | "http://www.referrer.com/bannerad/ba_intro.htm", 166 | "Mozilla/4.01", 167 | "(Macintosh; I; PPC)", 168 | ], 169 | [ 170 | "111.222.333.123", 171 | "AWAY", 172 | "- [01/Feb/1998:01:08:39 -0800]", 173 | "GET", 174 | "/bannerad/ad7.gif", 175 | "HTTP/1.0", 176 | "200", 177 | "9332", 178 | "http://www.referrer.com/bannerad/ba_intro.htm", 179 | "Mozilla/4.01", 180 | "(Macintosh; I; PPC)", 181 | ], 182 | [ 183 | "111.222.333.123", 184 | "AWAY", 185 | "- [01/Feb/1998:01:08:39 -0800]", 186 | "GET", 187 | "/bannerad/click.htm", 188 | "HTTP/1.0", 189 | "200", 190 | "28083", 191 | "http://www.referrer.com/bannerad/ba_intro.htm", 192 | "Mozilla/4.01", 193 | "(Macintosh; I; PPC)", 194 | ], 195 | ) 196 | ) 197 | # Read file 198 | self.assertIsInstance( 199 | log_manager.read( 200 | r"([(\d\.)]+) (.*) \[(.*?)\] (.*?) (\d+) (\d+) (.*?) (.*?) (\(.*?\))", 201 | headers=( 202 | "ip", 203 | "user", 204 | "date", 205 | "req", 206 | "ret", 207 | "size", 208 | "url", 209 | "browser", 210 | "host", 211 | ), 212 | ), 213 | Dataset, 214 | ) 215 | 216 | def test_csv_manager(self): 217 | # Test csv manager 218 | csv_manager = pyreports.io.create_file_manager( 219 | "csv", f"{tmp_folder}/test_csv.csv" 220 | ) 221 | # Write file 222 | csv_manager.write(["Matteo", "Guadrini", 45]) 223 | # Read file 224 | self.assertIsInstance(csv_manager.read(), Dataset) 225 | 226 | def test_json_manager(self): 227 | # Test json manager 228 | json_manager = pyreports.io.create_file_manager( 229 | "json", f"{tmp_folder}/test_json.json" 230 | ) 231 | # Write file 232 | json_manager.write(["Matteo", "Guadrini", 45]) 233 | # Read file 234 | self.assertIsInstance(json_manager.read(), Dataset) 235 | 236 | def test_yaml_manager(self): 237 | # Test yaml manager 238 | yaml_manager = pyreports.io.create_file_manager( 239 | "yaml", f"{tmp_folder}/test_yaml.yml" 240 | ) 241 | # Write file 242 | yaml_manager.write(["Matteo", "Guadrini", 45]) 243 | # Read file 244 | self.assertIsInstance(yaml_manager.read(), Dataset) 245 | 246 | def test_excel_manager(self): 247 | # Test excel manager 248 | excel_manager = pyreports.io.create_file_manager( 249 | "xlsx", f"{tmp_folder}/test_excel.xlsx" 250 | ) 251 | # Write file 252 | excel_manager.write(["Matteo", "Guadrini", 45]) 253 | # Read file 254 | self.assertIsInstance(excel_manager.read(), Dataset) 255 | 256 | def test_manager_for_file(self): 257 | # Test file manager 258 | file_manager = pyreports.io.manager("file", f"{tmp_folder}/test_file.txt") 259 | # Write file 260 | file_manager.write(["Matteo", "Guadrini", 45]) 261 | # Read file 262 | self.assertIsInstance(file_manager.read(), Dataset) 263 | 264 | def test_manager_for_csv(self): 265 | # Test csv manager 266 | csv_manager = pyreports.io.manager("csv", f"{tmp_folder}/test_csv.csv") 267 | # Write file 268 | csv_manager.write(["Matteo", "Guadrini", 45]) 269 | # Read file 270 | self.assertIsInstance(csv_manager.read(), Dataset) 271 | 272 | def test_manager_for_json(self): 273 | # Test json manager 274 | json_manager = pyreports.io.manager("csv", f"{tmp_folder}/test_json.json") 275 | # Write file 276 | json_manager.write(["Matteo", "Guadrini", 45]) 277 | # Read file 278 | self.assertIsInstance(json_manager.read(), Dataset) 279 | 280 | def test_manager_for_yaml(self): 281 | # Test yaml manager 282 | yaml_manager = pyreports.io.manager("csv", f"{tmp_folder}/test_yaml.yml") 283 | # Write file 284 | yaml_manager.write(["Matteo", "Guadrini", 45]) 285 | # Read file 286 | self.assertIsInstance(yaml_manager.read(), Dataset) 287 | 288 | def test_manager_for_excel(self): 289 | # Test excel manager 290 | excel_manager = pyreports.io.manager("csv", f"{tmp_folder}/test_excel.xlsx") 291 | # Write file 292 | excel_manager.write(["Matteo", "Guadrini", 45]) 293 | # Read file 294 | self.assertIsInstance(excel_manager.read(), Dataset) 295 | 296 | 297 | if __name__ == "__main__": 298 | unittest.main() 299 | --------------------------------------------------------------------------------