├── .circleci └── config.yml ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── COPYRIGHT ├── LICENSE ├── MANIFEST.in ├── README.md ├── RELEASING_AEPP.md ├── aepp ├── __init__.py ├── __version__.py ├── accesscontrol.py ├── catalog.py ├── classmanager.py ├── config.py ├── configs.py ├── connector.py ├── customerprofile.py ├── dataaccess.py ├── dataprep.py ├── datasets.py ├── datatypemanager.py ├── destination.py ├── destinationinstanceservice.py ├── edge.py ├── exportDatasetToDataLandingZone.py ├── fieldgroupmanager.py ├── flowservice.py ├── hygiene.py ├── identity.py ├── ingestion.py ├── observability.py ├── observability_gdpr.pickle ├── observability_identity.pickle ├── observability_ingestion.pickle ├── observability_queryService.pickle ├── observability_realTime.pickle ├── policy.py ├── privacyservice.py ├── queryservice.py ├── sandboxes.py ├── schema.py ├── schemamanager.py ├── segmentation.py ├── sensei.py ├── som.py ├── synchronizer.py ├── tags.py └── utils.py ├── docs ├── accesscontrol.md ├── add-privacy-service-api.png ├── catalog.md ├── classManager.md ├── customerprofile.md ├── dataTypeManager.md ├── dataaccess.md ├── dataprep.md ├── datasets.md ├── destination.md ├── destinationinstanceservice.md ├── edge.md ├── fieldGroupManager.md ├── flowservice.md ├── getting-started.md ├── hygiene.md ├── identity.md ├── ingestion.md ├── logging.md ├── main.md ├── observability.md ├── policy.md ├── privacyservice.md ├── queryservice.md ├── release-build.png ├── release-upload.png ├── releases.md ├── sandboxes.md ├── schema.md ├── schemaManager.md ├── segmentation.md ├── som.md ├── synchronizer.md └── tags.md ├── notebooks ├── .ipynb_checkpoints │ ├── 01 - Guide - Introduction to aepp-checkpoint.ipynb │ └── 02 - Guide - Introduction to Schema-checkpoint.ipynb ├── 01 - Guide - Introduction to aepp.ipynb ├── 02 - Guide - Introduction to Schema.ipynb ├── 03 - Guide - Introduction to Catalog Service.ipynb ├── 04 - Guide - Introduction to Customer Profile.ipynb ├── developer.adobe.project_list.png ├── developer.adobe.project_oauth_detail.png ├── developer.adobe.project_overview.png ├── example.json └── profile-entity-composition.png ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── catalog_test.py ├── dataaccess_test.py ├── datasets_test.py ├── destinationinstanceservice_test.py ├── exportDatasetToDatalandingZone_test.py ├── flowservice_test.py ├── schema_test.py └── som_test.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Use the latest 2.1 version of CircleCI pipeline process engine. 2 | # See: https://circleci.com/docs/2.0/configuration-reference 3 | version: 2.1 4 | 5 | # Orbs are reusable packages of CircleCI configuration that you may share across projects, enabling you to create encapsulated, parameterized commands, jobs, and executors that can be used across multiple projects. 6 | # See: https://circleci.com/docs/2.0/orb-intro/ 7 | orbs: 8 | # The python orb contains a set of prepackaged CircleCI configuration you can use repeatedly in your configuration files 9 | # Orb commands and jobs help you with common scripting around a language/tool 10 | # so you dont have to copy and paste it everywhere. 11 | # See the orb documentation here: https://circleci.com/developer/orbs/orb/circleci/python 12 | python: circleci/python@2.1.0 13 | 14 | # Define a job to be invoked later in a workflow. 15 | # See: https://circleci.com/docs/2.0/configuration-reference/#jobs 16 | jobs: 17 | build-and-test: # This is the name of the job, feel free to change it to better match what you're trying to do! 18 | # These next lines defines a Docker executors: https://circleci.com/docs/2.0/executor-types/ 19 | # You can specify an image from Dockerhub or use one of the convenience images from CircleCI's Developer Hub 20 | # A list of available CircleCI Docker convenience images are available here: https://circleci.com/developer/images/image/cimg/python 21 | # The executor is the environment in which the steps below will be executed - below will use a python 3.10.2 container 22 | # Change the version below to your required version of python 23 | docker: 24 | - image: cimg/python:3.10.18 25 | # Checkout the code as the first step. This is a dedicated CircleCI step. 26 | # The python orb's install-packages step will install the dependencies from a Pipfile via Pipenv by default. 27 | # Here we're making sure we use just use the system-wide pip. By default it uses the project root's requirements.txt. 28 | # Then run your tests! 29 | # CircleCI will report the results back to your VCS provider. 30 | steps: 31 | - checkout 32 | - python/install-packages: 33 | pkg-manager: pip 34 | # app-dir: ~/project/package-directory/ # If your requirements.txt isn't in the root directory. 35 | # pip-dependency-file: test-requirements.txt # if you have a different name for your requirements file, maybe one that combines your runtime and test requirements. 36 | - run: 37 | name: Run tests 38 | # This assumes pytest is installed via the install-package step above 39 | command: pytest 40 | 41 | # Invoke jobs via workflows 42 | # See: https://circleci.com/docs/2.0/configuration-reference/#workflows 43 | workflows: 44 | sample: # This is the name of the workflow, feel free to change it to better match your workflow. 45 | # Inside the workflow, you define the jobs you want to run. 46 | jobs: 47 | - build-and-test 48 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for choosing to contribute! 4 | 5 | The following are a set of guidelines to follow when contributing to this project. 6 | 7 | ## Code Of Conduct 8 | 9 | This project adheres to the Adobe [code of conduct](../CODE_OF_CONDUCT.md). By participating, 10 | you are expected to uphold this code. Please report unacceptable behavior to 11 | [Grp-opensourceoffice@adobe.com](mailto:Grp-opensourceoffice@adobe.com). 12 | 13 | ## Have A Question? 14 | 15 | Start by filing an issue. The existing committers on this project work to reach 16 | consensus around project direction and issue solutions within issue threads 17 | (when appropriate). 18 | 19 | ## Contributor License Agreement 20 | 21 | All third-party contributions to this project must be accompanied by a signed contributor 22 | license agreement. This gives Adobe permission to redistribute your contributions 23 | as part of the project. [Sign our CLA](https://opensource.adobe.com/cla.html). You 24 | only need to submit an Adobe CLA one time, so if you have submitted one previously, 25 | you are good to go! 26 | 27 | ## Code Reviews 28 | 29 | All submissions should come in the form of pull requests and need to be reviewed 30 | by project committers. Read [GitHub's pull request documentation](https://help.github.com/articles/about-pull-requests/) 31 | for more information on sending pull requests. 32 | 33 | Lastly, please follow the [pull request template](PULL_REQUEST_TEMPLATE.md) when 34 | submitting a pull request! 35 | 36 | ## From Contributor To Committer 37 | 38 | We love contributions from our community! If you'd like to go a step beyond contributor 39 | and become a committer with full write access and a say in the project, you must 40 | be invited to the project. The existing committers employ an internal nomination 41 | process that must reach lazy consensus (silence is approval) before invitations 42 | are issued. If you feel you are qualified and want to get more deeply involved, 43 | feel free to reach out to existing committers to have a conversation about that. 44 | 45 | ## Security Issues 46 | 47 | Security issues shouldn't be reported on this issue tracker. Instead, [file an issue to our security experts](https://helpx.adobe.com/security/alertus.html). 48 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ### Expected Behaviour 5 | 6 | ### Actual Behaviour 7 | 8 | ### Reproduce Scenario (including but not limited to) 9 | 10 | #### Steps to Reproduce 11 | 12 | #### Platform and Version 13 | 14 | #### Sample Code that illustrates the problem 15 | 16 | #### Logs taken while reproducing problem 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 6 | 7 | ## Related Issue 8 | 9 | 10 | 11 | 12 | 13 | 14 | ## Motivation and Context 15 | 16 | 17 | 18 | ## How Has This Been Tested? 19 | 20 | 21 | 22 | 23 | 24 | ## Screenshots (if appropriate): 25 | 26 | ## Types of changes 27 | 28 | 29 | 30 | - [ ] Bug fix (non-breaking change which fixes an issue) 31 | - [ ] New feature (non-breaking change which adds functionality) 32 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 33 | 34 | ## Checklist: 35 | 36 | 37 | 38 | 39 | - [ ] I have signed the [Adobe Open Source CLA](https://opensource.adobe.com/cla.html). 40 | - [ ] My code follows the code style of this project. 41 | - [ ] My change requires a change to the documentation. 42 | - [ ] I have updated the documentation accordingly. 43 | - [ ] I have read the **CONTRIBUTING** document. 44 | - [ ] I have added tests to cover my changes. 45 | - [ ] All new and existing tests passed. 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | build/ 3 | dist/ 4 | aepp.egg-info/ 5 | notebooks/myconfigFile.json 6 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Adobe Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our project and community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. 6 | 7 | We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. 8 | 9 | ## Our Standards 10 | 11 | Examples of behavior that contribute to a positive environment for our project and community include: 12 | 13 | * Demonstrating empathy and kindness toward other people 14 | * Being respectful of differing opinions, viewpoints, and experiences 15 | * Giving and gracefully accepting constructive feedback 16 | * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience 17 | * Focusing on what is best, not just for us as individuals but for the overall community 18 | 19 | Examples of unacceptable behavior include: 20 | 21 | * The use of sexualized language or imagery, and sexual attention or advances of any kind 22 | * Trolling, insulting or derogatory comments, and personal or political attacks 23 | * Public or private harassment 24 | * Publishing others’ private information, such as a physical or email address, without their explicit permission 25 | * Other conduct which could reasonably be considered inappropriate in a professional setting 26 | 27 | ## Our Responsibilities 28 | 29 | Project maintainers are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any instances of unacceptable behavior. 30 | 31 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for behaviors that they deem inappropriate, threatening, offensive, or harmful. 32 | 33 | ## Scope 34 | 35 | This Code of Conduct applies when an individual is representing the project or its community both within project spaces and in public spaces. Examples of representing a project or community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 36 | 37 | ## Enforcement 38 | 39 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by first contacting the project team. Oversight of Adobe projects is handled by the Adobe Open Source Office, which has final say in any violations and enforcement of this Code of Conduct and can be reached at Grp-opensourceoffice@adobe.com. All complaints will be reviewed and investigated promptly and fairly. 40 | 41 | The project team must respect the privacy and security of the reporter of any incident. 42 | 43 | Project maintainers who do not follow or enforce the Code of Conduct may face temporary or permanent repercussions as determined by other members of the project's leadership or the Adobe Open Source Office. 44 | 45 | ## Enforcement Guidelines 46 | 47 | Project maintainers will follow these Community Impact Guidelines in determining the consequences for any action they deem to be in violation of this Code of Conduct: 48 | 49 | **1. Correction** 50 | 51 | Community Impact: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. 52 | 53 | Consequence: A private, written warning from project maintainers describing the violation and why the behavior was unacceptable. A public apology may be requested from the violator before any further involvement in the project by violator. 54 | 55 | **2. Warning** 56 | 57 | Community Impact: A relatively minor violation through a single incident or series of actions. 58 | 59 | Consequence: A written warning from project maintainers that includes stated consequences for continued unacceptable behavior. Violator must refrain from interacting with the people involved for a specified period of time as determined by the project maintainers, including, but not limited to, unsolicited interaction with those enforcing the Code of Conduct through channels such as community spaces and social media. Continued violations may lead to a temporary or permanent ban. 60 | 61 | **3. Temporary Ban** 62 | 63 | Community Impact: A more serious violation of community standards, including sustained unacceptable behavior. 64 | 65 | Consequence: A temporary ban from any interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Failure to comply with the temporary ban may lead to a permanent ban. 66 | 67 | **4. Permanent Ban** 68 | 69 | Community Impact: Demonstrating a consistent pattern of violation of community standards or an egregious violation of community standards, including, but not limited to, sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. 70 | 71 | Consequence: A permanent ban from any interaction with the community. 72 | 73 | ## Attribution 74 | 75 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, 76 | available at [http://contributor-covenant.org/version/2/1][version] 77 | 78 | [homepage]: http://contributor-covenant.org 79 | [version]: http://contributor-covenant.org/version/2/1 80 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright 2022 Adobe. All rights reserved. 2 | 3 | Adobe holds the copyright for all the files found in this repository. 4 | 5 | See the LICENSE file for licensing information. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2013-2018 Docker, Inc. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | https://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.pickle -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Adobe Experience Platform API made for humans 2 | 3 | This repository will document the AEP wrapper on python. 4 | It is focusing on helping accessing the different endpoints of Adobe Experience Platform API. 5 | Documentation on the different endpoints can be found here : [AEP API doc](https://www.adobe.io/apis/experienceplatform/home/api-reference.html) 6 | The wrapper is currently named **aepp**, it stands for Adobe Experience Platform Python. 7 | 8 | ## Installation 9 | 10 | You can install the module directly from a pypi command: 11 | 12 | ```shell 13 | pip install aepp 14 | ``` 15 | 16 | The version of the wrapper can be seen by the following command (once loaded): 17 | 18 | ```python 19 | import aepp 20 | aepp.__version__ 21 | 22 | ``` 23 | 24 | **Consider upgrading regulary** 25 | 26 | ```shell 27 | pip install aepp --upgrade 28 | ``` 29 | 30 | **NOTE TO PYTHON 3.10.X** 31 | 32 | At the moment, not all packages are supported on python 3.10.X, therefore, please use this module with a python 3.9.X version.\ 33 | You can use pyenv to generate a local environment if required. 34 | 35 | ## Getting Started 36 | 37 | In order to get started, I have compiled a guide to help you initialize this module and what is required. 38 | You can find this documentation [here](./docs/getting-started.md) 39 | 40 | ## AEPP docs 41 | 42 | At the moment the current wrapper is containing the following sub modules: 43 | 44 | * [schema](./docs/schema.md) 45 | * [SchemaManager](./docs/schemaManager.md) 46 | * [FieldGroupManager](./docs/fieldGroupManager.md) 47 | * [DataTypeManager](./docs/dataTypeManager.md) 48 | * [ClassManager](./docs/classManager.md) 49 | * [queryservice](./docs/queryservice.md) (see note below for Interactive Queries) 50 | * [identity](./docs/identity.md) 51 | * [sandboxes](./docs/sandboxes.md) 52 | * [dataaccess](./docs/dataaccess.md) 53 | * [catalog](./docs/catalog.md) 54 | * [customerprofile](./docs/customerprofile.md) 55 | * [segmentation](./docs/segmentation.md) 56 | * [dataprep](./docs/dataprep.md) 57 | * [flowservice](./docs/flowservice.md) 58 | * [policy](./docs/policy.md) 59 | * [datasets](./docs/datasets.md) 60 | * [ingestion](./docs/ingestion.md) 61 | * [destination Authoring](./docs/destination.md) 62 | * [destination Instance](./docs/destinationinstanceservice.md) 63 | * [observability](./docs/observability.md) 64 | * [accesscontrol](./docs/accesscontrol.md) 65 | * [privacyservice](./docs/privacyservice.md) (see note below) 66 | * [data hygiene](./docs/hygiene.md) 67 | * [edge](./docs/edge.md) 68 | * [som](./docs/som.md) (see note below) 69 | * [synchronizer](./docs/synchronizer.md)(BETA) 70 | 71 | Last but not least, the core methods are described here: [main](./docs/main.md) 72 | 73 | ## Special classes 74 | 75 | The wrapper is having a class in all submodule in order to connect to the different service APIs.\ 76 | In addition to that, there are other classes that are provided in order to help you working with the API. 77 | 78 | ### Simple Object Manager 79 | 80 | In order to simplify the management of objects via python, especially when dealing with XDM object, we provide an abstraction that aim to simplify the manipulation of complex objects.\ 81 | The Simple Object Manager (SOM) is aiming at supporting the creation, manipulation or analysis of XDM messages.\ 82 | You can find all information about the methods available of that class (`Som`) in the related documentation. 83 | 84 | The Simple Object Manager documentation is located here: [SOM documentation](./docs/som.md) 85 | 86 | ### InteractiveQuery and InteractiveQuery2 classes 87 | 88 | These classes are implemented in the `queryservice` modulebased on the `pyGreSQL` and `psycopg2` module for python.\ 89 | It provides you the capability to realize query directly from your local Jupyter notebook and returns a dataframe. 90 | In order to use these classes, you would need to install these module and a PSQL server. 91 | On top of that, you would need to the psql server accessible in the environment path. 92 | 93 | ### SchemaManager, FieldGroupManager and DataTypeManager 94 | 95 | Since version 0.3.9, these classes are available from their respective modules, previously they were available from the `schema` module and alloy you to handle the different elements of the schema definition.\ 96 | You can use them to extract information on your schema definition. 97 | 98 | ### FlowManager 99 | 100 | The FlowManager is part of the `flowservice` module and allows you to group every aspect of a flow in a single class and simplify the search for the relationship between `sourceFlows`, `targetFlow` and the main flow elements. 101 | 102 | ### PrivacyService module 103 | 104 | The privacy service module is part of the AEP python wrapper (`aepp`) but requires a different JWT connection in console.adobe.io. 105 | Be careful that your JWT connection has the correct setup to access this API endpoints. 106 | 107 | ## Releases 108 | 109 | Release notes are accessible [here](./docs/releases.md). 110 | -------------------------------------------------------------------------------- /RELEASING_AEPP.md: -------------------------------------------------------------------------------- 1 | ## Publish new version of aepp 2 | We are currently using PyPI to for storing and distributing aepp packages, you can find the aepp project in https://pypi.org/project/aepp/. In this document, we will go through the steps for publishing new version of aepp. 3 | 4 | 1. Contact the owner @Julien Piccini to add your PyPI account as the collaborator for the aepp project 5 | 2. Set up your api token and save it in your local following the step "To make an API token" in https://pypi.org/help/#apitoken 6 | 3. Update the version in aepp/__version__.py and make necessary changes in setup.py under the project root if your changes require dependency update for aepp 7 | 4. Add a quick note about what has been changed in the new version you plan to publish in aepp/docs/releases.md 8 | 5. Install build and twine in python by running the following command in case you don't have them installed before 9 | ```shell 10 | pip install build 11 | pip install twine 12 | ``` 13 | 6. Run the command below to generate two files in a dist folder generated under the project root 14 | ```shell 15 | python -m build 16 | ``` 17 | ![release-build.png](docs%2Frelease-build.png) 18 | 7. Upload the two files in dist folder by running the command below 19 | ```shell 20 | python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/* 21 | ``` 22 | When prompted for username, enter `__token__` as username, and use the api token generated in step 2 as password 23 | ![release-upload.png](docs%2Frelease-upload.png) 24 | 8. You can find the latest published aepp from https://pypi.org/project/aepp/#history -------------------------------------------------------------------------------- /aepp/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | # Internal Library 12 | from aepp import config 13 | from aepp import connector 14 | from .configs import * 15 | from .__version__ import __version__ 16 | from typing import Union 17 | 18 | ## other libraries 19 | from copy import deepcopy 20 | from pathlib import Path 21 | import json 22 | 23 | connection = None 24 | 25 | 26 | def home(product: str = None, limit: int = 50): 27 | """ 28 | Return the IMS Organization setup and the container existing for the organization 29 | Arguments: 30 | product : OPTIONAL : specify one or more product contexts for which to return containers. If absent, containers for all contexts that you have rights to will be returned. The product parameter can be repeated for multiple contexts. An example of this parameter is product=acp 31 | limit : OPTIONAL : Optional limit on number of results returned (default = 50). 32 | """ 33 | global connection 34 | if connection is None: 35 | connection = connector.AdobeRequest( 36 | config_object=config.config_object, header=config.header 37 | ) 38 | endpoint = config.endpoints["global"] + "/data/core/xcore/" 39 | params = {"product": product, "limit": limit} 40 | myHeader = deepcopy(connection.header) 41 | myHeader["Accept"] = "application/vnd.adobe.platform.xcore.home.hal+json" 42 | res = connection.getData(endpoint, params=params, headers=myHeader) 43 | return res 44 | 45 | 46 | def getPlatformEvents( 47 | limit: int = 50, n_results: Union[int, str] = "inf", prop: str = None, **kwargs 48 | ) -> dict: 49 | """ 50 | Timestamped records of observed activities in Platform. The API allows you to query events over the last 90 days and create export requests. 51 | Arguments: 52 | limit : OPTIONAL : Number of events to retrieve per request (50 by default) 53 | n_results : OPTIONAL : Number of total event to retrieve per request. 54 | prop : OPTIONAL : An array that contains one or more of a comma-separated list of properties (prop="action==create,assetType==Sandbox") 55 | If you want to filter results using multiple values for a single filter, pass in a comma-separated list of values. (prop="action==create,update") 56 | """ 57 | global connection 58 | if connection is None: 59 | connection = connector.AdobeRequest( 60 | config_object=config.config_object, header=config.header 61 | ) 62 | endpoint = "https://platform.adobe.io/data/foundation/audit/events" 63 | params = {"limit": limit} 64 | if prop is not None: 65 | params["property"] = prop 66 | # myHeader = deepcopy(connection.header) 67 | lastPage = False 68 | data = list() 69 | while lastPage != True: 70 | res = connection.getData(endpoint, params=params) 71 | data += res.get("_embedded", {}).get("events", []) 72 | nextPage = res.get("_links", {}).get("next", {}).get('href','') 73 | if float(len(data)) >= float(n_results): 74 | lastPage = True 75 | if nextPage == "" and lastPage != True: 76 | lastPage = True 77 | else: 78 | start = nextPage.split("start=")[1].split("&")[0] 79 | queryId = nextPage.split("queryId=")[1].split("&")[0] 80 | params["queryId"] = queryId 81 | params["start"] = start 82 | return data 83 | 84 | 85 | def saveFile( 86 | module: str = None, 87 | file: object = None, 88 | filename: str = None, 89 | type_file: str = "json", 90 | encoding: str = "utf-8", 91 | ): 92 | """ 93 | Save the file in the approriate folder depending on the module sending the information. 94 | Arguments: 95 | module: REQUIRED: Module requesting the save file. 96 | file: REQUIRED: an object containing the file to save. 97 | filename: REQUIRED: the filename to be used. 98 | type_file: REQUIRED: the type of file to be saveed(default: json) 99 | encoding : OPTIONAL : encoding used to write the file. 100 | """ 101 | if module is None: 102 | raise ValueError("Require the module to create a folder") 103 | if file is None or filename is None: 104 | raise ValueError("Require a object for file and a name for the file") 105 | here = Path(Path.cwd()) 106 | folder = module.capitalize() 107 | new_location = Path.joinpath(here, folder) 108 | if new_location.exists() == False: 109 | new_location.mkdir() 110 | if type_file == "json": 111 | filename = f"{filename}.json" 112 | complete_path = Path.joinpath(new_location, filename) 113 | with open(complete_path, "w", encoding=encoding) as f: 114 | f.write(json.dumps(file, indent=4)) 115 | else: 116 | filename = f"{filename}.txt" 117 | complete_path = Path.joinpath(new_location, filename) 118 | with open(complete_path, "w", encoding=encoding) as f: 119 | f.write(file) 120 | -------------------------------------------------------------------------------- /aepp/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.3.11-2" 2 | -------------------------------------------------------------------------------- /aepp/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | import aepp 12 | 13 | config_object = { 14 | "org_id": "", 15 | "client_id": "", 16 | "tech_id": "", 17 | "pathToKey": "", 18 | "scopes": "", 19 | "secret": "", 20 | "date_limit": 0, 21 | "sandbox": "", 22 | "environment": "", 23 | "token": "", 24 | "jwtTokenEndpoint": "", 25 | "oauthTokenEndpointV1": "", 26 | "oauthTokenEndpointV2": "", 27 | "imsEndpoint": "" 28 | } 29 | 30 | header = {"Accept": "application/json", 31 | "Content-Type": "application/json", 32 | "Authorization": "", 33 | "x-api-key": config_object["client_id"], 34 | "x-gw-ims-org-id": config_object["org_id"], 35 | "x-sandbox-name": "prod" 36 | } 37 | 38 | # endpoints 39 | endpoints = { 40 | # global endpoint is https://platform.adobe.io in prod, otherwise https://platform-$ENV.adobe.io 41 | "global": "", 42 | "schemas": "/data/foundation/schemaregistry", 43 | "query": "/data/foundation/query", 44 | "catalog": "/data/foundation/catalog", 45 | "segmentation": "/data/core/ups", 46 | "export": "/data/foundation/export", 47 | "identity": "/data/core/", 48 | "sandboxes": "/data/foundation/sandbox-management", 49 | "sandboxTooling" : "/data/foundation/exim", 50 | "sensei": "/data/sensei", 51 | "access": "/data/foundation/access-control", 52 | "flow": "/data/foundation/flowservice", 53 | "privacy": "/data/core/privacy", 54 | "dataaccess": "/data/foundation/export", 55 | "mapping": "/data/foundation/conversion", 56 | "policy": "/data/foundation/dulepolicy", 57 | "dataset": "/data/foundation/dataset", 58 | "ingestion": "/data/foundation/import", 59 | "observability": "/data/infrastructure/observability/insights", 60 | "destinationAuthoring": "/data/core/activation/authoring", 61 | "destinationInstance" : "/data/core/activation/disflowprovider", 62 | "hygiene" : "/data/core/hygiene", 63 | "tags":"/unifiedtags", 64 | "folders":"/unifiedfolders", 65 | "streaming": { 66 | "inlet": "", 67 | "collection": "https://dcs.adobedc.net" 68 | }, 69 | "audit": "/data/foundation" 70 | } -------------------------------------------------------------------------------- /aepp/datasets.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | import aepp 12 | from aepp import connector 13 | from copy import deepcopy 14 | import logging 15 | from typing import Union 16 | from .configs import ConnectObject 17 | import json 18 | 19 | 20 | class DataSets: 21 | """ 22 | Class that provides methods to manage labels of datasets. 23 | A complete documentation ca be found here: 24 | https://www.adobe.io/apis/experienceplatform/home/api-reference.html#!acpdr/swagger-specs/dataset-service.yaml 25 | """ 26 | 27 | ## logging capability 28 | loggingEnabled = False 29 | logger = None 30 | 31 | REFERENCE_LABELS_CREATION = { 32 | "labels": [["C1", "C2"]], 33 | "optionalLabels": [ 34 | { 35 | "option": { 36 | "id": "https://ns.adobe.com/{TENANT_ID}/schemas/{SCHEMA_ID}", 37 | "contentType": "application/vnd.adobe.xed-full+json;version=1", 38 | "schemaPath": "/properties/repositoryCreatedBy", 39 | }, 40 | "labels": [["S1", "S2"]], 41 | } 42 | ], 43 | } 44 | 45 | def __init__( 46 | self, 47 | config: Union[dict,ConnectObject] = aepp.config.config_object, 48 | header: dict = aepp.config.header, 49 | loggingObject: dict = None, 50 | **kwargs, 51 | ): 52 | """ 53 | Instantiate the DataSet class. 54 | Arguments: 55 | config : OPTIONAL : config object in the config module. (DO NOT MODIFY) 56 | header : OPTIONAL : header object in the config module. (DO NOT MODIFY) 57 | loggingObject : OPTIONAL : logging object to log messages. 58 | Additional kwargs will update the header. 59 | """ 60 | if loggingObject is not None and sorted( 61 | ["level", "stream", "format", "filename", "file"] 62 | ) == sorted(list(loggingObject.keys())): 63 | self.loggingEnabled = True 64 | self.logger = logging.getLogger(f"{__name__}") 65 | self.logger.setLevel(loggingObject["level"]) 66 | if type(loggingObject["format"]) == str: 67 | formatter = logging.Formatter(loggingObject["format"]) 68 | elif type(loggingObject["format"]) == logging.Formatter: 69 | formatter = loggingObject["format"] 70 | if loggingObject["file"]: 71 | fileHandler = logging.FileHandler(loggingObject["filename"]) 72 | fileHandler.setFormatter(formatter) 73 | self.logger.addHandler(fileHandler) 74 | if loggingObject["stream"]: 75 | streamHandler = logging.StreamHandler() 76 | streamHandler.setFormatter(formatter) 77 | self.logger.addHandler(streamHandler) 78 | if type(config) == dict: ## Supporting either default setup or passing a ConnectObject 79 | config = config 80 | elif type(config) == ConnectObject: 81 | header = config.getConfigHeader() 82 | config = config.getConfigObject() 83 | self.connector = connector.AdobeRequest( 84 | config=config, 85 | header=header, 86 | loggingEnabled=self.loggingEnabled, 87 | logger=self.logger, 88 | ) 89 | self.header = self.connector.header 90 | self.header.update(**kwargs) 91 | if kwargs.get('sandbox',None) is not None: ## supporting sandbox setup on class instanciation 92 | self.sandbox = kwargs.get('sandbox') 93 | self.connector.config["sandbox"] = kwargs.get('sandbox') 94 | self.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 95 | self.connector.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 96 | else: 97 | self.sandbox = self.connector.config["sandbox"] 98 | self.endpoint = ( 99 | aepp.config.endpoints["global"] + aepp.config.endpoints["dataset"] 100 | ) 101 | 102 | def __str__(self): 103 | return json.dumps({'class':'DataSets','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 104 | 105 | def __repr__(self): 106 | return json.dumps({'class':'DataSets','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 107 | 108 | def getLabelSchemaTests(self, dataSetId: str = None) -> dict: 109 | """ 110 | Return the labels assigned to a dataSet 111 | Argument: 112 | dataSetId : REQUIRED : the dataSet ID to retrieve the labels 113 | """ 114 | if dataSetId is None: 115 | raise ValueError("Require a dataSet ID") 116 | if self.loggingEnabled: 117 | self.logger.debug(f"Starting getLabels") 118 | path = f"/datasets/{dataSetId}/labels" 119 | res = self.connector.getData(self.endpoint + path) 120 | return res 121 | 122 | def headLabels(self, dataSetId: str = None) -> dict: 123 | """ 124 | Return the head assigned to a dataSet 125 | Argument: 126 | dataSetId : REQUIRED : the dataSet ID to retrieve the head data 127 | """ 128 | if dataSetId is None: 129 | raise ValueError("Require a dataSet ID") 130 | if self.loggingEnabled: 131 | self.logger.debug(f"Starting headLabels") 132 | path = f"/datasets/{dataSetId}/labels" 133 | res = self.connector.headData(self.endpoint + path) 134 | return res 135 | 136 | def deleteLabels(self, dataSetId: str = None, ifMatch: str = None) -> dict: 137 | """ 138 | Delete the labels of a dataset. 139 | Arguments: 140 | dataSetId : REQUIRED : The dataset ID to delete the labels for. 141 | ifMatch : REQUIRED : the value is from the header etag of the headLabels. (use the headLabels method) 142 | """ 143 | if dataSetId is None: 144 | raise ValueError("Require a dataSet ID") 145 | if ifMatch is None: 146 | raise ValueError("Require the ifMatch parameter") 147 | if self.loggingEnabled: 148 | self.logger.debug(f"Starting deleteLabels") 149 | path = f"/datasets/{dataSetId}/labels" 150 | privateHeader = deepcopy(self.header) 151 | privateHeader["If-Match"] = ifMatch 152 | res = self.connector.deleteData(self.endpoint + path, headers=privateHeader) 153 | return res 154 | 155 | def createLabels(self, dataSetId: str = None, data: dict = None) -> dict: 156 | """ 157 | Assign labels to a dataset. 158 | Arguments: 159 | dataSetId : REQUIRED : The dataset ID to delete the labels for. 160 | data : REQUIRED : Dictionary setting the labels to be added. 161 | more info https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Datasets/postDatasetLabels 162 | """ 163 | if dataSetId is None: 164 | raise ValueError("Require a dataSet ID") 165 | if data is None or type(data) != dict: 166 | raise ValueError("Require a dictionary to pass labels") 167 | if self.loggingEnabled: 168 | self.logger.debug(f"Starting createLabels") 169 | path = f"/datasets/{dataSetId}/labels" 170 | res = self.connector.postData(self.endpoint + path, data=data) 171 | return res 172 | 173 | def updateLabels( 174 | self, dataSetId: str = None, data: dict = None, ifMatch: str = None 175 | ) -> dict: 176 | """ 177 | Update the labels (PUT method) 178 | dataSetId : REQUIRED : The dataset ID to delete the labels for. 179 | data : REQUIRED : Dictionary setting the labels to be added. 180 | more info https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Datasets/postDatasetLabels 181 | ifMatch : REQUIRED : the value is from the header etag of the headLabels.(use the headLabels method) 182 | """ 183 | if dataSetId is None: 184 | raise ValueError("Require a dataSet ID") 185 | if data is None or type(data) != dict: 186 | raise ValueError("Require a dictionary to pass labels") 187 | if ifMatch is None: 188 | raise ValueError("Require the ifMatch parameter") 189 | if self.loggingEnabled: 190 | self.logger.debug(f"Starting updateLabels") 191 | path = f"/datasets/{dataSetId}/labels" 192 | privateHeader = deepcopy(self.header) 193 | privateHeader["If-Match"] = ifMatch 194 | res = self.connector.putData( 195 | self.endpoint + path, data=data, headers=privateHeader 196 | ) 197 | return res 198 | -------------------------------------------------------------------------------- /aepp/destinationinstanceservice.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | # Internal Library 12 | import aepp 13 | from aepp import connector 14 | from typing import Union 15 | import logging 16 | from .configs import ConnectObject 17 | import json 18 | 19 | class DestinationInstanceService: 20 | loggingEnabled = False 21 | logger = None 22 | """ 23 | This class is referring to Destination Instance Service capability for AEP. 24 | """ 25 | def __init__(self, 26 | config: Union[dict,ConnectObject] = aepp.config.config_object, 27 | header: dict = aepp.config.header, 28 | loggingObject: dict = None, 29 | **kwargs,): 30 | """ 31 | Instantiating the class for destination instance service 32 | 33 | Arguments: 34 | loggingObject : OPTIONAL : logging object to log messages. 35 | config : OPTIONAL : config object in the config module. 36 | header : OPTIONAL : header object in the config module. 37 | possible kwargs: 38 | """ 39 | if loggingObject is not None and sorted( 40 | ["level", "stream", "format", "filename", "file"] 41 | ) == sorted(list(loggingObject.keys())): 42 | self.loggingEnabled = True 43 | self.logger = logging.getLogger(f"{__name__}") 44 | self.logger.setLevel(loggingObject["level"]) 45 | if type(loggingObject["format"]) == str: 46 | formatter = logging.Formatter(loggingObject["format"]) 47 | elif type(loggingObject["format"]) == logging.Formatter: 48 | formatter = loggingObject["format"] 49 | if loggingObject["file"]: 50 | fileHandler = logging.FileHandler(loggingObject["filename"]) 51 | fileHandler.setFormatter(formatter) 52 | self.logger.addHandler(fileHandler) 53 | if loggingObject["stream"]: 54 | streamHandler = logging.StreamHandler() 55 | streamHandler.setFormatter(formatter) 56 | self.logger.addHandler(streamHandler) 57 | if type(config) == dict: ## Supporting either default setup or passing a ConnectObject 58 | config = config 59 | elif type(config) == ConnectObject: 60 | header = config.getConfigHeader() 61 | config = config.getConfigObject() 62 | self.connector = connector.AdobeRequest( 63 | config=config, 64 | header=header, 65 | loggingEnabled=self.loggingEnabled, 66 | logger=self.logger, 67 | ) 68 | self.header = self.connector.header 69 | # self.header.update({"Accept": "application/json"}) 70 | self.header.update(**kwargs) 71 | if kwargs.get('sandbox',None) is not None: ## supporting sandbox setup on class instantiation 72 | self.sandbox = kwargs.get('sandbox') 73 | self.connector.config["sandbox"] = kwargs.get('sandbox') 74 | self.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 75 | self.connector.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 76 | else: 77 | self.sandbox = self.connector.config["sandbox"] 78 | self.endpoint = aepp.config.endpoints["global"] + aepp.config.endpoints["destinationInstance"] 79 | 80 | def __str__(self): 81 | return json.dumps({'class':'DestinationInstanceService','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 82 | 83 | def __repr__(self): 84 | return json.dumps({'class':'DestinationInstanceService','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 85 | 86 | def createAdHocDatasetExport(self, flowIdToDatasetIds: dict = None)->dict: 87 | """ 88 | Create an Adhoc Request based on the flowId and the datasetId passed in argument. 89 | Arguments: 90 | flowIdToDatasetIds : REQUIRED : dict containing the definition of flowId to datasetIds 91 | """ 92 | if self.loggingEnabled: 93 | self.logger.debug(f"Starting creating adhoc dataset export") 94 | if flowIdToDatasetIds is None or type(flowIdToDatasetIds) != dict: 95 | raise Exception("Require a dict for defining the flowId to datasetIds mapping") 96 | activationInfo = {'activationInfo': {'destinations': []}}; 97 | for flowId, datasetIds in flowIdToDatasetIds.items(): 98 | destination = {'flowId': flowId, 'datasets': []} 99 | for datasetId in datasetIds: 100 | dataset = {'id': datasetId} 101 | destination['datasets'].append(dataset) 102 | activationInfo['activationInfo']['destinations'].append(destination) 103 | self.header.update({"Accept":"application/vnd.adobe.adhoc.dataset.activation+json; version=1"}) 104 | path = "/adhocrun" 105 | res = self.connector.postData(self.endpoint + path, data=activationInfo) 106 | return res 107 | -------------------------------------------------------------------------------- /aepp/observability.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | import aepp 12 | from aepp import connector 13 | import pandas as pd 14 | import logging 15 | from typing import Union 16 | from .configs import ConnectObject 17 | import json 18 | 19 | class Observability: 20 | """ 21 | A class that presents the different methods available on the Observability API from AEP. 22 | A complete documentation of the methods can be found here: 23 | https://www.adobe.io/apis/experienceplatform/home/api-reference.html#!acpdr/swagger-specs/observability-insights.yaml 24 | """ 25 | 26 | ## logging capability 27 | loggingEnabled = False 28 | logger = None 29 | 30 | def __init__( 31 | self, 32 | config: Union[dict,ConnectObject] = aepp.config.config_object, 33 | header: dict = aepp.config.header, 34 | loggingObject: dict = None, 35 | **kwargs, 36 | ) -> None: 37 | """ 38 | Instanciate the observability API methods class. 39 | Arguments: 40 | loggingObject : OPTIONAL : logging object to log messages. 41 | config : OPTIONAL : config object in the config module. (DO NOT MODIFY) 42 | header : OPTIONAL : header object in the config module. (DO NOT MODIFY) 43 | """ 44 | if loggingObject is not None and sorted( 45 | ["level", "stream", "format", "filename", "file"] 46 | ) == sorted(list(loggingObject.keys())): 47 | self.loggingEnabled = True 48 | self.logger = logging.getLogger(f"{__name__}") 49 | self.logger.setLevel(loggingObject["level"]) 50 | if type(loggingObject["format"]) == str: 51 | formatter = logging.Formatter(loggingObject["format"]) 52 | elif type(loggingObject["format"]) == logging.Formatter: 53 | formatter = loggingObject["format"] 54 | if loggingObject["file"]: 55 | fileHandler = logging.FileHandler(loggingObject["filename"]) 56 | fileHandler.setFormatter(formatter) 57 | self.logger.addHandler(fileHandler) 58 | if loggingObject["stream"]: 59 | streamHandler = logging.StreamHandler() 60 | streamHandler.setFormatter(formatter) 61 | self.logger.addHandler(streamHandler) 62 | if type(config) == dict: ## Supporting either default setup or passing a ConnectObject 63 | config = config 64 | elif type(config) == ConnectObject: 65 | header = config.getConfigHeader() 66 | config = config.getConfigObject() 67 | self.connector = connector.AdobeRequest( 68 | config=config, 69 | header=header, 70 | loggingEnabled=self.loggingEnabled, 71 | logger=self.logger, 72 | ) 73 | self.header = self.connector.header 74 | self.header.update(**kwargs) 75 | if kwargs.get('sandbox',None) is not None: ## supporting sandbox setup on class instanciation 76 | self.sandbox = kwargs.get('sandbox') 77 | self.connector.config["sandbox"] = kwargs.get('sandbox') 78 | self.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 79 | self.connector.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 80 | else: 81 | self.sandbox = self.connector.config["sandbox"] 82 | self.endpoint = ( 83 | aepp.config.endpoints["global"] + aepp.config.endpoints["observability"] 84 | ) 85 | self.POST_METRICS = { 86 | "start": "2020-07-14T00:00:00.000Z", 87 | "end": "2020-07-22T00:00:00.000Z", 88 | "granularity": "day", 89 | "metrics": [ 90 | { 91 | "name": "timeseries.ingestion.dataset.recordsuccess.count", 92 | "filters": [ 93 | { 94 | "name": "dataSetId", 95 | "value": "5edcfb2fbb642119194c7d94|5eddb21420f516191b7a8dad", 96 | "groupBy": True, 97 | } 98 | ], 99 | "aggregator": "sum", 100 | "downsample": "sum", 101 | }, 102 | { 103 | "name": "timeseries.ingestion.dataset.dailysize", 104 | "filters": [ 105 | { 106 | "name": "dataSetId", 107 | "value": "5eddb21420f516191b7a8dad", 108 | "groupBy": False, 109 | } 110 | ], 111 | "aggregator": "sum", 112 | "downsample": "sum", 113 | }, 114 | ], 115 | } 116 | self._loadREFERENCES() 117 | 118 | def _loadREFERENCES(self): 119 | """ 120 | Load document as attributes if possible 121 | """ 122 | if self.loggingEnabled: 123 | self.logger.debug(f"Loading references") 124 | try: 125 | import importlib.resources as pkg_resources 126 | 127 | pathIdentity = pkg_resources.path("aepp", "observability_identity.pickle") 128 | pathIngestion = pkg_resources.path("aepp", "observability_ingestion.pickle") 129 | pathGDPR = pkg_resources.path("aepp", "observability_gdpr.pickle") 130 | pathQS = pkg_resources.path("aepp", "observability_queryService.pickle") 131 | pathRLTime = pkg_resources.path("aepp", "observability_realTime.pickle") 132 | except ImportError: 133 | # Try backported to PY<37 with pkg_resources. 134 | if self.loggingEnabled: 135 | self.logger.debug(f"Loading references - ImportError - 2nd try") 136 | try: 137 | import pkg_resources 138 | 139 | pathIdentity = pkg_resources.resource_filename( 140 | "aepp", "observability_identity.pickle" 141 | ) 142 | pathIngestion = pkg_resources.resource_filename( 143 | "aepp", "observability_ingestion.pickle" 144 | ) 145 | pathGDPR = pkg_resources.resource_filename( 146 | "aepp", "observability_gdpr.pickle" 147 | ) 148 | pathQS = pkg_resources.resource_filename( 149 | "aepp", "observability_queryService.pickle" 150 | ) 151 | pathRLTime = pkg_resources.resource_filename( 152 | "aepp", "observability_realTime.pickle" 153 | ) 154 | except: 155 | print("no supported files") 156 | if self.loggingEnabled: 157 | self.logger.debug(f"Failed loading references") 158 | try: 159 | with pathIdentity as f: 160 | self.REFERENCE_IDENTITY = pd.read_pickle(f) 161 | self.REFERENCE_IDENTITY = self.REFERENCE_IDENTITY.style.set_properties( 162 | subset=["Insights metric"], **{"width": "100px"} 163 | ) 164 | with pathIngestion as f: 165 | self.REFERENCE_INGESTION = pd.read_pickle(f) 166 | self.REFERENCE_INGESTION = ( 167 | self.REFERENCE_INGESTION.style.set_properties( 168 | subset=["Insights metric"], **{"width": "100px"} 169 | ) 170 | ) 171 | with pathGDPR as f: 172 | self.REFERENCE_GDPR = pd.read_pickle(f) 173 | self.REFERENCE_GDPR = self.REFERENCE_GDPR.style.set_properties( 174 | subset=["Insights metric"], **{"width": "100px"} 175 | ) 176 | with pathRLTime as f: 177 | self.REFERENCE_REALTIME = pd.read_pickle(f) 178 | self.REFERENCE_REALTIME = self.REFERENCE_REALTIME.style.set_properties( 179 | subset=["Insights metric"], **{"width": "100px"} 180 | ) 181 | with pathQS as f: 182 | self.REFERENCE_QUERYSERVICE = pd.read_pickle(f) 183 | self.REFERENCE_QUERYSERVICE = ( 184 | self.REFERENCE_QUERYSERVICE.style.set_properties( 185 | subset=["Insights metric"], **{"width": "100px"} 186 | ) 187 | ) 188 | except: 189 | if self.loggingEnabled: 190 | self.logger.debug(f"Failed loading references - backup to None") 191 | self.REFERENCE_IDENTITY = None 192 | self.REFERENCE_INGESTION = None 193 | self.REFERENCE_QUERYSERVICE = None 194 | self.REFERENCE_GDPR = None 195 | self.REFERENCE_REALTIME = None 196 | 197 | def __str__(self): 198 | return json.dumps({'class':'Observability','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 199 | 200 | def __repr__(self): 201 | return json.dumps({'class':'Observability','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 202 | 203 | def createMetricsReport(self, data: dict = None) -> dict: 204 | """ 205 | Using the POST method to retrieve metrics specified in the data dictionary. 206 | Please use the different REFERENCES attributes to know which metrics are supported. 207 | You have a template for the data dictionary on the POST_METRICS attribute. 208 | Arguments: 209 | data : REQUIRED : The metrics requested in the report creation. 210 | You can use the POST_METRICS attribute to see a template. 211 | """ 212 | if self.loggingEnabled: 213 | self.logger.debug(f"Starting createMetricsReport") 214 | path = "/metrics" 215 | res = self.connector.postData(self.endpoint + path, data=data) 216 | return res 217 | -------------------------------------------------------------------------------- /aepp/observability_gdpr.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/aepp/observability_gdpr.pickle -------------------------------------------------------------------------------- /aepp/observability_identity.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/aepp/observability_identity.pickle -------------------------------------------------------------------------------- /aepp/observability_ingestion.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/aepp/observability_ingestion.pickle -------------------------------------------------------------------------------- /aepp/observability_queryService.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/aepp/observability_queryService.pickle -------------------------------------------------------------------------------- /aepp/observability_realTime.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/aepp/observability_realTime.pickle -------------------------------------------------------------------------------- /aepp/privacyservice.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | import aepp 12 | from aepp import connector 13 | import logging 14 | from typing import Union 15 | from .configs import ConnectObject 16 | import json 17 | 18 | class Privacy: 19 | """ 20 | Class to instanciate a Privacy API connection. Ensure you have the correct access within you Adobe IO connection. 21 | Information about that class can be found here : https://docs.adobe.com/content/help/en/experience-platform/privacy/api/privacy-jobs.html 22 | """ 23 | 24 | SAMPLE_PAYLOAD = { 25 | "companyContexts": [{"namespace": "imsOrgID", "value": "{IMS_ORG}"}], 26 | "users": [ 27 | { 28 | "key": "DavidSmith", 29 | "action": ["access"], 30 | "userIDs": [ 31 | { 32 | "namespace": "email", 33 | "value": "dsmith@acme.com", 34 | "type": "standard", 35 | }, 36 | { 37 | "namespace": "ECID", 38 | "type": "standard", 39 | "value": "443636576799758681021090721276", 40 | "isDeletedClientSide": False, 41 | }, 42 | ], 43 | }, 44 | { 45 | "key": "user12345", 46 | "action": ["access", "delete"], 47 | "userIDs": [ 48 | { 49 | "namespace": "email", 50 | "value": "ajones@acme.com", 51 | "type": "standard", 52 | }, 53 | { 54 | "namespace": "loyaltyAccount", 55 | "value": "12AD45FE30R29", 56 | "type": "integrationCode", 57 | }, 58 | ], 59 | }, 60 | ], 61 | "include": ["Analytics", "AudienceManager"], 62 | "expandIds": False, 63 | "priority": "normal", 64 | "analyticsDeleteMethod": "anonymize", 65 | "regulation": "ccpa", 66 | } 67 | ## logging capability 68 | loggingEnabled = False 69 | logger = None 70 | 71 | def __init__( 72 | self, 73 | privacyScope: bool = True, 74 | aepScope: bool = False, 75 | config: Union[dict,ConnectObject] = aepp.config.config_object, 76 | header: dict = aepp.config.header, 77 | loggingObject: dict = None, 78 | **kwargs, 79 | ) -> None: 80 | """ 81 | Instanciate the class for Privacy Service call. 82 | Arguments: 83 | privacyScope : REQUIRED : set the connection retrieved process with the Privacy JWT scope (default True). 84 | aepScope : OPTIONAL : set the connection retrieved process with the AEP JWT scope if set to True (default False). 85 | config : OPTIONAL : config object in the config module. 86 | header : OPTIONAL : header object in the config module. 87 | kwargs: 88 | kwargs will update the header 89 | """ 90 | if loggingObject is not None and sorted( 91 | ["level", "stream", "format", "filename", "file"] 92 | ) == sorted(list(loggingObject.keys())): 93 | self.loggingEnabled = True 94 | self.logger = logging.getLogger(f"{__name__}") 95 | self.logger.setLevel(loggingObject["level"]) 96 | if type(loggingObject["format"]) == str: 97 | formatter = logging.Formatter(loggingObject["format"]) 98 | elif type(loggingObject["format"]) == logging.Formatter: 99 | formatter = loggingObject["format"] 100 | if loggingObject["file"]: 101 | fileHandler = logging.FileHandler(loggingObject["filename"]) 102 | fileHandler.setFormatter(formatter) 103 | self.logger.addHandler(fileHandler) 104 | if loggingObject["stream"]: 105 | streamHandler = logging.StreamHandler() 106 | streamHandler.setFormatter(formatter) 107 | self.logger.addHandler(streamHandler) 108 | if type(config) == dict: ## Supporting either default setup or passing a ConnectObject 109 | config = config 110 | elif type(config) == ConnectObject: 111 | header = config.getConfigHeader() 112 | config = config.getConfigObject() 113 | self.connector = connector.AdobeRequest( 114 | config=config, 115 | header=header, 116 | aepScope=aepScope, 117 | privacyScope=privacyScope, 118 | loggingEnabled=self.loggingEnabled, 119 | logger=self.logger, 120 | ) 121 | self.header = self.connector.header 122 | self.header.update(**kwargs) 123 | self.endpoint = ( 124 | aepp.config.endpoints["global"] + aepp.config.endpoints["privacy"] 125 | ) 126 | 127 | def __str__(self): 128 | return json.dumps({'class':'PrivacyService','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 129 | 130 | def __repr__(self): 131 | return json.dumps({'class':'PrivacyService','sandbox':self.sandbox,'clientId':self.connector.config.get("client_id"),'orgId':self.connector.config.get("org_id")},indent=2) 132 | 133 | def getJobs( 134 | self, regulation: str = None, limit: int = 50, status: str = None, **kwargs 135 | ) -> dict: 136 | """ 137 | Returns the job that are being processed on Adobe. 138 | Arguments: 139 | regulation : REQUIRED : The privacy regulation to return jobs from. (gdpr, ccpa, pdpa_tha) 140 | limit : OPTIONAL : The number of jobs to return in the response body.(default 50) 141 | status : OPTIONAL : Filters jobs by processing status. (complete, processing, submitted, error) 142 | Possible kwargs: see documentation : https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Privacy_Jobs/fetchAllJobs 143 | """ 144 | if regulation is None: 145 | raise Exception("Required regulation parameter") 146 | if self.loggingEnabled: 147 | self.logger.debug(f"Starting getJobs") 148 | path = "/jobs" 149 | params = {"size": limit, "regulation": regulation} 150 | if status is not None: 151 | params["status"] = status 152 | if len(kwargs) > 0: 153 | for key in kwargs: 154 | params[key] = str(kwargs[key]) 155 | res = self.connector.getData( 156 | self.endpoint + path, params=params, headers=self.header 157 | ) 158 | return res 159 | 160 | def getJob(self, jobId: str = None) -> dict: 161 | """ 162 | Return a specific job by its job ID. 163 | Arguments: 164 | jobId : REQUIRED : the Job ID to fetch 165 | """ 166 | if jobId is None: 167 | raise Exception("Require a job ID") 168 | if self.loggingEnabled: 169 | self.logger.debug(f"Starting getJob") 170 | path = f"/jobs/{jobId}" 171 | res = self.connector.getData(self.endpoint + path, headers=self.header) 172 | return res 173 | 174 | def postJob(self, data: dict = None) -> dict: 175 | """ 176 | Return a specific job by its job ID. 177 | Arguments: 178 | data : REQUIRED : data to be send in order to start a job. 179 | You can use the SAMPLE_PAYLOAD to help you create the data. 180 | """ 181 | if data is None or type(data) != dict: 182 | raise Exception("Require a dictionary to be passed") 183 | if self.loggingEnabled: 184 | self.logger.debug(f"Starting postJob") 185 | path = "/jobs/" 186 | res = self.connector.postData( 187 | self.endpoint + path, data=data, headers=self.header 188 | ) 189 | return res 190 | 191 | def postChildJob(self, jobId: str = None, data: dict = None) -> dict: 192 | """ 193 | This is to add a job on a parent Job. 194 | Argument: 195 | jobId : REQUIRED : the Job ID to append the job to 196 | data : REQUIRED : dictionary of data to be added. 197 | """ 198 | if jobId is None: 199 | raise Exception("Require a job ID") 200 | if data is None or type(data) != dict: 201 | raise Exception("Require a dictionary to be passed") 202 | if self.loggingEnabled: 203 | self.logger.debug(f"Starting gpostChildJob") 204 | path = f"/jobs/{jobId}/child-job" 205 | res = self.connector.postData( 206 | self.endpoint + path, data=data, headers=self.header 207 | ) 208 | return res 209 | -------------------------------------------------------------------------------- /aepp/sensei.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | # internal library 12 | import aepp 13 | from aepp import connector 14 | import logging 15 | from copy import deepcopy 16 | from typing import Union 17 | from .configs import ConnectObject 18 | 19 | 20 | class Sensei: 21 | """ 22 | This module is based on the Sensei Machine Learning API from Adobe Experience Platform. 23 | You can find more documentation on the endpoints here : https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/ 24 | """ 25 | 26 | # logging capability 27 | loggingEnabled = False 28 | logger = None 29 | 30 | def __init__( 31 | self, 32 | config: Union[dict,ConnectObject] = aepp.config.config_object, 33 | header: dict = aepp.config.header, 34 | loggingObject: dict = None, 35 | **kwargs, 36 | ) -> None: 37 | """ 38 | Initialize the class with the config header used. 39 | Arguments: 40 | loggingObject : OPTIONAL : logging object to log messages. 41 | config : OPTIONAL : config object in the config module. 42 | header : OPTIONAL : header object in the config module. 43 | Additional kwargs will update the header. 44 | """ 45 | if loggingObject is not None and sorted( 46 | ["level", "stream", "format", "filename", "file"] 47 | ) == sorted(list(loggingObject.keys())): 48 | self.loggingEnabled = True 49 | self.logger = logging.getLogger(f"{__name__}") 50 | self.logger.setLevel(loggingObject["level"]) 51 | if type(loggingObject["format"]) == str: 52 | formatter = logging.Formatter(loggingObject["format"]) 53 | elif type(loggingObject["format"]) == logging.Formatter: 54 | formatter = loggingObject["format"] 55 | if loggingObject["file"]: 56 | fileHandler = logging.FileHandler(loggingObject["filename"]) 57 | fileHandler.setFormatter(formatter) 58 | self.logger.addHandler(fileHandler) 59 | if loggingObject["stream"]: 60 | streamHandler = logging.StreamHandler() 61 | streamHandler.setFormatter(formatter) 62 | self.logger.addHandler(streamHandler) 63 | if type(config) == dict: ## Supporting either default setup or passing a ConnectObject 64 | config = config 65 | elif type(config) == ConnectObject: 66 | header = config.getConfigHeader() 67 | config = config.getConfigObject() 68 | self.connector = connector.AdobeRequest( 69 | config=config, 70 | header=header, 71 | loggingEnabled=self.loggingEnabled, 72 | logger=self.logger, 73 | ) 74 | self.header = self.connector.header 75 | self.header[ 76 | "Accept" 77 | ] = "application/vnd.adobe.platform.sensei+json;profile=mlInstanceListing.v1.json" 78 | self.header.update(**kwargs) 79 | if kwargs.get('sandbox',None) is not None: ## supporting sandbox setup on class instanciation 80 | self.sandbox = kwargs.get('sandbox') 81 | self.connector.config["sandbox"] = kwargs.get('sandbox') 82 | self.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 83 | self.connector.header.update({"x-sandbox-name":kwargs.get('sandbox')}) 84 | else: 85 | self.sandbox = self.connector.config["sandbox"] 86 | self.endpoint = ( 87 | aepp.config.endpoints["global"] + aepp.config.endpoints["sensei"] 88 | ) 89 | 90 | def getEngines(self, limit: int = 25, **kwargs) -> list: 91 | """ 92 | Return the list of all engines. 93 | Arguments: 94 | limit : OPTIONAL : number of element per requests 95 | kwargs: 96 | property : filtering, example value "name==test." 97 | """ 98 | if self.loggingEnabled: 99 | self.logger.debug(f"Starting getEngines") 100 | path = "/engines" 101 | params = {"limit": limit} 102 | if kwargs.get("property", False) != False: 103 | params["property"] = kwargs.get("property", "") 104 | res = self.connector.getData( 105 | self.endpoint + path, headers=self.header, params=params 106 | ) 107 | data = res["children"] 108 | return data 109 | 110 | def getEngine(self, engineId: str = None) -> dict: 111 | """ 112 | return a specific engine information based on its id. 113 | Arguments: 114 | engineId : REQUIRED : the engineId to return. 115 | """ 116 | if engineId is None: 117 | raise Exception("require an engineId parameter") 118 | if self.loggingEnabled: 119 | self.logger.debug(f"Starting getEngine") 120 | path = f"/engines/{engineId}" 121 | res = self.connector.getData(self.endpoint + path, headers=self.header) 122 | return res 123 | 124 | def getDockerRegistery(self) -> dict: 125 | """ 126 | Return the docker registery information. 127 | """ 128 | if self.loggingEnabled: 129 | self.logger.debug(f"Starting getDockerRegistery") 130 | path = "/engines/dockerRegistry" 131 | res = self.connector.getData(self.endpoint + path, headers=self.header) 132 | return res 133 | 134 | def deleteEngine(self, engineId: str = None) -> str: 135 | """ 136 | Delete an engine based on the id passed. 137 | Arguments: 138 | engineId : REQUIRED : Engine ID to be deleted. 139 | """ 140 | if engineId is None: 141 | raise Exception("require an engineId parameter") 142 | if self.loggingEnabled: 143 | self.logger.debug(f"Starting deleteEngine") 144 | path = f"/engines/{engineId}" 145 | res = self.connector.deleteData(self.endpoint + path, headers=self.header) 146 | return res 147 | 148 | def getMLinstances(self, limit: int = 25) -> list: 149 | """ 150 | Return a list of all of the ml instance 151 | Arguments: 152 | limit : OPTIONAL : number of elements retrieved. 153 | """ 154 | if self.loggingEnabled: 155 | self.logger.debug(f"Starting getMLinstances") 156 | path = "/mlInstances" 157 | params = {"limit": limit} 158 | res = self.connector.getData( 159 | self.endpoint + path, headers=self.header, params=params 160 | ) 161 | data = res["children"] 162 | return data 163 | 164 | def createMLinstances( 165 | self, name: str = None, engineId: str = None, description: str = None 166 | ): 167 | """ 168 | Create a ML instance with the name and instanceId provided. 169 | Arguments: 170 | name : REQUIRED : name of the ML instance 171 | engineId : REQUIRED : engine attached to the ML instance 172 | description : OPTIONAL : description of the instance. 173 | """ 174 | if self.loggingEnabled: 175 | self.logger.debug(f"Starting createMLinstances") 176 | path = "/mlInstances" 177 | privateHeader = deepcopy(self.header) 178 | privateHeader[ 179 | "Content" 180 | ] = "application/vnd.adobe.platform.sensei+json;profile=mlInstanceListing.v1.json" 181 | if name is None and engineId is None: 182 | raise Exception("Requires a name and an egineId") 183 | body = {"name": name, "engineId": engineId, "description": description} 184 | res = self.connector.getData( 185 | self.endpoint + path, headers=privateHeader, data=body 186 | ) 187 | return res 188 | -------------------------------------------------------------------------------- /aepp/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | # internal library 12 | from configparser import ConfigParser 13 | 14 | class Utils: 15 | 16 | @staticmethod 17 | def check_if_exists(section, field_name, config_path): 18 | config = ConfigParser() 19 | config.read(config_path) 20 | return config.get(section, field_name) or None 21 | 22 | 23 | @staticmethod 24 | def save_field_in_config(section, field_name, value, config_path): 25 | config = ConfigParser() 26 | config.read(config_path) 27 | config.set(section, field_name, value) 28 | with open(config_path, "w") as configfile: 29 | config.write(configfile) -------------------------------------------------------------------------------- /docs/accesscontrol.md: -------------------------------------------------------------------------------- 1 | # Access Control module in aepp 2 | 3 | This documentation will provide you an overview on how to use the `accesscontrol` module and different methods supported by this module.\ 4 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/access-control/).\ 5 | Alternatively, you can use the docstring in the methods to have more information. 6 | 7 | ## Menu 8 | - [Access Control module in aepp](#access-control-module-in-aepp) 9 | - [Menu](#menu) 10 | - [Importing the module](#importing-the-module) 11 | - [The AccessControl Class](#the-accesscontrol-class) 12 | - [AccessControl methods](accesscontrol-methods) 13 | 14 | ## Importing the module 15 | 16 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 17 | 18 | To import the module you can use the import statement with the `destination` keyword. 19 | 20 | ```python 21 | import aepp 22 | aepp.importConfigFile('myConfig_file.json') 23 | 24 | from aepp import accesscontrol 25 | ``` 26 | 27 | The accesscontrol module provides a class that you can use to list out permissions, list out resource types, and list all the effective policies for a user on given resources within a sandbox.\ 28 | The following documentation will provide you with more information on its capabilities. 29 | 30 | ## The AccessControl class 31 | 32 | You can instantiate an `AccessControl` class by using these parameters. 33 | Arguments: 34 | * config : OPTIONAL : it could be the instance of the ConnectObject class (preferred) or a dictionary containing the config information. Default will take the latest configuration loaded. 35 | * header : OPTIONAL : header object in the config module. (DO NOT MODIFY) 36 | * loggingObject : OPTIONAL : logging object to log messages.\ 37 | kwargs : 38 | * header options that you want to append to the header. Such as {"Accept":"accepted-value"} 39 | 40 | 41 | ## AccessControl methods 42 | 43 | ### getPermissions 44 | List all available permission names and resource types. 45 | 46 | 47 | ### getEffectivePolicies 48 | List all effective policies for a user on given resources within a sandbox.\ 49 | Arguments: 50 | * listElements : REQUIRED : List of resource urls. Example url : /resource-types/{resourceName} or /permissions/{highLevelPermissionName}\ 51 | example: "/permissions/manage-dataset" "/resource-types/schema" "/permissions/manage-schemas" 52 | 53 | 54 | ### getRoles 55 | Return all existing roles in the Company. 56 | 57 | 58 | ### getRole 59 | Retrieve a specific role based on the ID.\ 60 | Arguments: 61 | * roleId : REQUIRED : Role ID to be retrieved 62 | 63 | 64 | ### deleteRole 65 | Delete a role based on its ID.\ 66 | Argument: 67 | * roleId : REQUIRED : The role ID to be deleted 68 | 69 | 70 | ### patchRole 71 | PATCH the role with the attribute passed.\ 72 | Attribute can have the following action "add" "replace" "remove".\ 73 | Arguments: 74 | * roleId : REQUIRED : The role ID to be updated 75 | * roleDict : REQUIRED : The change to the role 76 | 77 | 78 | ### putRole 79 | PUT the role with the new definition passed.\ 80 | As a PUT method, the old definition will be replaced by the new one.\ 81 | Arguments: 82 | * roleId : REQUIRED : The role ID to be updated 83 | * roleDict : REQUIRED : The change to the role\ 84 | example: 85 | ```JSON 86 | { 87 | "name": "Administrator Role", 88 | "description": "Role for administrator type of responsibilities and access.", 89 | "roleType": "user-defined" 90 | } 91 | ``` 92 | 93 | ### getSubjects 94 | Get the subjects attached to the role specified in the roleId.\ 95 | Arguments: 96 | * roleId : REQUIRED : The roleId for which the subjects should be extracted 97 | 98 | 99 | ### patchSubjects 100 | Manage the subjects attached to a specific role\ 101 | Arguments: 102 | * roleId : REQUIRED : The role ID to update 103 | * subjectId : REQUIRED : The subject ID to be updated 104 | * operation : REQUIRED : The operation could be either "add" "replace" "remove" 105 | 106 | 107 | ### getPolicies 108 | Returns all the policies applying in your organization 109 | 110 | 111 | ### getPolicy 112 | Returns a specific policy based on its ID.\ 113 | Arguments: 114 | * policyId : REQUIRED : The policy ID to be retrieved 115 | 116 | 117 | ### deletePolicy 118 | Delete a specific policy based on its ID. 119 | Arguments: 120 | * policyId : REQUIRED : The policy ID to be deleted 121 | 122 | 123 | ### createPolicy 124 | Create a policy based on the definition passed 125 | Arguments: 126 | * policyDef : REQUIRED : The policy definition requires 127 | 128 | 129 | ### putPolicy 130 | Replace the policyID provided by the new definition passed. \ 131 | Arguments: 132 | * policyId : REQUIRED : The policy ID to replaced 133 | * policyDef : REQUIRED : The new definition of the policy ID. 134 | 135 | 136 | ### patchPolicy 137 | Patch the policyID provided with the operation provided\ 138 | Arguments: 139 | * policyId : REQUIRED : The policy ID to be updated 140 | * operation : REQUIRED : The operation to realise ("add" "replace" "remove") 141 | * attribute : REQUIRED : The attribute to be updated. Ex : "/description" 142 | * value : REQUIRED : The new value to be used 143 | 144 | 145 | ### getProducts 146 | List all entitled products 147 | 148 | 149 | ### getPermissionCategories 150 | Retrieve the permissions categories for a specific product\ 151 | Arguments: 152 | * productId : REQUIRED : The product you are looking for 153 | 154 | 155 | ### getPermissionSets 156 | Retrieve the permissions set of the product ID you want to acces.\ 157 | Arguments: 158 | * productId : REQUIRED : The product ID permissions set you want to retrieve -------------------------------------------------------------------------------- /docs/add-privacy-service-api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/docs/add-privacy-service-api.png -------------------------------------------------------------------------------- /docs/classManager.md: -------------------------------------------------------------------------------- 1 | # ClassManager for AEP Schema 2 | 3 | This module is a custom module built by Adobe Consulting team in order to improve efficiency of the Class manipulation, documentation and analysis.\ 4 | The Field Group Manager is built on top of all of the existing [AEP Schema Registry API](https://developer.adobe.com/experience-platform-apis/references/schema-registry/), and [Schema class](./schema.md). 5 | 6 | It is important to understand that a Schema is built by aggregating different field groups within a class.\ 7 | Therefore, when modifying the schema, what happens in the back-end is often a modification of the Field Group. 8 | 9 | ## Menu 10 | - [Instantiation](#instantiation) 11 | - [Class Manager attrbites](#class-manager-attributes) 12 | - [Class Manager methods](#class-manager-methods) 13 | 14 | ## Instantiation 15 | 16 | The `ClassManager` is a class that can be instantiated with different parameters: 17 | 18 | * aepclass : OPTIONAL : If you wish to load an existing class. 19 | * title : OPTIONAL : If you wish to override or define the title of your class. 20 | * behavior : OPTIONAL : If you want to define which behavioral element it extends.\ 21 | Default: 'https://ns.adobe.com/xdm/data/record', possible values: 'https://ns.adobe.com/xdm/data/time-series','https://ns.adobe.com/xdm/data/adhoc' 22 | * schemaAPI : OPTIONAL : To connect to your sandbox, you can pass the instance of the `Schema` class you want. 23 | * config : OPTIONAL : Recommended to pass a [`ConnectObject` instance](./getting-started.md#the-connectinstance-parameter) if you did not pass the schemaAPI. That would ensure the usage of the correct sandbox. 24 | * description : OPTIONAL : If you want to add a description to your class 25 | 26 | In the end these different parameters offer you different options to use the Field Group Manager. 27 | 28 | 29 | ### 1 Connecting to an existing Field Group 30 | 31 | In this case, you can either the `schemaAPI` parameter or the `config` parameter.\ 32 | If this is your use-case, you can adapt the following code below: 33 | 34 | ```python 35 | import aepp 36 | from aepp import schema 37 | from aepp import classmanager 38 | 39 | mySandbox = aepp.importConfigFile('myconfig.json',sandbox='mysandbox',connectInstance=True) 40 | mySchemaInstance = schema.Schema(config=mySandbox) 41 | 42 | myClasses = mySchemaInstance.getClasses() 43 | 44 | singleClass = [cl for cl in myClasses if cl['title'] == 'mytitle'][0] 45 | ### singleFG will be the altId of that `titleOfFieldGroup` field group 46 | 47 | ## option 1 : via schemaAPI parameter 48 | fgManager = classmanager.ClassManager(singleFG,schemaAPI=mySchemaInstance) 49 | 50 | ## option 2 : via config parameter 51 | fgManager = classmanager.ClassManager(singleFG,config=mySandbox) 52 | 53 | ``` 54 | 55 | ### 2 Creating a new class from scratch 56 | 57 | In this case, we would still need to pass the configuration or the schema API instance.\ 58 | If this is your use-case, you can adapt the following code below: 59 | 60 | ```python 61 | import aepp 62 | from aepp import schema 63 | from aepp import classmanager 64 | 65 | mySandbox = aepp.importConfigFile('myconfig.json',sandbox='mysandbox',connectInstance=True) 66 | mySchemaInstance = schema.Schema(config=mySandbox) 67 | 68 | ## option 1 69 | myClass = classmanager.ClassManager(title='my Class Title', schemaAPI=mySchemaInstance)## setting a title now 70 | 71 | ## option 2 72 | myClass = classmanager.ClassManager(config=mySandbox) 73 | 74 | ``` 75 | 76 | ## ClassManager attributes 77 | 78 | Once you have instantiated the field group manager you can access some attributes directly via this object.\ 79 | The attributes available are: 80 | 81 | * EDITABLE : `True` if it can be modified directly via ClassManager, `False` if native class 82 | * title : Title of the class 83 | * STATE : either "EXISTING" or "NEW" 84 | * id : $id of the class 85 | * altId : meta:altId of the field Group 86 | * behavior : The behavior used to create the class 87 | 88 | ## ClassManager methods 89 | 90 | The different methods available for ClassManager will be available below. 91 | 92 | ### setTitle 93 | Set a name for the Class.\ 94 | Arguments: 95 | * title : REQUIRED : a string to be used for the title of the class 96 | 97 | ### getField 98 | Returns the field definition you want want to obtain.\ 99 | Arguments: 100 | * path : REQUIRED : path with dot notation to which field you want to access 101 | 102 | 103 | ### searchField 104 | Search for a field name based the string passed.\ 105 | By default, partial match is enabled and allow case sensitivity option.\ 106 | Arguments: 107 | * string : REQUIRED : the string to look for for one of the field 108 | * partialMatch : OPTIONAL : if you want to look for complete string or not. (default True) 109 | * caseSensitive : OPTIONAL : if you want to compare with case sensitivity or not. (default False) 110 | 111 | ### searchAttribute 112 | Search for an attribute on the field of the field groups.\ 113 | Returns either the list of fields that match this search or their full definitions.\ 114 | Arguments: 115 | * attr : REQUIRED : a dictionary of key value pair(s). Example : {"type" : "string"} \ 116 | NOTE : If you wish to have the array type on top of the array results, use the key "arrayType". Example : {"type" : "array","arrayType":"string"}\ 117 | This will automatically set the joinType to "inner". Use type for normal search. 118 | * regex : OPTIONAL : if you want your value of your key to be matched via regex.\ 119 | Note that regex will turn every comparison value to string for a "match" comparison. 120 | * extendedResults : OPTIONAL : If you want to have the result to contain all details of these fields. (default False) 121 | * joinType : OPTIONAL : If you pass multiple key value pairs, how do you want to get the match.\ 122 | outer : provide the fields if any of the key value pair is matched.\ 123 | inner : provide the fields if all the key value pair matched. 124 | 125 | 126 | ### addField 127 | Add the field to the existing class definition.\ 128 | Returns `False` when the field could not be inserted.\ 129 | Arguments: 130 | * path : REQUIRED : path with dot notation where you want to create that new field. New field name should be included. 131 | * dataType : REQUIRED : the field type you want to create 132 | A type can be any of the following: "string","boolean","double","long","integer","number","short","byte","date","dateTime","boolean","object","array","dataType" 133 | NOTE : "array" type is to be used for array of objects. If the type is string array, use the boolean "array" parameter. 134 | * title : OPTIONAL : if you want to have a custom title. 135 | * objectComponents: OPTIONAL : A dictionary with the name of the fields contain in the "object" or "array of objects" specify, with their typed. 136 | Example : {'field1:'string','field2':'double'} 137 | * array : OPTIONAL : Boolean. If the element to create is an array. False by default. 138 | * enumValues : OPTIONAL : If your field is an enum, provid a dictionary of value and display name, such as : {'value':'display'} 139 | * enumType: OPTIONAL: If your field is an enum, indicates whether it is an enum (True) or suggested values (False)\ 140 | * ref : OPTIONAL : If you have selected "dataType" as your `datatype`, you should use this parameter to pass the reference. 141 | possible kwargs: 142 | * defaultPath : Define which path to take by default for adding new field on tenant. Default "customFields", possible alternative : "property". 143 | * description : if you want to add a description on your field 144 | 145 | 146 | ### removeField 147 | Remove a field from the definition based on the path provided.\ 148 | NOTE: A path that has received data cannot be removed from a class.\ 149 | Argument: 150 | * path : REQUIRED : The path to be removed from the definition. 151 | 152 | ### to_dict 153 | Generate a dictionary representing the class constitution\ 154 | Arguments: 155 | * typed : OPTIONAL : If you want the type associated with the class to be given. 156 | * save : OPTIONAL : If you wish to save the dictionary in a JSON file 157 | 158 | ### to_dataframe 159 | Generate a dataframe with the row representing each possible path.\ 160 | Arguments: 161 | * save : OPTIONAL : If you wish to save it with the title used by the field group. 162 | save as csv with the title used. Not title, used "unknown_fieldGroup_" + timestamp. 163 | * queryPath : OPTIONAL : If you want to have the query path to be used. 164 | * description : OPTIONAL : If you want to have the description used (default False) 165 | * xdmType : OPTIONAL : If you want to have the xdmType also returned (default False) 166 | * editable : OPTIONAL : If you can manipulate the structure of the field groups (default False) -> see [Editable](#editable-concept) 167 | * excludeObjects : OPTIONAL : Boolean that remove the lines that are defining objects/nodes. Default `False`. 168 | 169 | ### to_xdm 170 | Return the class definition as XDM 171 | 172 | ### createClass 173 | Create the custom classs 174 | 175 | ## EDITABLE concept 176 | 177 | If a class is native (`IndividualProfile` or `ExperienceEvent`), the class definition can not be changed. The EDITABLE attribute will be set to `False`. 178 | If the class is a custom class, you can change and add attributes to it. -------------------------------------------------------------------------------- /docs/dataTypeManager.md: -------------------------------------------------------------------------------- 1 | # DataTypeManager for AEP Schema 2 | 3 | This module is a custom module built by Adobe Consulting team in order to improve efficiency of the Data Type manipulation, documentation and analysis.\ 4 | The Data Type Manager is built on top of all of the existing [AEP Schema Registry API](https://developer.adobe.com/experience-platform-apis/references/schema-registry/), and [Schema class](./schema.md). 5 | 6 | It is important to understand that a Field Group is built by using different data types. 7 | The native data types are : 8 | * "object": For nested JSON objects. 9 | * "string": For textual data. 10 | * "integer": For whole numbers. 11 | * "number": For numeric values, including decimals. 12 | * "double": For double-precision floating-point numbers 13 | * "short": For short integer numbers. 14 | * "long": For long integer numbers 15 | * "boolean": For true/false values. 16 | * "datetime": For date and time values. 17 | * "date": For date values. 18 | 19 | These native data types can be manipulated directly via the Field Groups.\ 20 | The more complex representation, that represents multiple fields, are called complex Data Type.\ 21 | 22 | If you were to create some, that are custom, you can use the DataTypeManager to modify and manipulate them programatically.\ 23 | 24 | A modification to a Data Type will have repercussion in all the schemas and field groups that are using this data type. 25 | 26 | **Since version 0.3.9** 27 | **It is part of the `datatypemanager` module** 28 | 29 | ## Menu 30 | - [Instantiation](#instantiation) 31 | - [Data Type Manager methods](#data-type-manager-methods) 32 | 33 | ## Instantiation 34 | 35 | The DataTypeManager is a class that can be instantiated with different parameters. 36 | Arguments: 37 | * dataType : OPTIONAL : Either a data type id ($id or altId) or the data type dictionary itself. 38 | If dataType Id is passed, you need to provide the schemaAPI connection as well. 39 | * title : OPTIONAL : to set or override the title (default None, use the existing title or do not set one for new data type) 40 | * schemaAPI : OPTIONAL : It is required if $id or altId are used. It is the instance of the `Schema` class. 41 | * config : OPTIONAL : The config object in case you want to override the last configuration loaded. 42 | 43 | code example: 44 | 45 | ```python 46 | import aepp 47 | from aepp import schema 48 | from aepp import datatypemanager 49 | 50 | prod = aepp.importConfigFile('myConfig.json',connectInstance=True,sandbox='prod') 51 | 52 | schemaInstance = schema.Schema(config=prod) 53 | 54 | myDataTypes = schemaInstance.getDataTypes() 55 | ## Selection of a data type 56 | myDataType = 'https://ns.adobe.com/tenant/datatypes/257370e5e265b90a2f71341bead54cd5d46c10fd14e' 57 | 58 | dataTypeManager = datatypemanager.DataTypeManager(myDataType,config=prod) 59 | 60 | ``` 61 | 62 | You can also use the `FieldGroupManager` `getDataTypeManager` method [see FieldGroupManager](./fieldGroupManager.md#getdatatypemanager) 63 | 64 | The same way that what has been offer for Schema or Field Group, you can also instantiate a new `DataTypeManager` class without any definition, and create one from scratch with the methods provided. 65 | 66 | Example code: 67 | 68 | ```python 69 | import aepp 70 | from aepp import schema 71 | from aepp import datatypemanager 72 | 73 | prod = aepp.importConfigFile('myConfig.json',connectInstance=True,sandbox='prod') 74 | dataTypeManager = datatypemanager.DataTypeManager(config=prod) 75 | 76 | ``` 77 | 78 | ## Data Type Manager methods 79 | 80 | The following methods are available on your `DataTypeManager` instance. 81 | 82 | ### setTitle 83 | Set the title on the Data Type description\ 84 | Argument: 85 | * title : REQUIRED : The title to be set 86 | 87 | ### setDescription 88 | Set the description to the Data Type.\ 89 | Argument: 90 | * description : REQUIRED : The description to be added 91 | 92 | 93 | ### getField 94 | Returns the field definition you want want to obtain.\ 95 | Arguments: 96 | * path : REQUIRED : path with dot notation to which field you want to access 97 | 98 | ### searchField 99 | Search for a field name based the string passed.\ 100 | By default, partial match is enabled and allow case sensitivity option.\ 101 | Arguments: 102 | * string : REQUIRED : the string to look for for one of the field 103 | * partialMatch : OPTIONAL : if you want to look for complete string or not. (default True) 104 | * caseSensitive : OPTIONAL : if you want to compare with case sensitivity or not. (default False) 105 | 106 | ### searchAttribute 107 | Search for an attribute on the field of the data type.\ 108 | Returns either the list of fields that match this search or their full definitions.\ 109 | Arguments: 110 | * attr : REQUIRED : a dictionary of key value pair(s). Example : {"type" : "string"}\ 111 | NOTE : If you wish to have the array type on top of the array results, use the key "arrayType".\ 112 | Example : {"type" : "array","arrayType":"string"}\ 113 | This will automatically set the joinType to "inner". Use type for normal search. 114 | * regex : OPTIONAL : if you want your value of your key to be matched via regex.\ 115 | Note that regex will turn every comparison value to string for a "match" comparison. 116 | * extendedResults : OPTIONAL : If you want to have the result to contain all details of these fields. (default False) 117 | * joinType : OPTIONAL : If you pass multiple key value pairs, how do you want to get the match. 118 | * outer : provide the fields if any of the key value pair is matched. 119 | * inner : provide the fields if all the key value pair matched. 120 | 121 | ### addFieldOperation 122 | Return the operation to be used on the data type with the Patch method (patchDataType), based on the element passed in argument.\ 123 | Arguments: 124 | * path : REQUIRED : path with dot notation where you want to create that new field.\ 125 | In case of array of objects, use the "[]{}" notation 126 | * dataType : REQUIRED : the field type you want to create\ 127 | A type can be any of the following: "string","boolean","double","long","integer","short","byte","date","dateTime","boolean","object","array"\ 128 | NOTE : "array" type is to be used for array of objects. If the type is string array, use the boolean "array" parameter. 129 | * title : OPTIONAL : if you want to have a custom title. 130 | * objectComponents: OPTIONAL : A dictionary with the name of the fields contain in the "object" or "array of objects" specify, with their typed.\ 131 | Example : {'field1':'string','field2':'double'} 132 | * array : OPTIONAL : Boolean. If the element to create is an array. False by default. 133 | * enumValues : OPTIONAL : If your field is an enum, provid a dictionary of value and display name, such as : {'value':'display'} 134 | * enumType: OPTIONAL: If your field is an enum, indicates whether it is an enum (True) or suggested values (False)\ 135 | possible kwargs: 136 | * defaultPath : Define which path to take by default for adding new field on tenant. Default "property", possible alternative : "customFields" 137 | 138 | ### addField 139 | Add the field to the existing Data Type definition.\ 140 | Returns False when the field could not be inserted.\ 141 | Arguments: 142 | * path : REQUIRED : path with dot notation where you want to create that new field. New field name should be included. 143 | * dataType : REQUIRED : the field type you want to create\ 144 | A type can be any of the following: "string","boolean","double","long","int","integer","short","byte","date","datetime","date-time","boolean","object","array"\ 145 | NOTE : "array" type is to be used for array of objects. If the type is string array, use the boolean "array" parameter. 146 | * title : OPTIONAL : if you want to have a custom title. 147 | * objectComponents: OPTIONAL : A dictionary with the name of the fields contain in the "object" or "array of objects" specify, with their typed.\ 148 | Example : {'field1:'string','field2':'double'} 149 | * array : OPTIONAL : Boolean. If the element to create is an array. False by default. 150 | * enumValues : OPTIONAL : If your field is an enum, provid a dictionary of value and display name, such as : {'value':'display'} 151 | * enumType: OPTIONAL: If your field is an enum, indicates whether it is an enum (True) or suggested values (False)\ 152 | possible kwargs: 153 | * defaultPath : Define which path to take by default for adding new field on tenant. Default "property", possible alternative : "customFields" 154 | * description : The description of the field 155 | 156 | ### removeField 157 | Remove a field from the definition based on the path provided.\ 158 | NOTE: A path that has received data cannot be removed from a schema or field group.\ 159 | Argument: 160 | * path : REQUIRED : The path to be removed from the definition. 161 | 162 | ### to_dict 163 | Generate a dictionary representing the field group constitution\ 164 | Arguments: 165 | * typed : OPTIONAL : If you want the type associated with the field group to be given. 166 | * save : OPTIONAL : If you wish to save the dictionary in a JSON file 167 | 168 | ### to_som 169 | Generate a Som instance of the dictionary. Helping the manipulation of the dictionary if needed. 170 | Documentation on [SOM](./som.md) 171 | 172 | ### to_dataframe 173 | Generate a dataframe with the row representing each possible path.\ 174 | Arguments: 175 | * save : OPTIONAL : If you wish to save it with the title used by the field group.\ 176 | save as csv with the title used. Not title, used "unknown_fieldGroup_" + timestamp. 177 | * description : OPTIONAL : If you want to have the description used (default False) 178 | * xdmType : OPTIONAL : If you want to retrieve the xdm Data Type (default False) 179 | * required : OPTIONAL : Provide an extra column `required` to specify which fields are set as required 180 | 181 | 182 | ### to_xdm 183 | Return the Data Type definition as XDM 184 | 185 | ### updateDataType 186 | Update the Data Type with the modification done before.\ 187 | It uses the PUT method, replacing previous definition. 188 | 189 | ### createDataType 190 | Use the POST method to create the Data Type in the organization. 191 | 192 | 193 | ### importDataTypeDefinition 194 | Importing the flat representation of the data type. It could be a dataframe or a CSV file containing the data type element.\ 195 | Argument: 196 | * datatype : REQUIRED : The dataframe or csv of the data type\ 197 | It needs to contains the following columns : "path", "xdmType" 198 | * sep : OPTIONAL : In case your CSV is separated by something else than comma. Default (',') 199 | * sheet_name : OPTIONAL : In case you are uploading an Excel, you need to provide the sheet name 200 | -------------------------------------------------------------------------------- /docs/dataaccess.md: -------------------------------------------------------------------------------- 1 | # Data Access module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the Data Access module and different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/data-access/).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## Menu 9 | 10 | - [Data Access module in aepp](#data-access-module-in-aepp) 11 | - [Importing the module](#importing-the-module) 12 | - [The DataAccess class](#the-dataaccess-class) 13 | - [The Profile class attributes](#the-profile-attributes) 14 | - [The Profile methods](#the-profile-methods) 15 | - [Use Cases](#customer-profile-use-cases) 16 | 17 | 18 | ## Importing the module 19 | 20 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 21 | 22 | To import the module you can use the import statement with the `dataaccess` keyword. 23 | 24 | ```python 25 | import aepp 26 | prod = aepp.importConfigFile('myConfig_file.json',sandbox='prod',connectInstance=True) 27 | 28 | from aepp import dataaccess 29 | ``` 30 | 31 | The dataaccess module provides a class that you can use for generating and retrieving the different catalog objects.\ 32 | 33 | The following documentation will provide you with more information on its capabilities. 34 | 35 | ## The DataAccess class 36 | 37 | The Data Access class is the default API connector that you would encounter for any other submodules on this python module.\ 38 | This class can be instantiated by calling the `DataAcess()` from the `dataaccess` module. 39 | 40 | Following the previous method described above, you can realize this: 41 | 42 | ```python 43 | import aepp 44 | prod = aepp.importConfigFile('myConfig_file.json',sandbox='prod',connectInstance=True) 45 | 46 | from aepp import dataaccess 47 | myData = dataaccess.DataAccess(config=prod) 48 | ``` 49 | 50 | There are 3 possible parameters when intantiating the class: 51 | 52 | * config : OPTIONAL : mostly used to pass a ConnectObject instance that is linked to one sandbox. 53 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 54 | * loggingObject : OPTIONAL : A logging object that can be passed for debuging or logging elements, see [logging documentation](./logging.md) 55 | 56 | ### Data Access Methods 57 | Here you can find the different methods available once you have instantiated the data access class. 58 | 59 | #### getBatchFiles 60 | List all dataset files under a batch.\ 61 | Arguments: 62 | * batchId : REQUIRED : The batch ID to look for.\ 63 | Possible kwargs: 64 | * limit : A paging parameter to specify number of results per page. 65 | * start : A paging parameter to specify start of new page. For example: page=1 66 | 67 | 68 | #### getBatchFailed 69 | Lists all the dataset files under a failed batch.\ 70 | Arguments: 71 | * batchId : REQUIRED : The batch ID to look for. 72 | * path : OPTIONAL : The full name of the file. The contents of the file would be downloaded if this parameter is provided.\ 73 | For example: path=profiles.csv\ 74 | Possible kwargs: 75 | * limit : A paging parameter to specify number of results per page. 76 | * start : A paging parameter to specify start of new page. For example: page=1 77 | 78 | 79 | #### getBatchMeta 80 | Lists files under a batch's meta directory or download a specific file under it. The files under a batch's meta directory may include the following: 81 | * row_errors: A directory containing 0 or more files with parsing, conversion, and/or validation errors found at the row level. 82 | * input_files: A directory containing metadata for 1 or more input files submitted with the batch. 83 | * row_errors_sample.json: A root level file containing the sampled set of row errors for the UX.\ 84 | Arguments: 85 | * batchId : REQUIRED : The batch ID to look for. 86 | * path : OPTIONAL : The full name of the file. The contents of the file would be downloaded if this parameter is provided.\ 87 | Possible values for this query include the following: 88 | * row_errors 89 | * input_files 90 | * row_errors_sample.json\ 91 | Possible kwargs: 92 | * limit : A paging parameter to specify number of results per page. 93 | * start : A paging parameter to specify start of new page. For example: page=1 94 | 95 | #### getHeadFile 96 | Get headers regarding a file.\ 97 | Arguments: 98 | * dataSetFileId : REQURED : The ID of the dataset file you are retrieving. 99 | * path : REQUIRED : The full name of the file identified.\ 100 | For example: path=profiles.json 101 | 102 | 103 | #### getFiles 104 | Returns either a complete file or a directory of chunked data that makes up the file.\ 105 | The response contains a data array that may contain a single entry or a list of files belonging to that directory.\ 106 | Arguments: 107 | * dataSetFileId : REQUIRED : The ID of the dataset file you are retrieving. 108 | * path : OPTIONAL : The full name of the file. The contents of the file would be downloaded if this parameter is provided.\ 109 | For example: path=profiles.csv\ 110 | if the extension is .parquet, it will try to return the parquet data decoded (returns a io.BytesIO). 111 | * range : OPTIONAL : The range of bytes requested. For example: Range: bytes=0-100000 112 | * start : OPTIONAL : A paging parameter to specify start of new page. For example: start=fileName.csv 113 | * limit : OPTIONAL : A paging parameter to specify number of results per page. For example: limit=10 114 | 115 | #### getPreview 116 | Give a preview of a specific dataset\ 117 | Arguments: 118 | * datasetId : REQUIRED : the dataset ID to preview 119 | 120 | #### getResource 121 | Template for requesting data with a GET method.\ 122 | Arguments: 123 | * endpoint : REQUIRED : The URL to GET 124 | * params: OPTIONAL : dictionary of the params to fetch 125 | * format : OPTIONAL : Type of response returned. Possible values:\ 126 | * json : default 127 | * txt : text file 128 | * raw : a response object from the requests module 129 | 130 | #### getParquetFilesToDataFrame 131 | Get a list of paths and download all of the files, transform them into dataframe and return the aggregated dataframe\ 132 | Arguments: 133 | * dataSetFileId : REQUIRED : The ID of the dataset file you are retrieving. 134 | * path : REQUIRED : the list of name of the files. The contents of the files will be downloaded if this parameter is provided.\ 135 | For example of value: ["YWGJ48C8R3QWPPQ_part-00001-93b3f57d-a7e5-4887-8832-f6ab1d1706b1-c0000.snappy.parquet"]\ 136 | It is intended for data that has been saved as snappy.parquet file in AEP. 137 | 138 | #### transformDataToDataFrame 139 | By passing the result of the getFiles with the parquet file path in the parameter, tries to return a pandas dataframe of the records.\ 140 | Argument: 141 | * data : REQUIRED : The _io.BytesIO data format returned by the getFiles method. 142 | 143 | ## Data Access use-cases 144 | 145 | The Data Access module will enable you to identify and download files that are holding your data.\ 146 | All data that you have previously ingested within Adobe Experience Platform can be downloaded from this API endpoint. 147 | 148 | When working with Adobe Experience Platform, you work with Batch ingestion data. Due to that setup, you will also retrieve information regardings batch file. 149 | 150 | ### Retrieving Files from Platform 151 | 152 | The `getBatchFiles` method will retrieve all the files ingested under a Batch. 153 | ```python 154 | myData.getBatchFiles() 155 | ``` 156 | 157 | The important element is that the response will provide you with a link such as: 158 | ```JSON 159 | { 160 | "href": "https://platform.adobe.io/data/foundation/export/files/{FILE_ID_1}" 161 | } 162 | ``` 163 | 164 | This is the URL where you can have multiple files within a batch. 165 | You can also retrieve the `FILE ID` from the response and use it with another method 166 | 167 | ### The getFiles method 168 | 169 | From the `FILE ID` retrieved from the `getBatchFiles` method, you can directly use them with the `getFiles` method. 170 | 171 | ```python 172 | myData.getFiles(dataSetFileId='fileId') 173 | ``` 174 | 175 | **IMPORTANT** : This method will provide either the complete file or a list of chunk for a file. 176 | If a list of chunk data are exposed, you can download them separately by providing the path parameter. 177 | Example: 178 | 179 | ```python 180 | myData.getFiles(dataSetFileId='fileId',path='profile.csv') 181 | ``` -------------------------------------------------------------------------------- /docs/datasets.md: -------------------------------------------------------------------------------- 1 | # Datasets module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the Datasets module and different methods supported by this module.\ 4 | Contrary to the other documentation, due to the limited methods available, all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/dataset-service/).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## What is the Datasets in AEP ? 9 | 10 | The Dataset Service API provides several endpoints to help you manage data usage labels for existing datasets within the Data Lake.\ 11 | Data usage labels are part of Adobe Experience Platform Data Governance, which allows you to manage customer data and ensure compliance with regulations, restrictions, and policies applicable to data use.\ 12 | Dataset Service is separate from Catalog Service, which manages other dataset metadata.\ 13 | If you wish to create datasets and manage datasets elements, you should then refer to the [catalog documentation](./catalog.md) 14 | 15 | ## Importing the module 16 | 17 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 18 | 19 | To import the module you can use the import statement with the `datasets` keyword. 20 | 21 | ```python 22 | import aepp 23 | aepp.importConfigFile('myConfig_file.json') 24 | 25 | from aepp import datasets 26 | ``` 27 | 28 | The datasets module provides a class that you can use for managing your labels on your datasets. 29 | 30 | ## The Datasets class 31 | 32 | The Datasets class uses the default API connector that you would encounter for any other submodules on this python module.\ 33 | This class can be instantiated by calling the `Datasets()` from the `datasets` module. 34 | 35 | Following the previous method described above, you can realize this: 36 | 37 | ```python 38 | datasetslabels = datasets.Datasets() 39 | ``` 40 | 41 | 2 parameters are possible for the instantiation of the class: 42 | 43 | * config : OPTIONAL : config object in the config module. (example : aepp.config.config_object) 44 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 45 | 46 | ## Datasets methods 47 | 48 | * `getLabels` 49 | Return the labels assigned to a dataSet 50 | Argument: 51 | * dataSetId : REQUIRED : the dataSet ID to retrieve the labels 52 | 53 | * `headLabels` 54 | Return the head assigned to a dataSet. You would required the ETAG parameter to modify or delete the labels. 55 | Argument: 56 | * dataSetId : REQUIRED : the dataSet ID to retrieve the head data 57 | 58 | * `deleteLabels` 59 | Delete the labels of a dataset. 60 | Arguments: 61 | * dataSetId : REQUIRED : The dataset ID to delete the labels for. 62 | * ifMatch : REQUIRED : the value is from the header etag of the headLabels. (use the headLabels method) 63 | 64 | * `createLabels` 65 | Assign labels to a dataset. 66 | Arguments: 67 | * dataSetId : REQUIRED : The dataset ID to delete the labels for. 68 | * data : REQUIRED : Dictionary setting the labels to be added. 69 | more info https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Datasets/postDatasetLabels 70 | 71 | * `updateLabels` 72 | Update the labels (PUT method) 73 | * dataSetId : REQUIRED : The dataset ID to delete the labels for. 74 | * data : REQUIRED : Dictionary setting the labels to be added. 75 | more info https://www.adobe.io/apis/experienceplatform/home/api-reference.html#/Datasets/postDatasetLabels 76 | * ifMatch : REQUIRED : the value is from the header etag of the headLabels.(use the headLabels method) 77 | 78 | On top of these methods, you will have access to an attribute of your instance that will provide you a dictionary sample for creation of labels for a dataset.\ 79 | The name of the attribute is `REFERENCE_LABELS_CREATION`\ 80 | The output will be: 81 | 82 | ```JSON 83 | { 84 | "labels": [ 85 | [ 86 | "C1", 87 | "C2" 88 | ] 89 | ], 90 | "optionalLabels": [ 91 | { 92 | "option": { 93 | "id": "https://ns.adobe.com/{TENANT_ID}/schemas/{SCHEMA_ID}", 94 | "contentType": "application/vnd.adobe.xed-full+json;version=1", 95 | "schemaPath": "/properties/repositoryCreatedBy" 96 | }, 97 | "labels": [ 98 | [ 99 | "S1", 100 | "S2" 101 | ] 102 | ] 103 | } 104 | ] 105 | } 106 | ``` -------------------------------------------------------------------------------- /docs/destination.md: -------------------------------------------------------------------------------- 1 | # Destination Authoring module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the `destination` module and different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/destination-authoring/).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## Importing the module 9 | 10 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 11 | 12 | To import the module you can use the import statement with the `destination` keyword. 13 | 14 | ```python 15 | import aepp 16 | aepp.importConfigFile('myConfig_file.json') 17 | 18 | from aepp import destination 19 | ``` 20 | 21 | The destination module provides a class that you can use to generate a SDK taking care of transfering some information to specific destination endpoints.\ 22 | The following documentation will provide you with more information on its capabilities. 23 | 24 | ## The Authoring class 25 | 26 | The Authoring class is the default API connector that you would encounter for any other submodules on this python module.\ 27 | This class can be instantiated by calling the `Authoring()` from the `destination` module. 28 | 29 | Following the previous method described above, you can realize this: 30 | 31 | ```python 32 | mySDK = destination.Authoring() 33 | ``` 34 | 35 | 3 parameters are possible for the instantiation of the class: 36 | 37 | * config : OPTIONAL : config object in the config module. (example : aepp.config.config_object) 38 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 39 | * loggingObject : OPTIONAL : logging object to provide log of the application. 40 | 41 | 42 | ## Destination module use-cases 43 | 44 | The Destination module will enable you to generate a destination application in AEP.\ 45 | This application can then deliver aduience and profile data to the endpoint selected.\ 46 | The configuration of that application is stored in the Experience Platform. Note that you can also add some authentication methods to your application. 47 | 48 | The destination API supports real-time integration with destinations that have a REST API endpoint.\ 49 | The capabilities supported are: 50 | * Message transformation and aggregation 51 | * Profile Backfill 52 | * Configurable metadata integration to initialize audience setup and data transfer 53 | * Configurable authentication 54 | * A suite of testing & validation APIs for you to test and iterate your destination configurations 55 | 56 | NOTE: You need to have acces to the Activation package to use this API. 57 | 58 | The complete documentation of the destination Authoring API is available in the experience league: https://experienceleague.adobe.com/docs/experience-platform/destinations/destination-sdk/overview.html?lang=en 59 | 60 | ## Destination methods 61 | 62 | This part is describing the different methods available from that module, once you have generated your instance. 63 | 64 | * getDestinations 65 | Return a list of all destination SDK authored by the organization. 66 | 67 | * getDestination 68 | Return a destination specific configuration. 69 | Arguments: 70 | * destinationId : REQUIRED : The destination ID to be retrieved 71 | 72 | * deleteDestination 73 | Delete a specific destination based on its ID. 74 | Arguments: 75 | * destinationId : REQUIRED : The destination ID to be deleted 76 | 77 | * createDestination 78 | Create a destination based on the definition passed in argument. 79 | Arguments: 80 | * destinationObj : REQUIRED : Object containing the definition of the destination. 81 | 82 | * updateDestination 83 | Create a destination based on the definition passed in argument. 84 | Arguments: 85 | * destinationId : REQUIRED : The destination ID to be updated 86 | * destinationObj : REQUIRED : Object containing the definition of the destination. 87 | 88 | * getDestinationServers 89 | Retrieve a list of all destination server configurations for your IMS Organization 90 | 91 | * getDestinationServer 92 | Retrieve a specific destination server configuration by its ID. 93 | Arguments: 94 | * serverId : REQUIRED : destination server ID of the server 95 | 96 | * deleteDestinationServer 97 | Delete a destination server by its ID. 98 | Arguments: 99 | * serverId : REQUIRED : destination server ID to be deleted 100 | 101 | * createDestinationServer 102 | Create a new destination server configuration. 103 | Arguments: 104 | * serverObj : REQUIRED : dictionary containing the server destination configuration 105 | 106 | * updateDestinationServer 107 | Update the destination with a new definition (PUT request) 108 | Arguments: 109 | * serverId : REQUIRED : destination server ID to be updated 110 | * serverObj : REQUIRED : dictionary containing the server configuration 111 | 112 | * getAudienceTemplates 113 | Return a list of all audience templates for your IMS Organization 114 | 115 | * getAudienceTemplate 116 | Return a specific Audience Template. 117 | Arguments: 118 | * audienceId : REQUIRED : The ID of the audience template configuration that you want to retrieve. 119 | 120 | * deleteAudienceTemplate 121 | Delete a specific Audience Template. 122 | Arguments: 123 | * audienceId : REQUIRED : The ID of the audience template configuration that you want to delete 124 | 125 | * createAudienceTemplate 126 | Create a specific Audience Template based on a dictionary definition passed as parameter. 127 | Arguments: 128 | * templateObj : REQUIRED : The ID of the audience template configuration that you want to retrieve. 129 | 130 | * updateAudienceTemplate 131 | Update a specific Audience Template based on a dictionary definition passed as parameter. 132 | Arguments: 133 | * audienceId : REQUIRED : The ID of the audience template configuration that you want to delete 134 | * templateObj : REQUIRED : The ID of the audience template configuration that you want to retrieve. 135 | 136 | * getCredentials 137 | Retrieve a list of all credentials configurations for your IMS Organization 138 | 139 | * getCredential 140 | Return a specific credential based on its ID. 141 | Arguments: 142 | * credentialId : REQUIRED : The ID of the credential to retrieve 143 | 144 | * deleteCredential 145 | Delete a specific credential based on its ID. 146 | Arguments: 147 | * credentialId : REQUIRED : The ID of the credential to delete 148 | 149 | * createCredential 150 | Create a credential configuration based on the dictionary passed. 151 | Arguments: 152 | * credentialObj : REQUIRED : The credential object definition 153 | 154 | * updateCredential 155 | Update the credential configuration based on the dictionary and the credential ID passed. 156 | Arguments: 157 | * credentialId : REQUIRED : The credentialId to be updated 158 | * credentialObj : REQUIRED : The credential object definition 159 | 160 | * getSampleProfile 161 | Generate a sample profile of a destination given the correct arguments. 162 | Arguments: 163 | * destinationInstanceId : REQUIRED : Also known as order ID. The ID of the destination instance based on which you are generating sample profiles. (example: "49966037-32cd-4457-a105-2cbf9c01826a") 164 | Documentation on how to retrieve it: https://experienceleague.adobe.com/docs/experience-platform/destinations/destination-sdk/api/developer-tools-reference/destination-testing-api.html?lang=en#get-destination-instance-id 165 | * destinationId : REQUIRED : he ID of the destination configuration based on which you are generating sample profiles. The destination ID that you should use here is the ID that corresponds to a destination configuration, created using the createDestination method. 166 | * count : OPTIONAL : The number of sample profiles that you are generating. The parameter can take values between 1 - 1000. 167 | 168 | * getSampleDestination 169 | Returns a sample template corresponding to the destinationID passed. 170 | Argument 171 | * destinationConfigId : REQUIRED : The ID of the destination configuration for which you are generating a message transformation template. 172 | The destination ID that you should use here is the ID that corresponds to a destination configuration, created using the createDestination method 173 | 174 | * generateTestProfile 175 | Generate exported data by making a POST request to the testing/template/render endpoint and providing the destination ID of the destination configuration and the template you created using the sample template API endpoint 176 | Arguments: 177 | * destinationId : REQUIRED : The ID of the destination configuration for which you are rendering exported data. 178 | * template : REQUIRED : The character-escaped version of the template based on which you are rendering exported data. 179 | * profiles : OPTIONAL : list of dictionary returned by the getSampleProfile method 180 | 181 | * sendMessageToPartner 182 | Test the connection to your destination by sending messages to the partner endpoint. 183 | Optionally, you can send a list of profiles in the request. If you do not send any profiles, Experience Platform generates those internally. 184 | In this case, you can view the profiles that were used for validation in the response you receive from your getSampleProfile endpoint. 185 | Arguments: 186 | * destinationInstanceId : REQUIRED : Also known as order ID. The ID of the destination instance based on which you are generating sample profiles. 187 | See documentation for info on how to retrieve it: https://experienceleague.adobe.com/docs/experience-platform/destinations/destination-sdk/api/developer-tools-reference/destination-testing-api.html?lang=en#get-destination-instance-id 188 | * profiles : OPTIONAL : list of dictionary returned by the getSampleProfile method 189 | 190 | * getSubmissions 191 | List of all destinations submitted for publishing for your IMS Organization 192 | 193 | * getSubmission 194 | Get a specific destination submission status based on the ID passed. 195 | Argument: 196 | * destinationConfigId : REQUIRED : The ID of the destination configuration you have submitted for publishing. 197 | 198 | * SubmitDestination 199 | Submit a destination configuration for publishing 200 | Arguments: 201 | * destinationObj : REQUIRED : The object defining the destination config. (DestinationId, Access, AllowedOrgs) 202 | 203 | * updateSubmissionRequest 204 | Update the allowed organizations in a destination publish request. 205 | Arguments: 206 | * destinationConfigId : REQUIRED : The ID of the destination configuration you have submitted for publishing. 207 | * destinationObj : REQUIRED : The object defining the destination config. (DestinationId, Access, AllowedOrgs) -------------------------------------------------------------------------------- /docs/destinationinstanceservice.md: -------------------------------------------------------------------------------- 1 | # Destination Instance module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the `destinationinstanceservice` module and different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the destination instance API, please refer to this [API documentation](https://experienceleague.adobe.com/docs/experience-platform/destinations/api/ad-hoc-activation-api.html?lang=en).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## Importing the module 9 | 10 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 11 | 12 | To import the module you can use the import statement with the `destinationinstanceservice` keyword. 13 | 14 | ```python 15 | import aepp 16 | aepp.importConfigFile('myConfig_file.json') 17 | 18 | from aepp import destinationinstanceservice 19 | ``` 20 | 21 | The destinationinstanceservice module provides a class that you can use to create adhoc export tasks.\ 22 | The following documentation will provide you with more information on its capabilities. 23 | 24 | ## The Instance class 25 | 26 | The Authoring class is the default API connector that you would encounter for any other submodules on this python module.\ 27 | This class can be instantiated by calling the `Instance()` from the `destinationinstanceservice` module. 28 | 29 | Following the previous method described above, you can realize this: 30 | 31 | ```python 32 | mySDK = destinationinstanceservice.DestinationInstanceService() 33 | ``` 34 | 35 | 3 parameters are possible for the instantiation of the class: 36 | 37 | * config : OPTIONAL : config object in the config module. (example : aepp.config.config_object) 38 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 39 | * loggingObject : OPTIONAL : logging object to provide log of the application. 40 | 41 | ## Destination Instance Service module use-cases 42 | The Destination module will enable you to create adhoc request for export segments/datasets 43 | 44 | ## Destination Instance Service methods 45 | This part is describing the different methods available from that module, once you have generated your instance. 46 | 47 | * createAdHocDatasetExport 48 | create adhoc task for export datasets -------------------------------------------------------------------------------- /docs/edge.md: -------------------------------------------------------------------------------- 1 | # Edge module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the Edge module and the different methods supported by this module.\ 4 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://experienceleague.adobe.com/en/docs/experience-platform/edge-network-server-api/overview).\ 5 | Alternatively, you can use the docstring in the methods to have more information. 6 | 7 | ## Menu 8 | 9 | - [What is the edge module in aepp](#What-is-the-edge-module-in-aepp) 10 | - [Importing the module](importing-edgeserverside) 11 | - [The Edge class](#instantiation) 12 | - [The Edge class attributes](#the-edge-class-attributes) 13 | - [The Edge methods](#edge-methods) 14 | - [The IdentityMapHelper class](#the-identitymaphelper-class) 15 | 16 | ## What is the edge module in aepp 17 | 18 | The edge module is not technically an AEP feature. It is part of the AEP Web SDK and App SDK data collection methods that are used for data ingestion on applications.\ 19 | The most famous library use is the AEP Web SDK, that helps tracking user behavior on the web. 20 | 21 | 22 | ## Importing edge 23 | 24 | The `edge` module can be done directly via the normal `aepp` main module. 25 | 26 | ```python 27 | import aepp 28 | from aepp import edge 29 | 30 | ``` 31 | 32 | Note that the edge connection can work **with** or **without** authentication.\ 33 | It depends if your datastream has been setup to receive authenticated traffic or not.\ 34 | Most customers do not set authentication for Edge services.\ 35 | 36 | In case you need to use authenticated traffic you would need to import a config file to authenticate.\ 37 | For the sake of the example, we will use the authenticated calls in the example below. 38 | 39 | ```python 40 | import aepp 41 | from aepp import edge 42 | 43 | prod = aepp.importConfigFile('myConfig_file.json',connectInstance=True,sandbox='prod') 44 | 45 | ``` 46 | 47 | 48 | ## The Edge class 49 | 50 | The `edge` module contains an `Edge` class that can be instantiated with or without a configuration. Contrary to most of the other module classes.\ 51 | However, there are other attributes that you need to specify or can specifiy to use the Edge capability. 52 | 53 | The parameters available for instantiation of the class: 54 | * dataStreamId : REQUIRED : The datastream ID that to be used for collecting data 55 | * server : OPTIONAL : If you use a CNAME to send data, you can pass that CNAME server here. Default is `server.adobedc.net` 56 | * config : OPTIONAL : If you need / want to authenticate the call to the Edge network 57 | * version : OPTIONAL : By default the version `2` is used for Edge Server Side data collection. However, you can set the version to `1` to use client side data collection. 58 | 59 | A default implementation would be: 60 | 61 | ```py 62 | import aepp 63 | from aepp import edge 64 | 65 | prod = aepp.importConfigFile('myConfig_file.json',connectInstance=True,sandbox='prod') 66 | 67 | myEdge = edge.Edge('mydatastreamId',config=prod) 68 | 69 | ``` 70 | 71 | ### The Edge class attributes 72 | 73 | Once you have instantiated the class, you can access several attributes: 74 | 75 | * versionEdge : which version of edge you are using 76 | * server : which server is being used 77 | * endpoint : which endpoint is used to send the requests 78 | * params : The parameters that are used on each request. It contains mostly the `dataStreamId` 79 | * origin : Use to identify the origin of the call for the Edge requests. 80 | * token : Available only if you are authenticated. Token used for the requests. 81 | 82 | 83 | ### The Edge methods 84 | 85 | You can find the different methods available in the `Edge` instance. 86 | 87 | #### interact 88 | Send an interact calls. It usually return a response that can be used on the application.\ 89 | Arguments: 90 | * payload : OPTIONAL : In case you want to pass the whole payload yourself 91 | * xdm : OPTIONAL : In case you want to pass only XDM data 92 | * data : OPTIONAL : In case you want to pass the data object (can be passed with xdm) 93 | * scopes : OPTIONAL : In case you want to pass Target scopes/mbox in the request or the Offer Decisioning scope. It should be a list of strings. `["scopeId","__view__"]` 94 | * surfaces : OPTIONAL : In case you want to pass AJO surfaces in the request. List of strings. 95 | * params: OPTIONAL : If you want to pass additional query parameter. It takes a dictionary. 96 | * assuranceToken : OPTIONAL : If you want to pass an assurance token value for debugging via a session\ 97 | Usually one value, additional ones are separated by a pip such as: "dc9d59df-9b15-44d3-82d6-2f718ad5ec4a|7ddf4cc5-e304-4d95-991c-01359fe9a7de" 98 | 99 | 100 | #### collect 101 | In case you want to send multiple requests in one go. These are not returning response that can be used by the application.\ 102 | They are just sending data to AEP.\ 103 | You can send requests from different users.\ 104 | Arguments: 105 | * payloads : OPTIONAL : A list of payload to be send via Edge. 106 | * xdms : OPTIONAL : A list of xdm to be sent via Edge 107 | * data : OPTIONAL : A list of data to attach to the xdms calls (note that the list of xdms and data should be in the same order) 108 | * assuranceToken : OPTIONAL : If you want to pass an assurance token value for debugging via a session.\ 109 | Usually one value, additional ones are separated by a pip such as: "dc9d59df-9b15-44d3-82d6-2f718ad5ec4a|7ddf4cc5-e304-4d95-991c-01359fe9a7de" 110 | 111 | 112 | ## IdentityMapHelper 113 | 114 | One element that is always hard to conceptualize in the XDM body is the IdentityMap.\ 115 | In order to help the creation of such `identityMap` object, a class is offered to help : `IdentityMapHelper` 116 | 117 | ### Instantiation 118 | 119 | The instantiation of the `IdentityMapHelper` can be realized with or without any parameter.\ 120 | Possible arguments: 121 | * namespace : OPTIONAL : User namespace 122 | * identity : OPTIONAL : User Value for that namespace 123 | * primary : OPTIONAL : Default True. 124 | * state : OPTIONAL : Default ambiguous. possible options: 'authenticated' 125 | 126 | ```python 127 | import aepp 128 | from aepp import edge 129 | 130 | myIdMap = edge.IdentityMapHelper() 131 | 132 | ## or 133 | 134 | myIdMap = edge.IdentityMap('CRMID','myCRMID1',True,'authenticated') 135 | 136 | ``` 137 | 138 | ### IdentityMapHelper methods 139 | 140 | Here are the different methods available for `IdentityMapHelper` class. 141 | 142 | #### addIdentity 143 | Add an identity to the identityMap.\ 144 | Arguments: 145 | * namespace : REQUIRED : User namespace 146 | * identity : REQUIRED : User Value for that namespace 147 | * primary : OPTIONAL : Default False. 148 | * state : OPTIONAL : Default "ambigous", possible state: "authenticated" 149 | 150 | 151 | #### removePrimaryFlag 152 | remove the primary flag from the identity map.\ 153 | Arguments: 154 | * namespace : OPTIONAL : The namespace to remove the identity primary flag 155 | * identity : OPTIONAL : The identity to remove the identity flag. 156 | 157 | If nothing is provided, it will loop through all identities and remove the primary flag 158 | 159 | #### setPrimaryFlag 160 | Set an identity as primary identity.\ 161 | Arguments: 162 | * namespace : OPTIONAL : If you want to specify the namespace to set the primary identity.\ 163 | If no identity are provided and multiple identities are available, the first one is picked up to be primary. 164 | * identity : OPTIONAL : the identity to be used as primary. 165 | 166 | #### to_dict 167 | Returns the identityMap as a dictionary (python compatible) 168 | 169 | #### to_json 170 | Returns the identityMap as a JSON string (JS compatible) -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting started with aepp 2 | 3 | ## Menu 4 | 5 | - [Installing the module](#installing-the-module) 6 | - [Create a Developer Project ](#create-a-developer-project) 7 | - [Oauth Server-to-Server](#oauth-server-to-server) 8 | - [Oauth V1](#oauth-v1) 9 | - [Using the module](#using-the-module) 10 | - [Create a Config file](#create-a-config-file) 11 | - [Environments](#environments) 12 | - [Importing the config file](#importing-the-config-file) 13 | - [Alternative method for cloud configuration](#alternative-method-for-cloud-configuration) 14 | - [The ConnectInstance parameter](#the-connectinstance-parameter) 15 | - [Importing a sub module to work with](#importing-a-sub-module-to-work-with) 16 | - [Help](#help) 17 | 18 | ## Installing the module 19 | 20 | You would need to install the module in your environment.\ 21 | You can use the pip command to do so. 22 | 23 | ```shell 24 | pip install aepp --upgrade 25 | ``` 26 | 27 | You can use the upgrade argument when a release has been made. 28 | 29 | ## Create a Developer Project 30 | 31 | You will need to have a developer project that has access to the Adobe Experience Platform API.\ 32 | When creating a project you have the possibility to use 2 authentication methods. 33 | 34 | * OAuth-based authentication 35 | 36 | ### Oauth Server-to-Server 37 | 38 | In 2023, the Oauth Server-to-Server token has been introduced in the API environment of Adobe.\ 39 | `aepp` is now supporting this capabiliy and you can create an `Oauth Server-to-Server` integration.\ 40 | In the config file, it is named `oauthV2`. 41 | 42 | in developer.adobe.com, make sure you have developer rights and attaching the correct product profile to your integration.\ 43 | You will need to have the following information saved to be used later: 44 | - Client ID 45 | - Client secret 46 | - Technical Account ID 47 | - Scopes 48 | - IMS Org 49 | 50 | ### Oauth V1 51 | 52 | For internal usage of the aepp module, for Adobe teams, you can also use the oauth v1 to interact with other services. 53 | You will need to have the following information saved to be used later: 54 | - Client ID 55 | - Client secret 56 | - auth_code : note that this can be either a permanent or temporary code 57 | - IMS Org 58 | 59 | ## Using the module 60 | 61 | Once you have created the developer project in developer.adobe.com, you can start using the module.\ 62 | In order to start using the module, you will need to import it on your environment.\ 63 | This is where the `import` keyword is used for that module. 64 | 65 | 66 | ```python 67 | import aepp 68 | ``` 69 | 70 | ### Create a config file 71 | 72 | Once you have imported the module in your environment, you probably want to create a config file for authentication.\ 73 | The `createConfigFile` is the method directly available out of aepp module to help you create the configuration file needed.\ 74 | 75 | As explained above, there are 2 options: 76 | 77 | * Oauth V2 config file 78 | * Oauth V1 config (internal Adobe engineering) 79 | 80 | 81 | If you want to use OAuth-V2-based authentication, use the following code: 82 | 83 | ```python 84 | import aepp 85 | aepp.createConfigFile(destination='template_config.json', auth_type="oauthV2") 86 | ``` 87 | 88 | The resulting file will have different fields: 89 | 90 | ```JSON 91 | { 92 | "org_id": "", 93 | "client_id": "", 94 | "secret": "", 95 | "sandbox-name": "prod", 96 | "environment": "prod", 97 | "scopes": "" 98 | } 99 | ``` 100 | 101 | If you want to use OAuth-V1-based authentication, use the following code: 102 | 103 | ```python 104 | import aepp 105 | aepp.createConfigFile(destination='template_config.json', auth_type="oauthV1") 106 | ``` 107 | The resulting file will have different fields: 108 | 109 | ```JSON 110 | { 111 | "org_id": "", 112 | "client_id": "", 113 | "secret": "", 114 | "sandbox-name": "prod", 115 | "environment": "prod", 116 | "auth_code": "" 117 | } 118 | ``` 119 | 120 | In both cases, remove the `` and replace them with your information.\ 121 | All information are available on your project page on developer.adobe.com 122 | 123 | **Note** By default, we are setting the sandbox name to "prod". If you don't know what that value, you can override it via a parameter. 124 | 125 | **Note** The default behavior has been changed starting June 2023, where oauthV2 is the default type of configuration file created in case you omit the parameter. 126 | 127 | Parameter for `createConfigFile` method: 128 | 129 | * destination : OPTIONAL : The name of the file to be created (with a dedicated path if needed) 130 | * sandbox : OPTIONAL : You can directly set your sandbox name in this parameter. 131 | * auth_type : OPTIONAL : type of authentication, either "jwt" or "oauthV2" or "oauthV1" (default oauthV2) 132 | * verbose : OPTIONAL : set to true, gives you a print stateent where is the location. 133 | 134 | 135 | ### Environments 136 | 137 | By default, the environment is set to `prod`. This is different from the sandbox, as it refers to the physical environment where the organization was setup. 138 | 139 | For all AEP customers "prod" is what should be used, but for internal accounts it can be set to "stage" or "int". 140 | 141 | ### Importing the config file 142 | 143 | Once your config file has been generated, you can import it in your script by using the `importConfigFile` method. 144 | 145 | ```python 146 | import aepp 147 | aepp.importConfigFile('myConfig_file.json') 148 | ``` 149 | 150 | The type of authentication will be automatically determined based on the keys provided by the JSON config file. Be careful to not mix JWT and Oauth on the same config file.\ 151 | 152 | Parameter for `importConfigFile` method: 153 | * path: REQUIRED : path to the configuration file. Can be either a fully-qualified or relative. 154 | * connectInstance : OPTIONAL : If you want to return an instance of the ConnectObject class 155 | * auth_type : OPTIONAL : type of authentication, either "jwt" or "oauth". Detected based on keys present in config file. 156 | * sandbox : OPTIONAL : The sandbox to connect it. 157 | 158 | The `connectInstance` parameter is described below. see [Tip for multi sandbox work](#tip-for-multi-sandbox-work)\ 159 | The `sandbox` paramter is to facilitate your life, in case you want to use the same config file for multiple sandboxes. 160 | 161 | ### Alternative method for cloud configuration 162 | 163 | You can also use the configure method to setup the connection directly on the aepp setup.\ 164 | This approach is better if you don't want to use a file in your system.\ 165 | In that case, you can directly pass the elements in the configure method. 166 | 167 | If you want to use OAuth-V2-based authentication, simply use different parameters when calling `configure`: 168 | 169 | ```python 170 | import aepp 171 | aepp.configure( 172 | org_id=my_org_id, 173 | secret=my_secret, 174 | client_id=my_client_id, 175 | scopes=my_scopes, 176 | environment="prod" 177 | ) 178 | ``` 179 | 180 | If you instead want to use OAuth-V1-based authentication, simply use different parameters when calling `configure`: 181 | 182 | ```python 183 | import aepp 184 | aepp.configure( 185 | org_id=my_org_id, 186 | secret=my_secret, 187 | client_id=my_client_id, 188 | auth_code=my_auth_code, 189 | environment="prod" 190 | ) 191 | ``` 192 | 193 | **NOTE** : In both case, I didn't provide a `sandbox` parameter but this parameter does exist and can be used to setup a specific sandbox.\ 194 | By default, the `prod` sandbox will be used. To use that, use the code below (for JWT): 195 | 196 | ```python 197 | import aepp 198 | aepp.configure( 199 | org_id=my_org_id, 200 | tech_id=my_tech_id, 201 | secret=my_secret, 202 | private_key=my_key_as_string, 203 | client_id=my_client_id, 204 | environment="prod", 205 | sandbox=my_sandbox 206 | ) 207 | ``` 208 | 209 | **NOTE** The `environment` parameter is optional and defaults to "prod". 210 | 211 | ### The ConnectInstance parameter 212 | 213 | The `aepp` module contains a parameter named `connectInstance` for `importConfig` and `configure` methods that provide a way to store the configuration setup.\ 214 | As you import the config file, you will default any instantiation of the sub module to the latest loaded configuration.\ 215 | Using this parameter will make the methods returning an instance of the `ConnectObject` class.\ 216 | That will store the information required to connect to your IMS or sandbox setup (secret, client_id, tech_id, IMSorg, etc...) 217 | 218 | You can use that instance then in any of the sub module that is provided with the aepp package and that are related to the AEP API.\ 219 | You will be able to pass that instance to the `config` parameter of any submodule (see next section) 220 | 221 | Example: 222 | 223 | ```python 224 | import aepp 225 | myOrg1 = aepp.importConfigFile('org1_config.json',connectInstance=True) 226 | ``` 227 | 228 | ## Importing a sub module to work with 229 | 230 | You can then import the sub module and you will require to instantiate the class inside that module.\ 231 | The class has usually the same name than the sub module but with a capital letter. 232 | 233 | Example with schema sub module and Schema class. 234 | 235 | ```python 236 | import aepp 237 | aepp.importConfigFile('myConfig_file.json') 238 | ## using the connectInstance parameter 239 | config1 = aepp.importConfigFile('myConfig_file.json',connectInstance=True) 240 | 241 | from aepp import schema 242 | 243 | mySchemaInstance = schema.Schema() 244 | ## using the instance of config use 245 | mySchemaInstance = schema.Schema(config=config1) 246 | ``` 247 | 248 | This works exactly the same for all of the sub modules mentioned in the [README page](../README.md). 249 | Note the queryservice and privacyservice have exceptions mentioned on the README. 250 | 251 | The idea to have a class instantiated for each submodule has been made in order to allow to work with several sandboxes (or organization) in the same environment.\ 252 | You can always access the sandbox used by using the instance `sandbox` attribute.\ 253 | Following the previous example: 254 | 255 | ```python 256 | mySchemaInstance.sandbox ## will return which sandbox is configured in that environment. 257 | ``` 258 | 259 | ## Help 260 | 261 | You can always use the docstring definition to help you using the functions.\ 262 | I tried to give a clear documentation of what each function is capable of. 263 | 264 | ```python 265 | help(mySchemaInstance.getSchemas) 266 | ## returns 267 | 268 | #getSchemas(**kwargs) -> list method of aepp.schema.Schema instance 269 | # Returns the list of schemas retrieved for that instances in a "results" list. 270 | # Kwargs: 271 | # debug : if set to true, will print the result when error happens 272 | ``` 273 | -------------------------------------------------------------------------------- /docs/hygiene.md: -------------------------------------------------------------------------------- 1 | # Data Hygiene in AEP 2 | 3 | The data hygiene API is not yet officially presented in the AEP API catalog.\ 4 | You can find the details of the API in the [Experience League](https://experienceleague.adobe.com/en/docs/experience-platform/data-lifecycle/api/overview)\ 5 | The module is therefore relying on the information of Experience League and is here to help users of the AEP product to automate their Data Hygiene calls directly from `aepp` 6 | 7 | **NOTE** : All functionalities of the data hygiene setup is not yet provided to all customers. Some requires migration of your data platform, some are available after a licence purchase.\ 8 | Contact your adobe representative to know what can be used in your organization. 9 | 10 | ## Menu 11 | 12 | - [Data Hygiene in AEP](#data-hygiene-in-aep) 13 | - [Importing the module](importing-the-module) 14 | - [Generating a Hygiene instance](#generating-a-hygiene-instance) 15 | - [Data Hygiene Methods](#data-hygiene-methods) 16 | 17 | 18 | ## Importing the module 19 | 20 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 21 | 22 | To import the module you can use the import statement with the `hygiene` keyword. 23 | 24 | ```python 25 | import aepp 26 | prod = aepp.importConfigFile('myConfig_file.json', sandbox='prod', connectInstance=True) 27 | 28 | from aepp import hygiene 29 | ``` 30 | 31 | ## Generating a Hygiene instance 32 | 33 | Because you can connect to multiple AEP instance at once, or multiple sandboxes, you would need to setup an instance of the `Hygiene` class from that module.\ 34 | Following the previous method described above, you can realize this: 35 | 36 | ```python 37 | myHygieneSandbox = hygiene.Hygiene(config=prod) 38 | ``` 39 | 40 | Several parameters are possibles when instantiating the class:\ 41 | 42 | * config : OPTIONAL : mostly used to pass a ConnectObject instance that is linked to one sandbox as described in the example above. 43 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 44 | 45 | ### Using different ConnectObject for different sandboxes 46 | 47 | You can use the `connectInstance` parameter to load multiple sandbox configuration and save them for re-use later on, when instantiating the `Hygiene` class. 48 | As described above, it can be useful when you want to connect to multiple sandboxes with the same authentication.\ 49 | In that case, the 2 instances will be created as such: 50 | 51 | ```python 52 | import aepp 53 | prod = aepp.importConfigFile('myConfig_file.json', sandbox='prod', connectInstance=True) 54 | dev = aepp.importConfigFile('myConfig_file.json', sandbox='dev', connectInstance=True) 55 | 56 | from aepp import hygiene 57 | 58 | myHygieneSandbox1 = hygiene.Hygiene(config=prod) 59 | myHygieneSandbox2 = hygiene.Hygiene(config=dev) 60 | 61 | ``` 62 | 63 | ## Data Hygiene Methods 64 | 65 | In the below section, we will document all methods available for the Hygene class 66 | 67 | ### getQuotas 68 | 69 | Returns a list of quota types and their status.\ 70 | It allows you to monitor your Advanced data lifecycle management usage against your organization's quota limits for each job type.\ 71 | Arguments: 72 | * quotaType : OPTIONAL : If you wish to restrict to specific quota type.\ 73 | Possible values: 74 | * expirationDatasetQuota (Dataset expirations) 75 | * deleteIdentityWorkOrderDatasetQuota (Record delete) 76 | * fieldUpdateWorkOrderDatasetQuota (Record updates) 77 | 78 | ### getDatasetsExpirations 79 | 80 | allows you to schedule expiration dates for datasets in Adobe Experience Platform.\ 81 | A dataset expiration is only a timed-delayed delete operation. The dataset is not protected in the interim, so it may be be deleted by other means before its expiry is reached.\ 82 | It can take up to 24h after the date specified before the dataset is deleted from AEP.\ 83 | It can take up to 7 days for all services (UIS, UPS, CJA, etc...) to reflect the deletion impact.\ 84 | Arguments:\ 85 | Possible keywords:\ 86 | * author : Matches expirations whose created_by (ex: author=LIKE %john%, author=John Q. Public) 87 | * datasetId : Matches expirations that apply to specific dataset. (ex : datasetId=62b3925ff20f8e1b990a7434) 88 | * datasetName : Matches expirations whose dataset name contains the provided search string. The match is * case-insensitive. (ex : datasetName=Acme) 89 | * createdDate : Matches expirations that were created in the 24-hour window starting at the stated time. (ex : createdDate=2021-12-07) 90 | * createdFromDate : Matches expirations that were created at, or after, the indicated time. (ex : createdFromDate=2021-12-07T00:00:00Z) 91 | * createdToDate : Matches expirations that were created at, or before, the indicated time. (ex : createdToDate=2021-12-07T23:59:59.00Z) 92 | * completedToDate : Matches expirations that were completed during the specified interval. (ex: completedToDate=2021-11-11-06:00) 93 | * status : A comma-separated list of statuses. When included, the response matches dataset expirations whose current status is among those listed. (ex : status=pending,cancelled) 94 | * updatedDate : matches against a dataset expiration's update time instead of creation time. (updatedDate=2022-01-01) 95 | * full list : https://experienceleague.adobe.com/en/docs/experience-platform/data-lifecycle/api/dataset-expiration#appendix 96 | 97 | ### getDatasetExpiration 98 | 99 | To retrieve the specify dataset deletion.\ 100 | One of the 2 parameters is required.\ 101 | Arguments: 102 | * datasetId : OPTIONAL : the datasetId to look for 103 | * ttlId : OPTIONAL : The ttlId returned when setting the ttl. 104 | 105 | 106 | ### createDatasetExpiration 107 | 108 | Create or update an expiration date for a dataset through a PUT request.\ 109 | The PUT request uses either the datasetId or the ttlId.\ 110 | One of the 2 first parameters is required.\ 111 | Arguments: 112 | * datasetId : OPTIONAL : the datasetId to set expiration for 113 | * ttlId : OPTIONAL : The ID of the dataset expiration. 114 | * expiry : REQUIRED : the expiration in date such as "2024-12-31T23:59:59Z" 115 | * name : REQUIRED : name of the ttl setup 116 | * description : OPTIONAL : description of the ttl setup 117 | 118 | ### deleteDatasetExpiration 119 | 120 | You can cancel a dataset expiration by making a DELETE request.\ 121 | Arguments: 122 | * ttlId : REQUIRED : The ttlId of the dataset expiration that you want to cancel. 123 | 124 | 125 | ### createRecordDeleteRequest 126 | 127 | Delete records from a specific identity.\ 128 | **NOTE** : You should use the maximum number of identities in one request. Max is 100 K identities in the list.\ 129 | Argument: 130 | * datasetId : REQUIRED : default "ALL" for all dataset, otherwise a specific datasetId. 131 | * name : REQUIRED : Name of the deletion request job 132 | * identities : REQUIRED : list of namespace code and id to be deleted.\ 133 | example : 134 | ```python 135 | [{"namespace": { 136 | "code": "email" 137 | }, 138 | "id": "poul.anderson@example.com" 139 | }], 140 | ``` 141 | * description : OPTIONAL : Description of the job 142 | 143 | ### getWorkOrderStatus 144 | 145 | Return the status of a work order.\ 146 | Arguments: 147 | * workorderId : REQUIRED : The workorderId return by the job creation 148 | 149 | ### updateWorkOrder 150 | 151 | Update the work order\ 152 | Arguments: 153 | * workorderId : REQUIRED : The workorderId return by the job creation 154 | * name : REQUIRED : the new name of the work order 155 | * description : OPTIONAL : Description of the work order 156 | 157 | -------------------------------------------------------------------------------- /docs/logging.md: -------------------------------------------------------------------------------- 1 | # Logging in aepp 2 | 3 | The loggin capability enables you to log the activities of the modules in a file. 4 | 5 | ## Configuration 6 | 7 | You can enable logging capability when instantiating the different classes in the submodules. 8 | 9 | It requires to pass an object during the class instantiation process, to the parameter `loggingObject`.\ 10 | A template of the object can be created via the `generateLoggingObject` method, available directly in the `aepp` module.\ 11 | This method takes no argument and returns the following object: 12 | 13 | ```python 14 | {'level': 'WARNING', 15 | 'stream': True, 16 | 'file': False, 17 | 'format': '%(asctime)s::%(name)s::%(funcName)s::%(levelname)s::%(message)s::%(lineno)d', 18 | 'filename': 'aepp.log'} 19 | ``` 20 | 21 | Here are some description of the different keys: 22 | 23 | * level : Level of the logger to display information (NOTSET, DEBUG,INFO,WARNING,EROR,CRITICAL) 24 | * stream : If the logger should display print statements 25 | * file : If the logger should write the messages to a file 26 | * filename : name of the file where log are written 27 | * format : format of the logs in string or an instance of the logging.Formatter class. 28 | 29 | As pythonista may have realized, the logging capability has been created based on the native `logging` module of python.\ 30 | Documentation can be found [here](https://docs.python.org/3/library/logging.html).\ 31 | It also means that it doesn't require any new module import. 32 | 33 | Example of instantiation of a class with the logging object: 34 | 35 | ```python 36 | import aepp 37 | aepp.importConfigFile('myconfig.json') 38 | myLogging = aepp.generateLoggingObject() 39 | from aepp import catalog 40 | cat = catalog.Catalog(loggingObject=myLogging) 41 | 42 | ``` 43 | 44 | ## Capabilities 45 | 46 | The new functionality can either: 47 | 48 | * Stream data to the console. In that case, it will progressively replace some of the verbose option. 49 | 50 | * Write data to a log file. In that case, the default name of the log file will be `aepp.log`. You can change that in the config object. 51 | 52 | Be careful at which level you are instanciating the logging capability. The default level is `WARNING`. 53 | 54 | ## Format of the logs 55 | 56 | Here is the possible options for the format of the logs.\ 57 | Copy of the table available [here](https://docs.python.org/3/library/logging.html#:~:text=available%20to%20you.-,Attribute%20name,-Format) 58 | 59 | | Attribute name | Format | Description | 60 | | ------------------------|-----------------|---------------| 61 | | asctime | %(asctime)s | Human-readable time when the LogRecord was created. By default this is of the form '2003-07-08 16:49:45,896' | 62 | | created | %(created)f | Time when the LogRecord was created (as returned by `time.time())`. | 63 | | filename | %(filename)s | Filename portion of pathname. | 64 | | funcName | %(funcName)s | Name of function containing the logging call. | 65 | | levelname | %(levelname)s | Text logging level for the message ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'). | 66 | | levelno | %(levelno)s | Numeric logging level for the message (DEBUG, INFO, WARNING, ERROR, CRITICAL). | 67 | | lineno | %(lineno)d | Source line number where the logging call was issued (if available). | 68 | | message | %(message)s | The logged message, computed as msg % args. This is set when `Formatter.format()` is invoked. | 69 | | module | %(module)s | Module (name portion of filename). | 70 | | msecs | %(name)s | Millisecond portion of the time when the LogRecord was created. | 71 | | name | %(pathname)s | Name of the logger used to log the call. | 72 | | process | %(process)d | Process ID (if available). | 73 | | processName| %(processName)s| Process name (if available). | 74 | | relativeCreated| %(relativeCreated)d| Time in milliseconds when the LogRecord was created, relative to the time the logging module was loaded. | 75 | | thread | %(thread)d | Thread ID (if available). | 76 | | threadName| %(threadName)s| Thread name (if available). | 77 | -------------------------------------------------------------------------------- /docs/main.md: -------------------------------------------------------------------------------- 1 | # AEPP core methods 2 | 3 | This documentation is focused on the methods available directly from the `aepp` module. 4 | 5 | ## Creating a config file 6 | 7 | When starting with the `aepp` module, you will need to create a configuration file.\ 8 | The aepp module contains a method that will create a configuration file (JSON) as a template and you can just update the values.\ 9 | The method is : `createConfigFile`\ 10 | Arguments: 11 | 12 | * destination : OPTIONAL : if you wish to save the file at a specific location. 13 | * sandbox : OPTIONAL : You can directly set your sandbox name in this parameter. Default : `prod` 14 | * environment : OPTIONAL : This element is only for AEP core developer. **NOT TO BE CHANGED BY CLIENTS**. 15 | * auth_type : OPTIONAL : Default is OauthV2, but you can still use JWT (Deprecated in 2025) or OauthV1 (**for Internal only!**) 16 | * verbose : OPTIONAL : set to true, gives you a print statement where is the location. 17 | 18 | The JSON file is having this structure: 19 | 20 | ```python 21 | { 22 | "org_id": "", 23 | "client_id": "", 24 | "tech_id": "@techacct.adobe.com", 25 | "secret": "", 26 | "scopes": "scope", 27 | "sandbox-name": "prod", 28 | "environment" : "prod" 29 | } 30 | ``` 31 | 32 | Example: 33 | 34 | ```python 35 | import aepp 36 | aepp.createConfigFile(destination='myConfigFile.json') 37 | ``` 38 | 39 | ## Importing a config file 40 | 41 | Once you have created and updated the configuration file, you can (or need) to import it in order to have the information required for connecting to AEP.\ 42 | The method is the `importConfigFile`\ 43 | Argument: 44 | 45 | * path: REQUIRED : path to the configuration file. Can be either a fully-qualified or relative. 46 | * connectInstance : OPTIONAL : If you want to return an instance of the ConnectObject class. (default False) 47 | 48 | **NOTE**: `connectInstance` is default to `False`, but we strongly recommend to use it as best practice when you are having multiple sandbox environment. 49 | 50 | Example: 51 | 52 | ```py 53 | import aepp 54 | from aepp import schema ## to manipulate schema definition 55 | 56 | ## here I will create a connect instance to prod2 sandbox 57 | prod2 = aepp.importConfigFile('myconfig.json',sandbox='prod2',connectInstance=True) 58 | 59 | mySchema = schema.Schema(config=prod) 60 | ``` 61 | 62 | ## Configure connection 63 | 64 | The `configure` method directly available in the `aepp` module enables the possibility to pass all information required to connect to the AEP API without having to write them directly in a configuration file.\ 65 | This can be required when you are on the cloud on a stateless environment and you want to pass the connection info directly. 66 | 67 | Arguments: 68 | 69 | * org_id : REQUIRED : Organization ID 70 | * tech_id : REQUIRED : Technical Account ID 71 | * secret : REQUIRED : secret generated for your connection 72 | * client_id : REQUIRED : The client_id (old api_key) provided by the JWT connection. 73 | * scopes : REQUIRED : The scope used in the OauthV2 connection. 74 | * path_to_key : REQUIRED : If you have a file containing your private key value. (JWT connection only, deprecated in 2025) 75 | * private_key : REQUIRED : If you do not use a file but pass a variable directly.(JWT connection only, deprecated in 2025) 76 | * sandbox : OPTIONAL : If not provided, default to prod 77 | * connectInstance : OPTIONAL : If you want to return an instance of the ConnectObject class (default False) 78 | * environment : OPTIONAL : If not provided, default to prod 79 | 80 | ### The `connectInstance` parameter 81 | 82 | In an environment when you have multiple organization and / or multiple sandboxes to manage via `aepp`, it would be cumbersome to import the new environment any time you want to switch the Organization or the sandbox.\ 83 | For that use-case, we provide a way for you to save your configuration in an instance of a `ConnectObject` class.\ 84 | This class will save your organization, your sandbox and any information related to your configuration setup.\ 85 | Therefore, in instantiation of any class later on, such as Schema class per example, you can pass the appropriate instance to connect to the right organization. 86 | 87 | Example: 88 | 89 | ```python 90 | import aepp 91 | myOrg1 = aepp.importConfigFile('org1_config.json',connectInstance=True) 92 | myOrg2 = aepp.importConfigFile('org1_config.json',connectInstance=True) 93 | 94 | from aepp import catalog, schema 95 | 96 | ### conecting to the schema Registry endpoint for the org 1 97 | schema1 = schema.Schema(config=myOrg1) 98 | ## connecting for org 2 99 | schema2 = schema.Schema(config=myOrg2) 100 | 101 | ### Same for Catalog 102 | catalog2 = catalog.Catalog(config=myOrg2) 103 | catalog1 = catalog.Catalog(config=myOrg1) 104 | 105 | ``` 106 | 107 | ## Generating the logging object 108 | 109 | With the different submodule of `aepp`, you can generate logs information to monitor the state of your application running aepp.\ 110 | In order to pass how you want the log to be structured and which file to create for the log, you can create a logging object. 111 | The method is: `generateLoggingObject` 112 | 113 | A complete description of its usage is available on the [logging documentation](./logging.md) 114 | 115 | ## Home 116 | 117 | This method (`home`) provides information from your AEP setup.\ 118 | Arguments: 119 | 120 | * product : OPTIONAL : specify one or more product contexts for which to return containers. If absent, containers for all contexts that you have rights to will be returned. The product parameter can be repeated for multiple contexts. An example of this parameter is product=acp 121 | * limit : OPTIONAL : Optional limit on number of results returned (default = 50). 122 | 123 | Example: 124 | 125 | ```python 126 | import aepp 127 | aepp.importConfigFile('myConfig.json') 128 | 129 | conf = aepp.home() 130 | ``` 131 | 132 | ## Retrieve Users Log events 133 | 134 | The `getPlatformEvents` is a method that should return you with the information of what has been done during the last 90 days on your AEP instance by your users.\ 135 | Arguments: 136 | 137 | * limit : OPTIONAL : Number of events to retrieve per request (50 by default) 138 | * n_results : OPTIONAL : Number of total event to retrieve per request. 139 | * prop : OPTIONAL : An array that contains one or more of a comma-separated list of properties (prop="action==create,assetType==Sandbox") 140 | If you want to filter results using multiple values for a single filter, pass in a comma-separated list of values. (prop="action==create,update") 141 | -------------------------------------------------------------------------------- /docs/observability.md: -------------------------------------------------------------------------------- 1 | # Observability module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the `observability` module and different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/observability-insights/).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## Importing the module 9 | 10 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 11 | 12 | To import the module you can use the import statement with the `destination` keyword. 13 | 14 | ```python 15 | import aepp 16 | aepp.importConfigFile('myConfig_file.json') 17 | 18 | from aepp import observability 19 | ``` 20 | 21 | The destination module provides a class that you can use to generate a SDK taking care of transferring some information to specific destination endpoints.\ 22 | The following documentation will provide you with more information on its capabilities. 23 | 24 | ## The Observability class 25 | 26 | The Observability class allows you to discover statistics on the AEP processing.\ 27 | This class can be instantiated by calling the `Observability()` from the `observability` module. 28 | 29 | Following the previous method described above, you can realize this: 30 | 31 | ```python 32 | obs = observability.Observability() 33 | ``` 34 | 35 | 2 parameters are possible for the instantiation of the class: 36 | 37 | * config : OPTIONAL : config object in the config module. (example : aepp.config.config_object) 38 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 39 | * loggingObject : OPTIONAL : logging object to provide log of the application. 40 | -------------------------------------------------------------------------------- /docs/privacyservice.md: -------------------------------------------------------------------------------- 1 | # Privacy Service module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the Privacy Service module and different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the schema API, please refer to this [API documentation](https://developer.adobe.com/experience-platform-apis/references/privacy-service/).\ 6 | Alternatively, you can use the docstring in the methods to have more information. 7 | 8 | ## Important Information about the Privacy Service API 9 | 10 | The privacy service API is not really part of the AEP APIs but as it is described in the same page, there is here a support for it.\ 11 | If you wish to have access to this API, you will need to add a new service in your environment of Adobe.io.\ 12 | In console.adobe.io, you will need to add the Privacy Service element. 13 | 14 | ![Privacy Service](add-privacy-service-api.png) 15 | 16 | ## What is the Privacy Service ? 17 | 18 | The Privacy Service API is not really part of the AEP API as described above. It allows you to generate request to delete users data as they request it from the different regulation (GDPR, CCPA, ...).\ 19 | In order to achieve the requests, you will need to have the different user ID when the request is generated by the user. In order to help you in that task, Adobe provides an additional (JS) library that you can have on your website. 20 | 21 | [The privacy JS library](https://experienceleague.adobe.com/docs/experience-platform/privacy/js-library.html?lang=en) is the library that you can run on the browser in order to retrieve the different identities requires for the request to have the information required to run the Privacy Service requests. 22 | 23 | The Privacy Service is not only taking care of the Adobe Experience Platform data. It can take care of the data for the following different solutions: 24 | * Adobe Advertising Cloud 25 | * Adobe Analytics 26 | * Adobe Audience Manager 27 | * Adobe Campaign Standard 28 | * Adobe Customer Attributes (CRS) 29 | * Adobe Primetime Authentication 30 | * Adobe Target 31 | For more information, you can read the documentation about [supported tools](https://experienceleague.adobe.com/docs/experience-platform/privacy/experience-cloud-apps.html?lang=en#self-serve). 32 | 33 | ## Importing the module 34 | 35 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md).\ 36 | Note that on top of the default setting, you need to have added the Privacy Service in Adobe.io. 37 | 38 | To import the module you can use the import statement with the `privacyservice` keyword. 39 | 40 | ```python 41 | import aepp 42 | aepp.importConfigFile('myConfig_file.json') 43 | 44 | from aepp import privacyservice 45 | ``` 46 | 47 | The `privacyservice` module provides a class that will generate the connection to the API (see below).\ 48 | The following documentation will provide you with more information on its capabilities. 49 | 50 | ## The Privacy class 51 | 52 | The `Privacy` class uses the default API connector that you would encounter for any other submodules on this python module.\ 53 | This class can be instantiated by calling the `Privacy()` from the `privacyservice` module. 54 | 55 | Following the previous method described above, you can realize this: 56 | 57 | ```python 58 | privacy = privacyservice.Privacy() 59 | ``` 60 | 61 | 2 parameters are possible for the instantiation of the class: 62 | 63 | * privacyScope : REQUIRED : set the connection retrieved process with the Privacy JWT scope (default True) 64 | * aepScope : OPTIONAL : set the connection retrieved process with the AEP JWT scope if set to True (default False) 65 | * config : OPTIONAL : config object in the config module. (example : aepp.config.config_object) 66 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 67 | 68 | ## Use-cases 69 | 70 | ### Sample Payload 71 | 72 | The `Privacy` class, once instantiated provde a sample payload that you can use for your different requests.\ 73 | The payload is hosted on the `SAMPLE_PAYLOAD` attribute.\ 74 | It gives you the following elements: 75 | 76 | ```python 77 | { 78 | "companyContexts": [ 79 | { 80 | "namespace": "imsOrgID", 81 | "value": "{IMS_ORG}" 82 | } 83 | ], 84 | "users": [ 85 | { 86 | "key": "DavidSmith", 87 | "action": ["access"], 88 | "userIDs": [ 89 | { 90 | "namespace": "email", 91 | "value": "dsmith@acme.com", 92 | "type": "standard" 93 | }, 94 | { 95 | "namespace": "ECID", 96 | "type": "standard", 97 | "value": "443636576799758681021090721276", 98 | "isDeletedClientSide": False 99 | } 100 | ] 101 | }, 102 | { 103 | "key": "user12345", 104 | "action": ["access","delete"], 105 | "userIDs": [ 106 | { 107 | "namespace": "email", 108 | "value": "ajones@acme.com", 109 | "type": "standard" 110 | }, 111 | { 112 | "namespace": "loyaltyAccount", 113 | "value": "12AD45FE30R29", 114 | "type": "integrationCode" 115 | } 116 | ] 117 | } 118 | ], 119 | "include": ["Analytics", "AudienceManager"], 120 | "expandIds": False, 121 | "priority": "normal", 122 | "analyticsDeleteMethod": "anonymize", 123 | "regulation": "ccpa" 124 | } 125 | ``` 126 | 127 | ### Create Jobs & append 128 | 129 | You can create privacy requests by using the `postJob` method that is provided in the instance of the class.\ 130 | It takes only one argumentm, the dictionary that is provided to you can help you create the dictionary. 131 | 132 | You can have a look at this documentation to help you define the product values and Namespace qualifiers: [Appendix Privacy Service](https://experienceleague.adobe.com/docs/experience-platform/privacy/api/appendix.html?lang=en#namespace-qualifiers) 133 | 134 | Example: 135 | ```python 136 | ## all previous initialization are skipped here. 137 | privacy = privacyservice.Privacy() 138 | 139 | dictDefinition = { 140 | "companyContexts": [ 141 | { 142 | "namespace": "AGS 862", 143 | "value": "12DSQ234SUFE@adobe.com" 144 | } 145 | ], 146 | "users": [ 147 | { 148 | "key": "DavidSmith", 149 | "action": ["access"], 150 | "userIDs": [ 151 | { 152 | "namespace": "email", 153 | "value": "dsmith@acme.com", 154 | "type": "standard" 155 | }, 156 | { 157 | "namespace": "ECID", 158 | "type": "standard", 159 | "value": "443636576799758681021090721276", 160 | "isDeletedClientSide": False 161 | } 162 | ] 163 | }, 164 | { 165 | "key": "user12345", 166 | "action": ["access","delete"], 167 | "userIDs": [ 168 | { 169 | "namespace": "email", 170 | "value": "ajones@acme.com", 171 | "type": "standard" 172 | }, 173 | { 174 | "namespace": "loyaltyAccount", 175 | "value": "12AD45FE30R29", 176 | "type": "integrationCode" 177 | } 178 | ] 179 | } 180 | ], 181 | "include": ["AdobeCloudPlatform"], 182 | "expandIds": False, 183 | "priority": "normal", 184 | "analyticsDeleteMethod": "anonymize", 185 | "regulation": "gdpr" 186 | } 187 | 188 | myJob = privacy.postJob(dictDefinition) 189 | ``` 190 | 191 | The response hosted in `myJob` will be like the following one: 192 | 193 | ```JSON 194 | { 195 | "jobs": [ 196 | { 197 | "jobId": "6fc09b53-c24f-4a6c-9ca2-c6076b0842b6", 198 | "customer": { 199 | "user": { 200 | "key": "DavidSmith", 201 | "action": [ 202 | "access" 203 | ] 204 | } 205 | } 206 | }, 207 | { 208 | "jobId": "6fc09b53-c24f-4a6c-9ca2-c6076be029f3", 209 | "customer": { 210 | "user": { 211 | "key": "user12345", 212 | "action": [ 213 | "access" 214 | ] 215 | } 216 | } 217 | }, 218 | { 219 | "jobId": "6fc09b53-c24f-4a6c-9ca2-c6076bd023j1", 220 | "customer": { 221 | "user": { 222 | "key": "user12345", 223 | "action": [ 224 | "delete" 225 | ] 226 | } 227 | } 228 | } 229 | ], 230 | "requestStatus": 1, 231 | "totalRecords": 3 232 | } 233 | ``` 234 | 235 | You can use the `jobId` to add additional element to that job, by using the `postChildJob` method. 236 | 237 | ### Get Jobs 238 | 239 | The 2nd main use-case of that API is to check on the status of the job.\ 240 | Once you have received a `jobId`, you can check the status of the job with it by using the `getJob` method. 241 | 242 | `privacy.getJob("6fc09b53-c24f-4a6c-9ca2-c6076bd023j1")` 243 | 244 | Alternatively, you can use the `getJobs()` method to retrieve all jobs of your organization. -------------------------------------------------------------------------------- /docs/release-build.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/docs/release-build.png -------------------------------------------------------------------------------- /docs/release-upload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/docs/release-upload.png -------------------------------------------------------------------------------- /docs/synchronizer.md: -------------------------------------------------------------------------------- 1 | # Synchronizer (BETA) 2 | 3 | The `synchronizer` is a sub module that lives on top of several sub modules of aepp (schema, schemamanger,fieldgroupmanager, datatypemanager, classmanager, catalog, identity).\ 4 | **NOTE** : The synchronizer module is currently a **work in progress** and is expected to support more capabilities in the future. Some elements could change in the future and the module is not stable since stated otherwise here. 5 | 6 | The module is intended to create or update elements between sandboxes within an organization. 7 | The current supported artefacts for the synchronization job are: 8 | * data type 9 | * field group 10 | * schema 11 | * class 12 | * descriptors 13 | * identity 14 | * datasets 15 | 16 | 17 | ## Synchronizer class 18 | 19 | The `Synchronizer` class is part of the `synchronizer` module and it takes the following elements as a parameter: 20 | * baseSandbox : REQUIRED : name of the base sandbox 21 | * targets : REQUIRED : list of target sandboxes name as strings 22 | * config : REQUIRED : ConnectObject with the configuration. Make sure that the configuration of your API allows connection to all targeted sandboxes. 23 | * region : OPTIONAL : region of the sandboxes. default is 'nld2', possible values are: "va7" or "aus5 or "can2" or "ind2" 24 | * localEnvironment : OPTIONAL : if True, it will use the local environment. Default is False. ## WIP 25 | 26 | ### Synchronizer attributes 27 | 28 | Once instantiated the synchronizer object will contains certain attributes: 29 | * baseConfig : the config object with the base sandbox configuration to connect to the base sandbox (a `ConnectInstance` [instance](/getting-started.md#the-connectInstance-parameter)) 30 | * dict_targetsConfig : A dictionary of the different target sandbox configuration object (children of `ConnectInstance` class) 31 | * region : The region used for the Identity Management for the Target and base Sandbox 32 | * dict_targetComponents : A dictionary of the target components that has been created. A cache mechanisme to optimize the future usage of these components in the future. 33 | 34 | 35 | ### Synchronizer methods: 36 | 37 | The following methods are available once you have instantianted the `Synchronizer` class. 38 | 39 | #### syncComponent 40 | Synchronize a component to the target sandbox(es).\ 41 | The component could be a string (name or id of the component in the base sandbox) or a dictionary with the definition of the component.\ 42 | If the component is a string, you have to have provided a base sandbox in the constructor.\ 43 | Arguments: 44 | * component : REQUIRED : name or id of the component or a dictionary with the component definition 45 | * componentType : OPTIONAL : type of the component (e.g. "schema", "fieldgroup", "datatypes", "class", "identity", "dataset"). Required if a string is passed.\ 46 | It is not required but if the type cannot be inferred from the component, it will raise an error. 47 | * verbose : OPTIONAL : if True, it will print the details of the synchronization process 48 | 49 | 50 | ## Notes on Synchronization capabilities 51 | 52 | The synchronization capabilities are very similar to the sandbox tooling. 53 | 54 | Due to the potential issue with ID management, the synchronizer bases its capabilities on name of the artefact.\ 55 | It means that the **name** of the schema, class, field group, data type, dataset, identity namespace are used. 56 | 57 | As of today, the synchronization will realize the following operation for the different artefacts: 58 | 59 | Operation |Schema | Class | Field Groups | Data Type | Descriptors | Dataset | Identity | Tags | 60 | --| -- | -- | -- | -- | -- | -- | -- | -- | 61 | Create | Supported | Supported | Supported | Supported | Supported | Supported | Supported | Planned | 62 | Update | Supported | Supported | Supported | Supported | Suppported | - | - | Planned | 63 | Delete | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | Not Supported | 64 | 65 | It is not supported to delete an artefact or delete a field in an Field Group or Data Type via the Synchronizer.\ 66 | The synchronizer only supports additive operations 67 | 68 | The synchronizer will automatically resolve the dependency to create the elements require for the artefact used.\ 69 | Example:\ 70 | Synchronizing a dataset will automatically synchronize the underlying schema and the different field groups.\ 71 | If the schema is in a relationship with another schema (lookup), the associated lookup schema will also be created and the associated created. (note: The dataset associated with the lookup schema won't be created) 72 | 73 | ### Create 74 | 75 | For all artefacts, if the element does not exist in the target sandbox, it will automatically create it.\ 76 | The synchronizer automatically resolves all dependencies, which mean that the associated elements Schema associated to a dataset, or field group associated to a schema or a data type associated to a field groups are automatically created as well. 77 | 78 | As of today, the schema and datasets are not enabled for profile per default during creation. 79 | 80 | 81 | ### Update 82 | 83 | The **Update** operation is provided the capacity to add new fields to `field groups` or `data type` in the base and replicate that change to the target change.\ 84 | The removal of fields are not supported as it could be a breaking change in the target sandboxes. 85 | 86 | It also supports the addition of a field group to a schema and replicate that change to all target sandboxes. 87 | 88 | 89 | ## Incoming features 90 | 91 | * Tags (dataset) 92 | * Audiences 93 | * Local file system as source 94 | * Profile enabling capabilities 95 | 96 | 97 | -------------------------------------------------------------------------------- /docs/tags.md: -------------------------------------------------------------------------------- 1 | # Tags module in aepp 2 | 3 | This documentation will provide you some explanation on how to use the tags module and the different methods supported by this module.\ 4 | It will include some examples but be aware that not all methods will be documented here.\ 5 | To have a full view on the different API endpoints specific to the tags API, please refer to this API documentation. (to be published)\ 6 | Alternatively, you can use the docstring in the methods to have more information.\ 7 | 8 | ## Menu 9 | 10 | - [Tags module in aepp](#tags-module-in-aepp) 11 | - [Menu](#menu) 12 | - [Importing the module](#importing-the-module) 13 | - [The Tags class](#the-sandboxes-class) 14 | - [Tags attributes](#sandboxes-attributes) 15 | - [Tags methods](#sandboxes-methods) 16 | 17 | 18 | ## Importing the module 19 | 20 | Before importing the module, you would need to import the configuration file, or alternatively provide the information required for the API connection through the configure method. [see getting starting](./getting-started.md) 21 | 22 | To import the module you can use the import statement with the `tags` keyword. 23 | 24 | ```python 25 | import aepp 26 | prod = aepp.importConfigFile('myConfig_file.json',connectInstance=True,sandbox='prod') 27 | 28 | from aepp import tags 29 | ``` 30 | 31 | The tags module provides a class `Tags` that you can use for generating and retrieving tags.\ 32 | The following documentation will provide you with more information on its usage. 33 | 34 | ## The Tags class 35 | 36 | The `Tags` class is generating a connection to use the different methods directly on your AEP sandbox / instance.\ 37 | This class can be instantiated by calling the `Tags()` from the `tags` module. 38 | 39 | Following the previous method described above, you can realize this: 40 | 41 | ```python 42 | import aepp 43 | from aepp import tags 44 | 45 | prod = aepp.importConfigFile('myConfig_file.json',connectInstance=True,sandbox='prod') 46 | myTags = tags.Tags(config=prod) 47 | ``` 48 | 49 | 3 parameters are possible for the instantiation of the class: 50 | 51 | * config : OPTIONAL : the connect object instance created when you use `importConfigFile` with connectInstance parameter. Default to latest loaded configuration. 52 | * header : OPTIONAL : header object in the config module. (example: aepp.config.header) 53 | * loggingObject : OPTIONAL : logging object to provide log of the application. 54 | 55 | **Note**: Kwargs can be used to update the header used in the connection. 56 | 57 | ## Tags attributes 58 | 59 | Once you have instantiated the `Tags` class, you have access to some attributes: 60 | 61 | * sandbox : provide which sandbox is currently being used 62 | * header : provide the default header which is used for the requests. 63 | * loggingEnabled : if the logging capability has been used 64 | * endpoint : the default endpoint used for all methods. 65 | 66 | ## Tags methods 67 | 68 | The following elements are all the methods available once you have instantiated the `Tags` class. 69 | 70 | ### getCategories 71 | Retrieve the categories of the tags. 72 | 73 | ### getCategory 74 | Retrieve the tag category based on the ID passed.\ 75 | Arguments: 76 | * tagCategoryId : REQUIRED : The Id of the tag category to retrieve 77 | 78 | 79 | ### createCategory 80 | Create a tag category\ 81 | Arguments: 82 | * name : REQUIRED : name of the category 83 | * description : OPTIONAL : description of the category 84 | 85 | 86 | ### patchCategory 87 | Patch the category with new definition\ 88 | Arguments: 89 | * tagCategoryId : REQUIRED : The ID of the category to update 90 | * operation : OPTIONAL : A dictionary that provides the operation to performed\ 91 | ex: 92 | ```python 93 | { 94 | "op": "replace", 95 | "path": "description", 96 | "value": "Updated sample description" 97 | } 98 | ``` 99 | * op : OPTIONAL : In case the individual value for "op" in the operation is provided. Possible value: "replace" 100 | * path : OPTIONAL : In case the individual value for "path" in the operation is provided. 101 | * value : OPTIONAL : In case the individual value for "value" in the operation is provided. 102 | 103 | ### deleteTagCategory 104 | Delete the tag category based on its ID.\ 105 | Arguments: 106 | * tagCategoryId : REQUIRED : Tag Category ID to be deleted 107 | 108 | ### getTags 109 | Retrieve a list of tag based on the categoryId\ 110 | Arguments: 111 | * tagCategoryId : OPTIONAL : The id of the category to get your tags 112 | 113 | ### getTag 114 | Retrieve a specific tag based on its ID.\ 115 | Argument: 116 | * tagId : REQUIRED : The tag ID to be used 117 | 118 | ### createTag 119 | Create a new tag.\ 120 | Arguments: 121 | * name : REQUIRED : Name of the tag 122 | * tagCategoryId : OPTIONAL : The category ID of the tag 123 | 124 | 125 | ### patchTag 126 | Update a specific Tag\ 127 | Arguments: 128 | * tagId : REQUIRED : The tag Id to be updated 129 | * operation : OPTIONAL : The full operation dictionary 130 | ex: 131 | ```python 132 | { 133 | "op": "replace", 134 | "path": "description", 135 | "value": "Updated sample description" 136 | } 137 | ``` 138 | * op : OPTIONAL : In case the individual value for "op" in the operation is provided. default value: "replace" 139 | * path : OPTIONAL : In case the individual value for "path" in the operation is provided. 140 | * value : OPTIONAL : In case the individual value for "value" in the operation is provided. 141 | 142 | ### deleteTag 143 | Delete a tag by its ID.\ 144 | Arguments: 145 | * tagId : REQUIRED : The Tag Id to be deleted 146 | 147 | 148 | ### validateTags 149 | Validate if specific tag Ids exist.\ 150 | Arguments: 151 | * tagsIds : REQUIRE : List of tag Ids to validate 152 | 153 | ### getFolders 154 | Retrieve the folders for the tags.\ 155 | Arguments: 156 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 157 | 158 | ### getSubFolders 159 | Return the list of subfolders.\ 160 | Arguments: 161 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 162 | * folderId : REQUIRED : The folder ID that you want to retrieve 163 | 164 | ### getSubFolder 165 | Return a specific sub folder\ 166 | Arguments: 167 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 168 | * folderId : REQUIRED : The folder ID that you want to retrieve 169 | 170 | ### deleteSubFolder 171 | Delete a specific subFolder\ 172 | Arguments: 173 | * folderType : REQUIRED : Default "segment", possible values: "datasets" 174 | * folderId : REQUIRED : The folder ID you want to delete 175 | 176 | ### createSubFolder 177 | Create a sub Folder.\ 178 | Arguments: 179 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 180 | * name : REQUIRED : Name of the folder 181 | * parentId : REQUIRED : The parentID attached to your folder 182 | 183 | ### updateFolder 184 | Update an existing folder name\ 185 | Arguments: 186 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 187 | * folderId : REQUIRED : the folder ID you want to rename 188 | * name : OPTIONAL : The new name you want to give that folder 189 | * parentFolderId : OPTIONAL : The new parent folder id 190 | 191 | ### validateFolder 192 | Validate if a folder is eligible to have objects in it\ 193 | Arguments: 194 | * folderType : REQUIRED : Default "segment", possible values: "dataset" 195 | * folderId : REQUIRED : The Folder ID 196 | 197 | -------------------------------------------------------------------------------- /notebooks/developer.adobe.project_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/notebooks/developer.adobe.project_list.png -------------------------------------------------------------------------------- /notebooks/developer.adobe.project_oauth_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/notebooks/developer.adobe.project_oauth_detail.png -------------------------------------------------------------------------------- /notebooks/developer.adobe.project_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/notebooks/developer.adobe.project_overview.png -------------------------------------------------------------------------------- /notebooks/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "org_id": "", 3 | "client_id": "", 4 | "secret": "", 5 | "sandbox-name": "prod", 6 | "environment": "prod", 7 | "scopes": "" 8 | } -------------------------------------------------------------------------------- /notebooks/profile-entity-composition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pitchmuc/aepp/62bb74ed0c0580fcb63710266613169e7fcf77d2/notebooks/profile-entity-composition.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | requests 3 | pandas 4 | PyJWT[crypto]>=1.7.1 5 | PyJWT>=1.7.1 6 | tenacity 7 | deprecation 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | import codecs 12 | import os 13 | 14 | import setuptools 15 | 16 | def read(rel_path: str): 17 | here = os.path.abspath(os.path.dirname(__file__)) 18 | with codecs.open(os.path.join(here, rel_path), 'r') as fp: 19 | return fp.read() 20 | 21 | 22 | def get_version(rel_path: str): 23 | for line in read(rel_path).splitlines(): 24 | if line.startswith('__version__'): 25 | delim = '"' if '"' in line else "'" 26 | return line.split(delim)[1] 27 | else: 28 | raise RuntimeError("Unable to find version string.") 29 | 30 | with open("README.md", "r") as fh: 31 | long_description = fh.read() 32 | 33 | setuptools.setup( 34 | name="aepp", # Replace with your own username 35 | version=get_version("aepp/__version__.py"), 36 | author="Julien Piccini", 37 | author_email="piccini.julien@gmail.com", 38 | description="Package to manage AEP API endpoint and some helper functions", 39 | long_description=long_description, 40 | long_description_content_type="text/markdown", 41 | url="https://github.com/adobe/aepp", 42 | packages=setuptools.find_packages(), 43 | include_package_data=True, 44 | classifiers=[ 45 | "Programming Language :: Python :: 3", 46 | "Operating System :: OS Independent", 47 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 48 | "Topic :: Utilities", 49 | "Topic :: Internet", 50 | "Topic :: Software Development :: Libraries", 51 | "Development Status :: 2 - Pre-Alpha" 52 | ], 53 | python_requires='>=3.10', 54 | install_requires=['pandas', "requests", 55 | "PyJWT", "pathlib2", "PyJWT[crypto]", "tenacity", "deprecation"], 56 | ) 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | -------------------------------------------------------------------------------- /tests/catalog_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.schema import Schema 12 | import unittest 13 | from unittest.mock import patch, MagicMock 14 | 15 | 16 | class CatalogTest(unittest.TestCase): 17 | 18 | def test_catalog_get_catalog__resource(self): 19 | assert True 20 | 21 | def test_catalog_decode_stream_batch(self): 22 | assert True 23 | 24 | 25 | def test_catalog_json_stream_messages(self): 26 | assert True 27 | 28 | def test_catalog_get_batches(self): 29 | assert True 30 | 31 | def test_catalog_get_failed_batches_df(self): 32 | assert True 33 | 34 | def test_catalog_get_batch(self): 35 | assert True 36 | 37 | def test_catalog_create_batch(self): 38 | assert True 39 | 40 | def test_catalog_get_resources(self): 41 | assert True 42 | 43 | def test_catalog_get_data_sets(self): 44 | assert True 45 | 46 | def test_catalog_create_datasets(self): 47 | assert True 48 | 49 | def test_catalog_get_data_set(self): 50 | assert True 51 | 52 | def test_catalog_delete_data_set(self): 53 | assert True 54 | 55 | def test_catalog_get_data_set_views(self): 56 | assert True 57 | 58 | def test_catalog_get_data_set_view(self): 59 | assert True 60 | 61 | def test_catalog_get_dataset_view_files(self): 62 | assert True 63 | 64 | def test_catalog_enable_dataset_profile(self): 65 | assert True 66 | 67 | def test_catalog_enable_dataset_identity(self): 68 | assert True 69 | 70 | def test_catalog_disable_dataset_profile(self): 71 | assert True 72 | 73 | def test_catalog_disable_dataset_identity(self): 74 | assert True 75 | 76 | def test_catalog_create_union_profile_dataset(self): 77 | assert True 78 | 79 | def test_catalog_get_mapper_errors(self): 80 | assert True -------------------------------------------------------------------------------- /tests/dataaccess_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.schema import Schema 12 | import unittest 13 | from unittest.mock import patch, MagicMock 14 | 15 | class DataAccessTest(unittest.TestCase): 16 | 17 | def test_dataaccess_get_batch_files(self): 18 | assert True 19 | 20 | def test_dataaccess_get_batch_failed(self): 21 | assert True 22 | 23 | def test_dataaccess_get_batch_meta(self): 24 | assert True 25 | 26 | def test_dataaccess_getHeadFile(self): 27 | assert True 28 | 29 | def test_dataaccess_get_files(self): 30 | assert True 31 | 32 | def test_dataaccess_get_preview(self): 33 | assert True 34 | 35 | def test_dataaccess_get_resource(self): 36 | assert True -------------------------------------------------------------------------------- /tests/datasets_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.schema import Schema 12 | import unittest 13 | from unittest.mock import patch, MagicMock 14 | 15 | class DatasetsTest(unittest.TestCase): 16 | 17 | def test_datasets_get_label_schema(self): 18 | assert True 19 | 20 | def test_datasets_head_label(self): 21 | assert True 22 | 23 | def test_datasets_delete_labels(self): 24 | assert True 25 | 26 | def test_datasets_create_labels(self): 27 | assert True 28 | 29 | def test_datasets_update_labels(self): 30 | assert True 31 | -------------------------------------------------------------------------------- /tests/destinationinstanceservice_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.destinationinstanceservice import DestinationInstanceService 12 | import unittest 13 | from unittest.mock import patch, MagicMock, ANY 14 | 15 | 16 | class DestinationInstanceServiceTest(unittest.TestCase): 17 | ADHOC_INPUT = {"flow1": ["dataset1"], "flow2": ["dataset2", "dataset3"]} 18 | ADHOC_EXPECTED_PAYLOAD = {'activationInfo': {'destinations': [{'flowId': 'flow1', 'datasets': [{'id': 'dataset1'}]}, {'flowId': 'flow2', 'datasets': [{'id': 'dataset2'}, {'id': 'dataset3'}]}]}} 19 | 20 | @patch("aepp.connector.AdobeRequest") 21 | def test_create_adhoc_dataset_export(self, mock_connector): 22 | instance_conn = mock_connector.return_value 23 | instance_conn.postData.return_value = {'foo'} 24 | destination_instance_service_obj = DestinationInstanceService() 25 | result = destination_instance_service_obj.createAdHocDatasetExport(self.ADHOC_INPUT) 26 | assert(result is not None) 27 | instance_conn.postData.assert_called_once() 28 | instance_conn.postData.assert_called_with(ANY, data=self.ADHOC_EXPECTED_PAYLOAD) 29 | 30 | @patch("aepp.connector.AdobeRequest") 31 | def test_create_adhoc_dataset_export_invalid_input(self, mock_connector): 32 | destination_instance_service_obj = DestinationInstanceService() 33 | with self.assertRaises(Exception) as cm: 34 | destination_instance_service_obj.createAdHocDatasetExport(None) 35 | self.assertEqual('Require a dict for defining the flowId to datasetIds mapping', str(cm.exception)) 36 | -------------------------------------------------------------------------------- /tests/exportDatasetToDatalandingZone_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.exportDatasetToDataLandingZone import ExportDatasetToDataLandingZone 12 | import unittest 13 | from unittest.mock import patch, MagicMock, ANY 14 | from tenacity import RetryError 15 | 16 | 17 | class ExportDatasetToDataLandingZoneTest(unittest.TestCase): 18 | flow_response = { 19 | "id": "df193495-9837-407f-9d50-e51efc067cb5", 20 | "sourceConnectionIds": [ 21 | "78754315-852c-4c4e-8f1a-d12b83264f3f" 22 | ], 23 | "state": "disabled", 24 | "etag": "test_etag"} 25 | 26 | connection_spec = { 27 | "name": "test", 28 | "id": "test_connection_id" 29 | } 30 | 31 | 32 | source_connection = { 33 | "id": "test_source_connection_id", 34 | "params": { 35 | "datasets": [{ 36 | "dataSetId": "test" 37 | }]} 38 | } 39 | 40 | base_connection = { 41 | "id": "test_base_connection_id" 42 | } 43 | 44 | target_connection = { 45 | "id": "test_target_connection_id" 46 | } 47 | 48 | flow_run = { 49 | "items": [ 50 | { 51 | "id": "test_id", 52 | "metrics": { 53 | "durationSummary": { 54 | "startedAtUTC": 1691184699, 55 | "completedAtUTC": 1691184819 56 | }, 57 | "fileSummary": { 58 | "outputFileCount": 24 59 | }, 60 | "sizeSummary": { 61 | "outputBytes": 77494320 62 | }, 63 | "recordSummary": { 64 | "outputRecordCount": 1065802 65 | } 66 | } 67 | } 68 | ] 69 | } 70 | 71 | adhoc_success_response = { 72 | "destinations":[ 73 | { 74 | "datasets":[ 75 | { 76 | "id":"641ce00b8f31811b98dd3b56", 77 | "statusURL":"https: //platform-stage.adobe.io/data/foundation/flowservice/runs/aa39ad3d-24ba-4579-9cdc-55536c408721", 78 | "flowId":"3eaa2c0b-e24b-46bd-8eee-bbaf9d6cf2cb" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | 85 | 86 | adhoc_non_retry_error = { 87 | "error_code": "401013", 88 | "message": "Oauth token is not valid" 89 | } 90 | 91 | adhoc_retry_error = { 92 | "message": "Following order ID(s) are not ready for dataset export" 93 | } 94 | 95 | config = { 96 | "org_id": "3ADF23C463D98F640A494032@AdobeOrg", 97 | "client_id": "35e6e4d205274c4ca1418805ac41153b", 98 | "tech_id": "test005@techacct.adobe.com", 99 | "pathToKey": "/Users/Downloads/config/private.key", 100 | "auth_code": "", 101 | "secret": "test", 102 | "date_limit": 0, 103 | "sandbox": "prod", 104 | "environment": "stage", 105 | "token": "token", 106 | "jwtTokenEndpoint": "https://ims-na1.adobelogin.com/ims/exchange/jwt/", 107 | "oauthTokenEndpoint": "", 108 | "imsEndpoint": "https://ims-na1-stg1.adobelogin.com", 109 | "private_key": "" 110 | } 111 | 112 | @patch('aepp.utils.Utils.check_if_exists', MagicMock(return_value = "test_dataflow_id")) 113 | @patch('aepp.flowservice.FlowService.getFlow', MagicMock(return_value = flow_response)) 114 | @patch('aepp.flowservice.FlowService.getSourceConnection', MagicMock(return_value = source_connection)) 115 | @patch('aepp.flowservice.FlowService.getRun', MagicMock(return_value = flow_run)) 116 | @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_success_response)) 117 | @patch("aepp.connector.AdobeRequest", MagicMock()) 118 | def test_create_dataflow_if_exist(self): 119 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 120 | export_obj.createDataFlowRunIfNotExists(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=False, config_path="test", entity_name="test", initial_delay=0) 121 | 122 | @patch('aepp.utils.Utils.check_if_exists', MagicMock(return_value = "test_dataflow_id")) 123 | @patch('aepp.flowservice.FlowService.getFlow', MagicMock(return_value = flow_response)) 124 | @patch('aepp.flowservice.FlowService.getSourceConnection', MagicMock(return_value = source_connection)) 125 | @patch("aepp.connector.AdobeRequest", MagicMock()) 126 | def test_create_dataflow_on_schedule(self): 127 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 128 | export_obj.createDataFlowRunIfNotExists(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=True, config_path="test",entity_name="test", initial_delay=0) 129 | 130 | @patch('aepp.flowservice.FlowService.createConnection', MagicMock(return_value = base_connection)) 131 | @patch('aepp.flowservice.FlowService.getConnectionSpecIdFromName', MagicMock(return_value = connection_spec)) 132 | @patch('aepp.flowservice.FlowService.createSourceConnection', MagicMock(return_value = source_connection)) 133 | @patch('aepp.flowservice.FlowService.createTargetConnection', MagicMock(return_value = target_connection)) 134 | @patch('aepp.flowservice.FlowService.createFlow', MagicMock(return_value = flow_response)) 135 | @patch('aepp.utils.Utils.save_field_in_config', MagicMock()) 136 | @patch("aepp.connector.AdobeRequest", MagicMock()) 137 | def test_create_dataflow_if_not_exist(self): 138 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 139 | return_dataflow_id = export_obj.createDataFlow(dataset_id="test", compression_type="gzip", data_format="parquet", export_path="test", on_schedule=False, config_path="test", entity_name="test") 140 | assert (return_dataflow_id == self.flow_response["id"]) 141 | 142 | @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_success_response)) 143 | @patch("aepp.connector.AdobeRequest", MagicMock()) 144 | def test_retry_on_success_response(self): 145 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 146 | assert(export_obj.retryOnNotReadyException("test", "test", 1, 1) == self.adhoc_success_response) 147 | 148 | @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_non_retry_error)) 149 | @patch("aepp.connector.AdobeRequest", MagicMock()) 150 | def test_no_retry_error(self): 151 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 152 | assert (export_obj.retryOnNotReadyException("test", "test", 1, 1) == self.adhoc_non_retry_error) 153 | 154 | @patch('aepp.destinationinstanceservice.DestinationInstanceService.createAdHocDatasetExport', MagicMock(return_value = adhoc_retry_error)) 155 | @patch("aepp.connector.AdobeRequest", MagicMock()) 156 | def test_retry_error(self): 157 | export_obj = ExportDatasetToDataLandingZone(config= self.config, header= MagicMock()) 158 | try: 159 | export_obj.retryOnNotReadyException("test", "test", 1, 1) 160 | self.fail("expect a retry error") 161 | except RetryError: 162 | pass -------------------------------------------------------------------------------- /tests/flowservice_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.schema import Schema 12 | import unittest 13 | from unittest.mock import patch, MagicMock 14 | 15 | 16 | class FlowserviceTest(unittest.TestCase): 17 | 18 | def test_flowservice_get_resource(self): 19 | assert True 20 | 21 | def test_flowservice_get_connections(self): 22 | assert True 23 | 24 | def test_flowservice_create_connection(self): 25 | assert True 26 | 27 | def test_flowservice_create_streaming_connection(self): 28 | assert True 29 | 30 | def test_flowservice_get_connection(self): 31 | assert True 32 | 33 | def test_flowservice_connection(self): 34 | assert True 35 | 36 | def test_flowservice_delete_connection(self): 37 | assert True 38 | 39 | def test_flowservice_get_connection_specs(self): 40 | assert True 41 | 42 | def test_flowservice_get_connection_specs_map(self): 43 | assert True 44 | 45 | def test_flowservice_get_connection_spec(self): 46 | assert True 47 | 48 | def test_flowservice_get_connection_specid_from_name(self): 49 | assert True 50 | 51 | def test_flowservice_get_flows(self): 52 | assert True 53 | 54 | def test_flowservice_get_flow(self): 55 | assert True 56 | 57 | def test_flowservice_delete_flow(self): 58 | assert True 59 | 60 | def test_flowservice_create_flow(self): 61 | assert True 62 | 63 | def test_flowservice_create_flow_data_lake_to_data_landing_zone(self): 64 | assert True 65 | 66 | def test_flowservice_create_data_landing_zone_to_datalake(self): 67 | assert True 68 | 69 | def test_flowservice_updateFlow(self): 70 | assert True 71 | 72 | def test_flowservice_get_flow_specs(self): 73 | assert True 74 | 75 | def test_flowservice_get_flow_spec_id_from_names(self): 76 | assert True 77 | 78 | def test_flowservice_get_flow_spec(self): 79 | assert True 80 | 81 | def test_flowservice_get_runs(self): 82 | assert True 83 | 84 | def test_flowservice_create_run(self): 85 | assert True 86 | 87 | def test_flowservice_get_run(self): 88 | assert True 89 | 90 | def test_flowservice_get_source_connections(self): 91 | assert True 92 | 93 | def test_flowservice_get_source_connection(self): 94 | assert True 95 | 96 | 97 | def test_flowsevrice_delete_source_connection(self): 98 | assert True 99 | 100 | def test_flowservice_create_source_connection(self): 101 | assert True 102 | 103 | def test_flowservice_create_source_connection_streaming(self): 104 | assert True 105 | 106 | def test_flowservice_create_source_connectionDataLandingZone(self): 107 | assert True 108 | 109 | def test_flowservice_create_source_connection_datalake(self): 110 | assert True 111 | 112 | def test_flowservice_update_source_connection(self): 113 | assert True 114 | 115 | 116 | def test_flowservice_get_target_connections(self): 117 | assert True 118 | 119 | def test_flowservice_get_target_connection(self): 120 | assert True 121 | 122 | def test_flowservice_delete_target_connection(self): 123 | assert True 124 | 125 | def test_flowservice_create_target_connection(self): 126 | assert True 127 | 128 | def test_flowservice_create_target_connection_data_landin_zone(self): 129 | assert True 130 | 131 | def test_flowservice_create_target_connection_datalake(self): 132 | assert True 133 | 134 | def test_flowservice_update_target_connection(self): 135 | assert True 136 | 137 | def test_flowservice_update_policy(self): 138 | assert True 139 | 140 | def test_flowservice_get_landing_zone_container(self): 141 | assert True 142 | 143 | def test_flowservice_get_landing_zone_credential(self): 144 | assert True 145 | 146 | def test_flowservice_explore_landing_zone(self): 147 | assert True 148 | 149 | def test_flowservice_get_landing_zone_content(self): 150 | assert True 151 | -------------------------------------------------------------------------------- /tests/schema_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Adobe. All rights reserved. 2 | # This file is licensed to you under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. You may obtain a copy 4 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | # Unless required by applicable law or agreed to in writing, software distributed under 7 | # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 8 | # OF ANY KIND, either express or implied. See the License for the specific language 9 | # governing permissions and limitations under the License. 10 | 11 | from aepp.schema import Schema 12 | import unittest 13 | from unittest.mock import patch, MagicMock 14 | 15 | 16 | class SchemaTest(unittest.TestCase): 17 | 18 | @patch("aepp.connector.AdobeRequest") 19 | def test_schema_get_resource(self, mock_connector): 20 | instance_conn = mock_connector.return_value 21 | instance_conn.getData.return_value = "foo" 22 | schema_obj = Schema() 23 | result = schema_obj.getResource(MagicMock(), MagicMock(), MagicMock(), MagicMock()) 24 | assert(result is not None) 25 | instance_conn.getData.assert_called_once() 26 | 27 | @patch("aepp.connector.AdobeRequest") 28 | def test_schema_update_sandbox(self, mock_connector): 29 | schema_obj = Schema() 30 | test_sandbox = "prod" 31 | schema_obj.updateSandbox(test_sandbox) 32 | assert(schema_obj.sandbox == test_sandbox) 33 | 34 | @patch("aepp.connector.AdobeRequest") 35 | def test_schema_get_stats(self, mock_connector): 36 | instance_conn = mock_connector.return_value 37 | instance_conn.getData.return_value = MagicMock() 38 | schema_obj = Schema() 39 | stats_result = schema_obj.getStats() 40 | assert (stats_result is not None) 41 | assert(stats_result == instance_conn.getData.return_value) 42 | instance_conn.getData.assert_called_once() 43 | 44 | @patch("aepp.connector.AdobeRequest") 45 | def test_schema_get_tenant_id(self, mock_connector): 46 | instance_conn = mock_connector.return_value 47 | instance_conn.getStats.return_value = MagicMock() 48 | schema_obj = Schema() 49 | tenant_id_result = schema_obj.getTenantId() 50 | assert (tenant_id_result is not None) 51 | instance_conn.getData.assert_called_once() 52 | 53 | @patch("aepp.connector.AdobeRequest") 54 | def test_schema_get_behavior(self, mock_connector): 55 | instance_conn = mock_connector.return_value 56 | instance_conn.getData.return_value = {"results":[1234,5678]} 57 | schema_obj = Schema() 58 | stats_result = schema_obj.getBehaviors() 59 | assert (stats_result is not None) 60 | assert (stats_result == instance_conn.getData.return_value.get("results")) 61 | instance_conn.getData.assert_called_once() --------------------------------------------------------------------------------