├── .cfnnag_global_suppress_list ├── .dockerignore ├── .eslintrc.json ├── .gitallowed ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── feature_request.md │ └── general-issues.md ├── PULL_REQUEST_TEMPLATE.md └── dependabot.yml ├── .gitignore ├── .gitlab-ci.yml ├── .gitlab ├── ci-templates │ ├── build.yml │ ├── cfn-lint.yml │ ├── cfn-nag.yml │ ├── data-pipeline-spark-etl.yml │ ├── docs.yml │ ├── etl-plugin-samples.yml │ ├── git-secrets-scan.yml │ ├── license-check.yml │ ├── mutations-check.yml │ ├── postbuild-scan.yml │ ├── prlint.yml │ ├── semgrep.yml │ └── viperlight.yml ├── issue_templates │ ├── bugs.md │ └── features.md └── merge_request_templates │ └── Default.md ├── .mergify.yml ├── .npmignore ├── .npmrc ├── .projen ├── deps.json ├── files.json └── tasks.json ├── .projenrc.js ├── .semgrepignore ├── .viperlightignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE.txt ├── README.md ├── SECURITY.md ├── buildspec.yml ├── cdk.json ├── codescan-prebuild-custom.sh ├── deployment ├── build-open-source-dist.sh ├── build-s3-dist-1.sh ├── build-s3-dist.sh ├── cdk-solution-helper │ ├── README.md │ ├── index.js │ ├── package-lock.json │ └── package.json ├── exclude-open-source-list.txt ├── post-build-1 │ └── index.js ├── run-unit-tests.sh ├── solution_config ├── test-build-dist-1.sh ├── test-build-dist.sh └── test │ └── index.js ├── docs ├── .gitignore ├── en │ ├── analytics │ │ ├── analyzes │ │ │ └── index.md │ │ ├── dashboard │ │ │ ├── acquisition.md │ │ │ ├── custom-analysis.md │ │ │ ├── details.md │ │ │ ├── device.md │ │ │ ├── engagement.md │ │ │ ├── index.md │ │ │ └── retention.md │ │ ├── data-mgmt │ │ │ ├── index.md │ │ │ ├── metadata-mgmt.md │ │ │ └── traffic-source.md │ │ ├── explore │ │ │ ├── attribution.md │ │ │ ├── event.md │ │ │ ├── funnel.md │ │ │ ├── index.md │ │ │ ├── path.md │ │ │ └── retention.md │ │ └── index.md │ ├── architecture-details.md │ ├── architecture.md │ ├── aws-services.md │ ├── deployment │ │ ├── common-parameters.md │ │ ├── domain-parameters.md │ │ ├── index.md │ │ ├── oidc-parameters.md │ │ ├── tls-note.md │ │ ├── with-cognito.md │ │ ├── with-oidc.md │ │ └── within-vpc.md │ ├── faq.md │ ├── getting-started │ │ ├── 1.create-project.md │ │ ├── 2.config-pipeline.md │ │ ├── 3.integrate-sdk.md │ │ ├── 4.view-dashboard.md │ │ └── index.md │ ├── identity-mgmt.md │ ├── images │ │ ├── OIDC │ │ │ ├── adfs-9.png │ │ │ ├── authing-authorization-configuration.png │ │ │ ├── authing-endpoint-info.png │ │ │ ├── cognito-hostUI-new.jpeg │ │ │ ├── cognito-new-console-clientID.png │ │ │ ├── cognito-new-console-userpoolID.png │ │ │ ├── keycloak-OIDC-config.png │ │ │ └── keycloak-example-realm.png │ │ ├── analytics │ │ │ ├── dashboard │ │ │ │ ├── acquisition.png │ │ │ │ ├── activity.png │ │ │ │ ├── details.png │ │ │ │ ├── device.png │ │ │ │ ├── engagement.png │ │ │ │ ├── path.png │ │ │ │ └── retention.png │ │ │ └── explore │ │ │ │ ├── access-studio-en.png │ │ │ │ ├── access-studio.png │ │ │ │ ├── acquisition-dashboard.png │ │ │ │ ├── activity-dashboard.png │ │ │ │ ├── analyze-add-dataset.png │ │ │ │ ├── analyze-adjust-pivot-en.png │ │ │ │ ├── analyze-adjust-pivot.png │ │ │ │ ├── analyze-calculated-field-en.png │ │ │ │ ├── analyze-calculated-field.png │ │ │ │ ├── analyze-new-dataset.png │ │ │ │ ├── analyzes-en.png │ │ │ │ ├── analyzes.png │ │ │ │ ├── attribution-en.png │ │ │ │ ├── crash-dashboard.png │ │ │ │ ├── custom-dashboard-en.png │ │ │ │ ├── custom-dashboard.png │ │ │ │ ├── dashboard-en.png │ │ │ │ ├── dashboard.png │ │ │ │ ├── engagement-dashboard.png │ │ │ │ ├── explore-en.png │ │ │ │ ├── explore-event-en.png │ │ │ │ ├── explore-event.png │ │ │ │ ├── explore-funnel-en.png │ │ │ │ ├── explore-funnel.png │ │ │ │ ├── explore-path-en.png │ │ │ │ ├── explore-path.png │ │ │ │ ├── explore-retention-en.png │ │ │ │ ├── explore-retention.png │ │ │ │ ├── explore.png │ │ │ │ ├── metadata-en.png │ │ │ │ └── metadata.png │ │ ├── architecture │ │ │ ├── 01-architecture-end-to-end.png │ │ │ ├── 02-architecture-ingestion-module.png │ │ │ ├── 03-architecture-data-processing.png │ │ │ ├── 04-architecture-data-modeling-redshift.png │ │ │ ├── 05-architecture-data-modeling-scan-metadata.png │ │ │ ├── 06-architecture-data-modeling-athena.png │ │ │ ├── 07-architecture-reporting.png │ │ │ └── 08-analytics-studio.png │ │ ├── favicon.ico │ │ ├── get-started │ │ │ ├── explore-dashboard-zh.png │ │ │ ├── explore-dashboard.png │ │ │ └── qs-dashboard.png │ │ ├── launch-stack.webp │ │ ├── pipe-mgmt │ │ │ ├── concepts-v11.png │ │ │ ├── concepts.png │ │ │ └── table_relationship.png │ │ ├── sdk-manual │ │ │ ├── android_raw_folder.png │ │ │ ├── replace-in-file.png │ │ │ ├── swift_add_amplify_config_json_file.png │ │ │ ├── swift_add_package.png │ │ │ └── swift_add_package_url.png │ │ └── solution-overview.webp │ ├── index.html │ ├── notices.md │ ├── pipeline-mgmt │ │ ├── app-registration.md │ │ ├── basic.md │ │ ├── data-modeling │ │ │ └── configure-data-modeling.md │ │ ├── data-processing │ │ │ ├── configure-execution-para.md │ │ │ ├── configure-plugin.md │ │ │ ├── data-schema.md │ │ │ └── index.md │ │ ├── index.md │ │ ├── ingestion │ │ │ ├── configure-ingestion-endpoint.md │ │ │ ├── create-data-sink-w-kafka.md │ │ │ ├── create-data-sink-w-kinesis.md │ │ │ ├── create-data-sink-w-s3.md │ │ │ └── index.md │ │ ├── pipe-mgmt.md │ │ ├── quicksight │ │ │ └── configure-quicksight.md │ │ └── vpc-prerequisites.md │ ├── plan-deployment │ │ ├── cost.md │ │ ├── regions.md │ │ └── security.md │ ├── resources │ │ ├── aws-china-prerequisites.md │ │ └── upload-ssl-certificate.md │ ├── revisions.md │ ├── sdk-manual │ │ ├── android.md │ │ ├── flutter.md │ │ ├── http-api.md │ │ ├── index.md │ │ ├── migrate-from-third-party-sdks.md │ │ ├── react-native.md │ │ ├── swift.md │ │ ├── user-identifier.md │ │ ├── web.md │ │ └── wechat.md │ ├── solution-overview │ │ ├── features-and-benefits.md │ │ ├── index.md │ │ └── use-cases.md │ ├── source.md │ ├── stylesheets │ │ └── extra.css │ ├── troubleshooting.md │ ├── uninstall.md │ ├── upgrade.md │ └── well-architected-pillars.md ├── images │ ├── OIDC │ │ ├── adfs-9.png │ │ ├── authing-authorization-configuration.png │ │ ├── authing-endpoint-info.png │ │ ├── cognito-hostUI-new.jpeg │ │ ├── cognito-new-console-clientID.png │ │ ├── cognito-new-console-userpoolID.png │ │ ├── keycloak-OIDC-config.png │ │ └── keycloak-example-realm.png │ ├── analytics │ │ ├── dashboard │ │ │ ├── acquisition.png │ │ │ ├── activity.png │ │ │ ├── details.png │ │ │ ├── device.png │ │ │ ├── engagement.png │ │ │ ├── path.png │ │ │ └── retention.png │ │ └── explore │ │ │ ├── access-studio-en.png │ │ │ ├── access-studio.png │ │ │ ├── acquisition-dashboard.png │ │ │ ├── activity-dashboard.png │ │ │ ├── analyze-add-dataset.png │ │ │ ├── analyze-adjust-pivot-en.png │ │ │ ├── analyze-adjust-pivot.png │ │ │ ├── analyze-calculated-field-en.png │ │ │ ├── analyze-calculated-field.png │ │ │ ├── analyze-new-dataset.png │ │ │ ├── analyzes-en.png │ │ │ ├── analyzes.png │ │ │ ├── attribution-en.png │ │ │ ├── crash-dashboard.png │ │ │ ├── custom-dashboard-en.png │ │ │ ├── custom-dashboard.png │ │ │ ├── dashboard-en.png │ │ │ ├── dashboard.png │ │ │ ├── engagement-dashboard.png │ │ │ ├── explore-en.png │ │ │ ├── explore-event-en.png │ │ │ ├── explore-event.png │ │ │ ├── explore-funnel-en.png │ │ │ ├── explore-funnel.png │ │ │ ├── explore-path-en.png │ │ │ ├── explore-path.png │ │ │ ├── explore-retention-en.png │ │ │ ├── explore-retention.png │ │ │ ├── explore.png │ │ │ ├── metadata-en.png │ │ │ └── metadata.png │ ├── architecture │ │ ├── 01-architecture-end-to-end.png │ │ ├── 02-architecture-ingestion-module.png │ │ ├── 03-architecture-data-processing.png │ │ ├── 04-architecture-data-modeling-redshift.png │ │ ├── 05-architecture-data-modeling-scan-metadata.png │ │ ├── 06-architecture-data-modeling-athena.png │ │ ├── 07-architecture-reporting.png │ │ └── 08-analytics-studio.png │ ├── favicon.ico │ ├── get-started │ │ ├── explore-dashboard-zh.png │ │ ├── explore-dashboard.png │ │ └── qs-dashboard.png │ ├── launch-stack.webp │ ├── pipe-mgmt │ │ ├── concepts-v11.png │ │ ├── concepts.png │ │ └── table_relationship.png │ ├── sdk-manual │ │ ├── android_raw_folder.png │ │ ├── replace-in-file.png │ │ ├── swift_add_amplify_config_json_file.png │ │ ├── swift_add_package.png │ │ └── swift_add_package_url.png │ └── solution-overview.webp ├── mkdocs.base.yml ├── mkdocs.en.yml ├── mkdocs.zh.yml ├── overrides │ └── partials │ │ └── integrations │ │ └── analytics │ │ └── custom.html ├── stylesheets │ └── extra.css └── zh │ ├── analytics │ ├── analyzes │ │ └── index.md │ ├── dashboard │ │ ├── acquisition.md │ │ ├── custom-analysis.md │ │ ├── details.md │ │ ├── device.md │ │ ├── engagement.md │ │ ├── index.md │ │ └── retention.md │ ├── data-mgmt │ │ ├── index.md │ │ ├── metadata-mgmt.md │ │ └── traffic-source.md │ ├── explore │ │ ├── attribution.md │ │ ├── event.md │ │ ├── funnel.md │ │ ├── index.md │ │ ├── path.md │ │ └── retention.md │ └── index.md │ ├── architecture-details.md │ ├── architecture.md │ ├── aws-services.md │ ├── deployment │ ├── common-parameters.md │ ├── domain-parameters.md │ ├── index.md │ ├── oidc-parameters.md │ ├── tls-note.md │ ├── with-cognito.md │ ├── with-oidc.md │ └── within-vpc.md │ ├── faq.md │ ├── getting-started │ ├── 1.create-project.md │ ├── 2.config-pipeline.md │ ├── 3.integrate-sdk.md │ ├── 4.view-dashboard.md │ └── index.md │ ├── identity-mgmt.md │ ├── images │ ├── OIDC │ │ ├── adfs-9.png │ │ ├── authing-authorization-configuration.png │ │ ├── authing-endpoint-info.png │ │ ├── cognito-hostUI-new.jpeg │ │ ├── cognito-new-console-clientID.png │ │ ├── cognito-new-console-userpoolID.png │ │ ├── keycloak-OIDC-config.png │ │ └── keycloak-example-realm.png │ ├── analytics │ │ ├── dashboard │ │ │ ├── acquisition.png │ │ │ ├── activity.png │ │ │ ├── details.png │ │ │ ├── device.png │ │ │ ├── engagement.png │ │ │ ├── path.png │ │ │ └── retention.png │ │ └── explore │ │ │ ├── access-studio-en.png │ │ │ ├── access-studio.png │ │ │ ├── acquisition-dashboard.png │ │ │ ├── activity-dashboard.png │ │ │ ├── analyze-add-dataset.png │ │ │ ├── analyze-adjust-pivot-en.png │ │ │ ├── analyze-adjust-pivot.png │ │ │ ├── analyze-calculated-field-en.png │ │ │ ├── analyze-calculated-field.png │ │ │ ├── analyze-new-dataset.png │ │ │ ├── analyzes-en.png │ │ │ ├── analyzes.png │ │ │ ├── attribution-en.png │ │ │ ├── crash-dashboard.png │ │ │ ├── custom-dashboard-en.png │ │ │ ├── custom-dashboard.png │ │ │ ├── dashboard-en.png │ │ │ ├── dashboard.png │ │ │ ├── engagement-dashboard.png │ │ │ ├── explore-en.png │ │ │ ├── explore-event-en.png │ │ │ ├── explore-event.png │ │ │ ├── explore-funnel-en.png │ │ │ ├── explore-funnel.png │ │ │ ├── explore-path-en.png │ │ │ ├── explore-path.png │ │ │ ├── explore-retention-en.png │ │ │ ├── explore-retention.png │ │ │ ├── explore.png │ │ │ ├── metadata-en.png │ │ │ └── metadata.png │ ├── architecture │ │ ├── 01-architecture-end-to-end.png │ │ ├── 02-architecture-ingestion-module.png │ │ ├── 03-architecture-data-processing.png │ │ ├── 04-architecture-data-modeling-redshift.png │ │ ├── 05-architecture-data-modeling-scan-metadata.png │ │ ├── 06-architecture-data-modeling-athena.png │ │ ├── 07-architecture-reporting.png │ │ └── 08-analytics-studio.png │ ├── favicon.ico │ ├── get-started │ │ ├── explore-dashboard-zh.png │ │ ├── explore-dashboard.png │ │ └── qs-dashboard.png │ ├── launch-stack.webp │ ├── pipe-mgmt │ │ ├── concepts-v11.png │ │ ├── concepts.png │ │ └── table_relationship.png │ ├── sdk-manual │ │ ├── android_raw_folder.png │ │ ├── replace-in-file.png │ │ ├── swift_add_amplify_config_json_file.png │ │ ├── swift_add_package.png │ │ └── swift_add_package_url.png │ └── solution-overview.webp │ ├── index.html │ ├── notices.md │ ├── pipeline-mgmt │ ├── app-registration.md │ ├── basic.md │ ├── data-modeling │ │ └── configure-data-modeling.md │ ├── data-processing │ │ ├── configure-execution-para.md │ │ ├── configure-plugin.md │ │ ├── data-schema.md │ │ └── index.md │ ├── index.md │ ├── ingestion │ │ ├── configure-ingestion-endpoint.md │ │ ├── create-data-sink-w-kafka.md │ │ ├── create-data-sink-w-kinesis.md │ │ ├── create-data-sink-w-s3.md │ │ └── index.md │ ├── pipe-mgmt.md │ ├── quicksight │ │ └── configure-quicksight.md │ └── vpc-prerequisites.md │ ├── plan-deployment │ ├── cost.md │ ├── regions.md │ └── security.md │ ├── resources │ ├── aws-china-prerequisites.md │ └── upload-ssl-certificate.md │ ├── revisions.md │ ├── sdk-manual │ ├── android.md │ ├── flutter.md │ ├── http-api.md │ ├── index.md │ ├── react-native.md │ ├── swift.md │ ├── user-identifier.md │ ├── web.md │ └── wechat.md │ ├── solution-overview │ ├── features-and-benefits.md │ ├── index.md │ └── use-cases.md │ ├── source.md │ ├── stylesheets │ └── extra.css │ ├── troubleshooting.md │ ├── uninstall.md │ ├── upgrade.md │ └── well-architected-pillars.md ├── e2e-deploy.sh ├── examples ├── README.md ├── custom-plugins │ ├── .gitignore │ ├── build.sh │ ├── custom-enrich │ │ ├── .dockerignore │ │ ├── README.md │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── gradle │ │ │ ├── config │ │ │ │ ├── checkstyle │ │ │ │ │ └── sun_checks.xml │ │ │ │ └── scripts │ │ │ │ │ ├── coverage.gradle │ │ │ │ │ ├── idea.gradle │ │ │ │ │ └── style.gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── lombok.config │ │ ├── settings.gradle │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── example │ │ │ │ └── clickstream │ │ │ │ └── CustomUriEnrich.java │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── example │ │ │ │ └── clickstream │ │ │ │ ├── BaseSparkTest.java │ │ │ │ └── CustomUriEnrichTest.java │ │ │ └── resources │ │ │ └── test.snappy.parquet │ └── custom-sdk-transformer │ │ ├── .dockerignore │ │ ├── README.md │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── gradle │ │ ├── config │ │ │ ├── checkstyle │ │ │ │ └── sun_checks.xml │ │ │ └── scripts │ │ │ │ ├── coverage.gradle │ │ │ │ ├── idea.gradle │ │ │ │ └── style.gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── lombok.config │ │ ├── settings.gradle │ │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── example │ │ │ │ └── clickstream │ │ │ │ └── transformer │ │ │ │ ├── v1 │ │ │ │ ├── Cleaner.java │ │ │ │ ├── Constants.java │ │ │ │ ├── ETLMetric.java │ │ │ │ ├── EventParamsConverter.java │ │ │ │ ├── KvConverter.java │ │ │ │ ├── MyTransformer.java │ │ │ │ └── UserPropertiesConverter.java │ │ │ │ └── v2 │ │ │ │ ├── ClickStreamDataTypes.java │ │ │ │ ├── MyTransformerV2.java │ │ │ │ ├── RowResult.java │ │ │ │ ├── TransformerUdf.java │ │ │ │ └── model │ │ │ │ ├── AttributeTypeValue.java │ │ │ │ ├── Event.java │ │ │ │ ├── EventParameter.java │ │ │ │ ├── Item.java │ │ │ │ └── User.java │ │ └── resources │ │ │ └── schema.json │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── clickstream │ │ │ └── transformer │ │ │ ├── BaseSparkTest.java │ │ │ ├── v1 │ │ │ └── MyTransformerTest.java │ │ │ └── v2 │ │ │ ├── MyTransformerV2Test.java │ │ │ └── TransformerUdfTest.java │ │ └── resources │ │ ├── data_with_item.json │ │ └── original_data.json └── standalone-data-generator │ ├── .gitignore │ ├── README.md │ ├── application │ ├── AppProvider.py │ ├── notepad │ │ ├── NotepadApp.py │ │ ├── NotepadEventType.py │ │ └── NotepadScreen.py │ └── shopping │ │ ├── EventSample.py │ │ ├── ExceptionEvent.py │ │ ├── Products.py │ │ ├── ScreenEvent.py │ │ ├── ShoppingApp.py │ │ ├── ShoppingEventType.py │ │ └── ShoppingScreen.py │ ├── configure.py │ ├── create_event.py │ ├── create_event_real_time.py │ ├── enums.py │ ├── model │ ├── App.py │ ├── AppInfo.py │ ├── User.py │ └── device │ │ ├── MobileDevice.py │ │ └── WebDevice.py │ ├── send_event.py │ ├── send_event_real_time.py │ ├── util │ └── util.py │ └── weighted │ └── weighted.py ├── frontend ├── .eslintignore ├── .eslintrc.json ├── .gitattributes ├── .gitignore ├── .npmignore ├── .npmrc ├── .prettierignore ├── .prettierrc.json ├── .projen │ ├── deps.json │ ├── files.json │ └── tasks.json ├── LICENSE ├── README.md ├── config │ ├── BrowserLogger.js │ ├── env.js │ ├── getHttpsConfig.js │ ├── jest │ │ ├── babelTransform.js │ │ ├── cssTransform.js │ │ └── fileTransform.js │ ├── modules.js │ ├── paths.js │ ├── webpack.config.js │ ├── webpack │ │ └── persistentCache │ │ │ └── createEnvironmentHash.js │ └── webpackDevServer.config.js ├── esbuild.ts ├── package.json ├── public │ ├── apple-touch-icon.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── favicon.ico │ ├── index.html │ ├── locales │ │ ├── en-US │ │ │ ├── analytics.json │ │ │ ├── application.json │ │ │ ├── common.json │ │ │ ├── help.json │ │ │ ├── home.json │ │ │ ├── pipeline.json │ │ │ ├── plugin.json │ │ │ ├── project.json │ │ │ └── user.json │ │ └── zh-CN │ │ │ ├── analytics.json │ │ │ ├── application.json │ │ │ ├── common.json │ │ │ ├── help.json │ │ │ ├── home.json │ │ │ ├── pipeline.json │ │ │ ├── plugin.json │ │ │ ├── project.json │ │ │ └── user.json │ └── robots.txt ├── scripts │ ├── build.js │ ├── start.js │ └── test.js ├── src │ ├── App.tsx │ ├── AppRouter.tsx │ ├── AppSignInPage.tsx │ ├── apis │ │ ├── analytics.ts │ │ ├── application.ts │ │ ├── pipeline.ts │ │ ├── plugin.ts │ │ ├── project.ts │ │ ├── resource.ts │ │ ├── system.ts │ │ ├── traffic.ts │ │ └── user.ts │ ├── assets │ │ └── images │ │ │ ├── android.webp │ │ │ ├── howitworks.webp │ │ │ ├── ios.webp │ │ │ └── pipelineArch.webp │ ├── components │ │ ├── common │ │ │ ├── CopyCode.tsx │ │ │ ├── CopyIcon.tsx │ │ │ ├── Divider.tsx │ │ │ ├── ErrorText.tsx │ │ │ ├── ExtendIcon.tsx │ │ │ ├── ExternalLinkGroup.tsx │ │ │ ├── InfoLink.tsx │ │ │ ├── Loading.tsx │ │ │ ├── RoleRoute.tsx │ │ │ ├── alert │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── separated-list │ │ │ │ ├── index.tsx │ │ │ │ └── styles.module.scss │ │ │ └── title │ │ │ │ ├── InfoTitle.tsx │ │ │ │ └── SectionTitle.tsx │ │ ├── eventselect │ │ │ ├── AnalyticsType.ts │ │ │ ├── ConditionItem.tsx │ │ │ ├── DropDownContainer.tsx │ │ │ ├── EventItem.tsx │ │ │ ├── GroupSelectContainer.tsx │ │ │ ├── RetentionItem.tsx │ │ │ ├── RetentionSelect.tsx │ │ │ ├── comps │ │ │ │ ├── AttributePreview.tsx │ │ │ │ ├── CategoryList.tsx │ │ │ │ ├── EventPreview.tsx │ │ │ │ ├── ItemsList.tsx │ │ │ │ ├── Radius.tsx │ │ │ │ ├── RelationAnd.tsx │ │ │ │ └── RelationOr.tsx │ │ │ ├── reducer │ │ │ │ ├── AnalyticsEventSelect.tsx │ │ │ │ ├── AnalyticsSegmentFilter.tsx │ │ │ │ ├── analyticsEventSelectReducer.ts │ │ │ │ └── analyticsSegmentFilterReducer.ts │ │ │ └── svg │ │ │ │ ├── and.svg │ │ │ │ └── or.svg │ │ ├── layouts │ │ │ ├── AnalyticsHeader.tsx │ │ │ ├── AnalyticsLayout.tsx │ │ │ ├── AnalyticsNavigation.tsx │ │ │ ├── CommonLayout.tsx │ │ │ ├── CustomBreadCrumb.tsx │ │ │ ├── Footer.tsx │ │ │ ├── Header.tsx │ │ │ ├── HelpInfo.tsx │ │ │ └── Navigation.tsx │ │ └── pipeline │ │ │ └── PipelineStatus.tsx │ ├── context │ │ ├── AnalyticsEventsContext.tsx │ │ ├── AppContext.ts │ │ ├── StateContext.tsx │ │ ├── SystemInfoContext.ts │ │ ├── UserContext.ts │ │ └── reducer.ts │ ├── hooks │ │ └── useFetchEvents.ts │ ├── i18n.ts │ ├── index.scss │ ├── index.tsx │ ├── pages │ │ ├── alarms │ │ │ ├── AlarmList.tsx │ │ │ └── comps │ │ │ │ ├── AlarmsHeader.tsx │ │ │ │ ├── AlarmsTable.tsx │ │ │ │ └── ProjectWithAlarm.tsx │ │ ├── analytics │ │ │ ├── AnalyticsHome.tsx │ │ │ ├── analytics-utils.ts │ │ │ ├── analyzes │ │ │ │ ├── AnalyticsAnalyzes.tsx │ │ │ │ └── full │ │ │ │ │ └── AnalyticsAnalyzesFullWindow.tsx │ │ │ ├── attribution │ │ │ │ └── AnalyticsAttribution.tsx │ │ │ ├── comps │ │ │ │ ├── AnalyticsCustomHeader.tsx │ │ │ │ ├── AnalyticsCustomHeaderBg.tsx │ │ │ │ ├── AttributeGroup.tsx │ │ │ │ ├── DashboardHeader.tsx │ │ │ │ ├── ExploreDatePicker.tsx │ │ │ │ ├── ExploreDateRangePicker.tsx │ │ │ │ ├── ExploreEmbedFrame.tsx │ │ │ │ ├── SelectDashboardModal.tsx │ │ │ │ └── StartNodeSelect.tsx │ │ │ ├── dashboard │ │ │ │ ├── AnalyticsDashboard.tsx │ │ │ │ ├── create │ │ │ │ │ └── CreateDashboard.tsx │ │ │ │ ├── detail │ │ │ │ │ └── AnalyticsDashboardDetail.tsx │ │ │ │ └── full │ │ │ │ │ └── AnalyticsDashboardFullWindow.tsx │ │ │ ├── data-management │ │ │ │ ├── AnalyticsDataManagement.tsx │ │ │ │ └── MetadataDetails.tsx │ │ │ ├── event │ │ │ │ └── AnalyticsEvent.tsx │ │ │ ├── explore │ │ │ │ └── AnalyticsExplore.tsx │ │ │ ├── funnel │ │ │ │ └── AnalyticsFunnel.tsx │ │ │ ├── metadata │ │ │ │ ├── comps │ │ │ │ │ ├── MetadataDataVolume.tsx │ │ │ │ │ ├── MetadataPlatform.tsx │ │ │ │ │ ├── MetadataSDK.tsx │ │ │ │ │ └── MetadataSource.tsx │ │ │ │ ├── event-parameters │ │ │ │ │ ├── MetadataParameterSplitPanel.tsx │ │ │ │ │ └── MetadataParametersTable.tsx │ │ │ │ ├── events │ │ │ │ │ ├── MetadataEventSplitPanel.tsx │ │ │ │ │ └── MetadataEventsTable.tsx │ │ │ │ ├── table │ │ │ │ │ ├── MetadataDetailsTable.tsx │ │ │ │ │ ├── MetadataDictionaryTable.tsx │ │ │ │ │ ├── MetadataTable.tsx │ │ │ │ │ ├── MetadataTableHeader.tsx │ │ │ │ │ └── table-config.ts │ │ │ │ └── user-attributes │ │ │ │ │ ├── MetadataUserAttributeSplitPanel.tsx │ │ │ │ │ └── MetadataUserAttributesTable.tsx │ │ │ ├── path │ │ │ │ └── AnalyticsPath.tsx │ │ │ ├── realtime │ │ │ │ └── AnalyticsRealtime.tsx │ │ │ ├── retention │ │ │ │ └── AnalyticsRetention.tsx │ │ │ ├── styles │ │ │ │ └── table-select.scss │ │ │ └── traffic-source │ │ │ │ ├── ChannelGroup.tsx │ │ │ │ ├── SourceCategory.tsx │ │ │ │ ├── TrafficSourceHome.tsx │ │ │ │ ├── modal │ │ │ │ ├── ChannelGroupModal.tsx │ │ │ │ └── SourceCategoryModal.tsx │ │ │ │ └── reducer │ │ │ │ └── trafficReducer.ts │ │ ├── application │ │ │ ├── create │ │ │ │ ├── CreateApplication.tsx │ │ │ │ └── comp │ │ │ │ │ └── RegisterApp.tsx │ │ │ └── detail │ │ │ │ ├── ApplicationDetail.tsx │ │ │ │ └── comp │ │ │ │ ├── ConfigAndroidSDK.tsx │ │ │ │ ├── ConfigFlutterSDK.tsx │ │ │ │ ├── ConfigIOSSDK.tsx │ │ │ │ ├── ConfigReactNativeSDK.tsx │ │ │ │ └── ConfigWebSDK.tsx │ │ ├── common │ │ │ ├── DomainNameWithStatus.tsx │ │ │ ├── LabelTag.tsx │ │ │ ├── Tags.tsx │ │ │ ├── common-components.tsx │ │ │ ├── use-column-widths.ts │ │ │ └── use-local-storage.ts │ │ ├── error-page │ │ │ ├── AccessDenied.tsx │ │ │ └── ReSignIn.tsx │ │ ├── home │ │ │ ├── Home.tsx │ │ │ └── comps │ │ │ │ ├── BenefitsFeatures.tsx │ │ │ │ ├── GetStarted.tsx │ │ │ │ ├── HomeHeader.tsx │ │ │ │ ├── HowItWorks.tsx │ │ │ │ ├── MoreResource.tsx │ │ │ │ └── UseCases.tsx │ │ ├── pipelines │ │ │ ├── comps │ │ │ │ ├── BasicInfo.tsx │ │ │ │ └── PipelineHeader.tsx │ │ │ ├── create │ │ │ │ ├── CreatePipeline.tsx │ │ │ │ ├── review │ │ │ │ │ └── BasicInfo.tsx │ │ │ │ └── steps │ │ │ │ │ ├── BasicInformation.tsx │ │ │ │ │ ├── ConfigIngestion.tsx │ │ │ │ │ ├── DataProcessing.tsx │ │ │ │ │ ├── Reporting.tsx │ │ │ │ │ ├── ReviewAndLaunch.tsx │ │ │ │ │ └── buffer │ │ │ │ │ ├── BufferKDS.tsx │ │ │ │ │ ├── BufferMSK.tsx │ │ │ │ │ ├── BufferS3.tsx │ │ │ │ │ └── MSKRequirements.tsx │ │ │ └── detail │ │ │ │ ├── PipelineDetail.tsx │ │ │ │ └── comps │ │ │ │ ├── AlarmTable.tsx │ │ │ │ ├── Alarms.tsx │ │ │ │ ├── Ingestion.tsx │ │ │ │ ├── Monitoring.tsx │ │ │ │ ├── Processing.tsx │ │ │ │ ├── Reporting.tsx │ │ │ │ └── Tags.tsx │ │ ├── plugins │ │ │ ├── PluginList.tsx │ │ │ ├── comps │ │ │ │ ├── PluginHeader.tsx │ │ │ │ └── PluginTable.tsx │ │ │ └── create │ │ │ │ └── CreatePlugin.tsx │ │ ├── projects │ │ │ ├── Projects.tsx │ │ │ ├── comps │ │ │ │ ├── Environment.tsx │ │ │ │ ├── ProjectsHeader.tsx │ │ │ │ └── SplitPanel.tsx │ │ │ ├── create │ │ │ │ └── CreateProject.tsx │ │ │ └── detail │ │ │ │ ├── ProjectDetail.tsx │ │ │ │ └── comp │ │ │ │ ├── NonePipeline.tsx │ │ │ │ └── ProjectPipeline.tsx │ │ └── user │ │ │ ├── CreateUser.tsx │ │ │ ├── SettingUser.tsx │ │ │ ├── UserList.tsx │ │ │ ├── UserTable.tsx │ │ │ └── UserTableHeader.tsx │ ├── react-app-env.d.ts │ ├── reportWebVitals.ts │ ├── setupTests.ts │ ├── ts │ │ ├── const.ts │ │ ├── explore-types.ts │ │ ├── guideConst.ts │ │ ├── init.ts │ │ ├── request.ts │ │ ├── test │ │ │ ├── url.test.ts │ │ │ └── utils.test.ts │ │ ├── url.ts │ │ └── utils.ts │ └── types │ │ ├── analytics.d.ts │ │ ├── application.d.ts │ │ ├── config.d.ts │ │ ├── pipeline.d.ts │ │ ├── plugin.d.ts │ │ ├── project.d.ts │ │ ├── resources.d.ts │ │ ├── system.d.ts │ │ └── user.ts ├── test │ ├── app.test.tsx │ ├── msk-redshift.test.tsx │ ├── service-available.test.tsx │ ├── update-pipeline.test.tsx │ └── utils │ │ ├── url.test..ts │ │ └── utils.test.ts ├── tsconfig.dev.json └── tsconfig.json ├── nx.json ├── package.json ├── pnpm-lock.yaml ├── pnpm-workspace.yaml ├── projenrc ├── nx.js └── pnpm.js ├── scripts └── build.ts ├── solution-manifest.yaml ├── sonar-project.properties ├── src ├── alb-control-plane-stack.ts ├── analytics │ ├── analytics-on-redshift.ts │ ├── lambdas │ │ ├── clear-expired-events-workflow │ │ │ ├── check-clear-status.ts │ │ │ └── clear-expired-events.ts │ │ ├── custom-resource │ │ │ ├── create-redshift-namespace.ts │ │ │ ├── create-redshift-user.ts │ │ │ ├── create-schemas.ts │ │ │ ├── get-source-prefix.ts │ │ │ ├── redshift-associate-iam-role.ts │ │ │ └── redshift-serverless.ts │ │ ├── load-data-workflow │ │ │ ├── check-load-status.ts │ │ │ ├── create-load-manifest.ts │ │ │ ├── has-more-job-new.ts │ │ │ ├── load-manifest-to-redshift.ts │ │ │ ├── put-ods-source-to-store.ts │ │ │ ├── refresh-views.ts │ │ │ └── skip-running-workflow.ts │ │ ├── redshift-data.ts │ │ ├── refresh-materialized-views-workflow │ │ │ ├── check-refresh-mv-status.ts │ │ │ ├── check-refresh-sp-status.ts │ │ │ ├── check-refresh-workflow-start.ts │ │ │ ├── check-start-sp-refresh.ts │ │ │ ├── get-refresh-viewlist.ts │ │ │ ├── parse-timezone-with-appId-list.ts │ │ │ ├── refresh-basic-view.ts │ │ │ └── refresh-sp.ts │ │ ├── scan-metadata-workflow │ │ │ ├── check-metadata-workflow-start.ts │ │ │ ├── check-scan-metadata-status.ts │ │ │ ├── scan-metadata.ts │ │ │ ├── store-metadata-into-ddb.ts │ │ │ └── update-workflow-info.ts │ │ └── sql-execution-sfn │ │ │ └── sql-execution-step-fn.ts │ ├── parameter.ts │ └── private │ │ ├── app-schema.ts │ │ ├── athena-saved-queries.ts │ │ ├── athena │ │ └── query.ts │ │ ├── clear-expired-events-workflow.ts │ │ ├── condition.ts │ │ ├── constant.ts │ │ ├── load-ods-data-workflow.ts │ │ ├── metrics-common-workflow.ts │ │ ├── metrics-redshift-cluster.ts │ │ ├── metrics-redshift-serverless.ts │ │ ├── model.ts │ │ ├── redshift-associate-iam-role.ts │ │ ├── redshift-serverless.ts │ │ ├── refresh-materialized-views-workflow.ts │ │ ├── scan-metadata-workflow.ts │ │ ├── sql-def.ts │ │ ├── sql-exectution-stepfuncs.ts │ │ ├── sql-execution-workflow.ts │ │ ├── sqls │ │ ├── athena │ │ │ ├── clickstream-event-query.sql │ │ │ ├── clickstream-event-user-session-query.sql │ │ │ ├── clickstream-item-query.sql │ │ │ ├── clickstream-session-query.sql │ │ │ └── clickstream-user-query.sql │ │ └── redshift │ │ │ ├── clickstream-log-v2.sql │ │ │ ├── dashboard │ │ │ ├── clickstream_acquisition_country_new_user.sql │ │ │ ├── clickstream_acquisition_country_new_user_sp.sql │ │ │ ├── clickstream_acquisition_day_traffic_source_user.sql │ │ │ ├── clickstream_acquisition_day_traffic_source_user_sp.sql │ │ │ ├── clickstream_acquisition_day_user_acquisition.sql │ │ │ ├── clickstream_acquisition_day_user_acquisition_sp.sql │ │ │ ├── clickstream_acquisition_day_user_view_cnt.sql │ │ │ ├── clickstream_acquisition_day_user_view_cnt_sp.sql │ │ │ ├── clickstream_acquisition_intra_day_user_mv.sql │ │ │ ├── clickstream_device_crash_rate.sql │ │ │ ├── clickstream_device_crash_rate_sp.sql │ │ │ ├── clickstream_device_user_device.sql │ │ │ ├── clickstream_device_user_device_sp.sql │ │ │ ├── clickstream_engagement_day_event_view.sql │ │ │ ├── clickstream_engagement_day_event_view_sp.sql │ │ │ ├── clickstream_engagement_entrance.sql │ │ │ ├── clickstream_engagement_entrance_sp.sql │ │ │ ├── clickstream_engagement_event_name.sql │ │ │ ├── clickstream_engagement_event_name_sp.sql │ │ │ ├── clickstream_engagement_exit.sql │ │ │ ├── clickstream_engagement_exit_sp.sql │ │ │ ├── clickstream_engagement_kpi.sql │ │ │ ├── clickstream_engagement_kpi_sp.sql │ │ │ ├── clickstream_engagement_page_screen_view.sql │ │ │ ├── clickstream_engagement_page_screen_view_detail.sql │ │ │ ├── clickstream_engagement_page_screen_view_detail_sp.sql │ │ │ ├── clickstream_engagement_page_screen_view_sp.sql │ │ │ ├── clickstream_event_base_view.sql │ │ │ ├── clickstream_event_base_view_sp.sql │ │ │ ├── clickstream_event_view_v3.sql │ │ │ ├── clickstream_lifecycle_view_v2.sql │ │ │ ├── clickstream_lifecycle_weekly_view_v3.sql │ │ │ ├── clickstream_retention_base_view.sql │ │ │ ├── clickstream_retention_dau_wau.sql │ │ │ ├── clickstream_retention_dau_wau_sp.sql │ │ │ ├── clickstream_retention_event_overtime.sql │ │ │ ├── clickstream_retention_event_overtime_sp.sql │ │ │ ├── clickstream_retention_user_new_return.sql │ │ │ ├── clickstream_retention_user_new_return_sp.sql │ │ │ └── clickstream_retention_view_v3.sql │ │ │ ├── event-v2.sql │ │ │ ├── grant-permissions-to-bi-user-1.sql │ │ │ ├── grant-permissions-to-bi-user-2.sql │ │ │ ├── item-v2.sql │ │ │ ├── migrate │ │ │ ├── fn-combine-json-list.sql │ │ │ ├── fn-parse-utm-from-url.sql │ │ │ ├── fn-rm-object-props.sql │ │ │ ├── fn-transform-event-custom-props.sql │ │ │ ├── fn-transform-user-custom-props.sql │ │ │ ├── sp-migrate-data-to-v2.sql │ │ │ ├── sp-migrate-event-to-v2.sql │ │ │ ├── sp-migrate-item-to-v2.sql │ │ │ ├── sp-migrate-session-to-v2.sql │ │ │ └── sp-migrate-user-to-v2.sql │ │ │ ├── refresh-mv-sp-status.sql │ │ │ ├── session.sql │ │ │ ├── sp-clear-expired-data.sql │ │ │ ├── sp-clickstream-log-non-atomic.sql │ │ │ ├── sp-clickstream-log.sql │ │ │ ├── sp-merge-event-v2.sql │ │ │ ├── sp-merge-item-v2.sql │ │ │ ├── sp-merge-session.sql │ │ │ ├── sp-merge-user-v2.sql │ │ │ ├── sp-scan-metadata.sql │ │ │ └── user-v2.sql │ │ └── utils.ts ├── base-lib │ ├── .gitattributes │ ├── .gitignore │ ├── .npmignore │ ├── .npmrc │ ├── .projen │ │ ├── deps.json │ │ ├── files.json │ │ └── tasks.json │ ├── LICENSE │ ├── README.md │ ├── package.json │ ├── src │ │ ├── auth │ │ │ ├── authorizer.ts │ │ │ └── index.ts │ │ ├── common │ │ │ ├── fetch.ts │ │ │ ├── index.ts │ │ │ ├── powertools.ts │ │ │ ├── sdk-client-config.ts │ │ │ ├── solution-info.ts │ │ │ └── utils.ts │ │ ├── constant │ │ │ ├── constant.ts │ │ │ ├── dashboard.ts │ │ │ ├── explore-types.ts │ │ │ └── index.ts │ │ ├── index.ts │ │ └── model │ │ │ ├── index.ts │ │ │ └── metadata.ts │ ├── test │ │ ├── solution-info.test.ts │ │ └── utils.test.ts │ ├── tsconfig.dev.json │ └── tsconfig.json ├── cloudfront-control-plane-stack.ts ├── common │ ├── alb.ts │ ├── aspects.ts │ ├── cfn-nag.ts │ ├── custom-resource-exec-in-redshift.ts │ ├── custom-resources.ts │ ├── lambda.ts │ ├── lambda │ │ └── tags.ts │ ├── logs.ts │ ├── metrics.ts │ ├── model.ts │ ├── parameters.ts │ ├── s3-asset.ts │ ├── s3.ts │ ├── sg.ts │ ├── solution-bucket.ts │ ├── solution-vpc.ts │ ├── sqs.ts │ ├── stack.ts │ ├── vpc-utils.ts │ └── workflow.ts ├── control-plane │ ├── alb-lambda-portal.ts │ ├── auth │ │ └── index.ts │ ├── backend │ │ ├── Dockerfile │ │ ├── batch-insert-ddb-custom-resource-construct.ts │ │ ├── click-stream-api.ts │ │ ├── event-bus-construct.ts │ │ ├── insert-admin-user.ts │ │ ├── lambda │ │ │ ├── api │ │ │ │ ├── .dockerignore │ │ │ │ ├── .gitattributes │ │ │ │ ├── .gitignore │ │ │ │ ├── .npmignore │ │ │ │ ├── .npmrc │ │ │ │ ├── .projen │ │ │ │ │ ├── deps.json │ │ │ │ │ ├── files.json │ │ │ │ │ └── tasks.json │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── common │ │ │ │ │ ├── constants.ts │ │ │ │ │ ├── dynamodb-client.ts │ │ │ │ │ ├── explore-types.ts │ │ │ │ │ ├── model-ln.ts │ │ │ │ │ ├── paginator.ts │ │ │ │ │ ├── parameter-reflect.ts │ │ │ │ │ ├── powertools.ts │ │ │ │ │ ├── request-valid.ts │ │ │ │ │ ├── sdk-client.ts │ │ │ │ │ ├── sfn.ts │ │ │ │ │ ├── stack-params-valid.ts │ │ │ │ │ ├── types.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── config │ │ │ │ │ └── dictionary.json │ │ │ │ ├── i18n.ts │ │ │ │ ├── index.ts │ │ │ │ ├── locales │ │ │ │ │ ├── en-US.json │ │ │ │ │ └── zh-CN.json │ │ │ │ ├── middle-ware │ │ │ │ │ ├── access-log.ts │ │ │ │ │ ├── auth-oidc.ts │ │ │ │ │ ├── auth-role.ts │ │ │ │ │ ├── error-handler.ts │ │ │ │ │ ├── inject-context.ts │ │ │ │ │ └── response-time.ts │ │ │ │ ├── model │ │ │ │ │ ├── application.ts │ │ │ │ │ ├── dictionary.ts │ │ │ │ │ ├── metadata.ts │ │ │ │ │ ├── pipeline.ts │ │ │ │ │ ├── plugin.ts │ │ │ │ │ ├── project.ts │ │ │ │ │ ├── stacks.ts │ │ │ │ │ ├── traffic.ts │ │ │ │ │ └── user.ts │ │ │ │ ├── package.json │ │ │ │ ├── router │ │ │ │ │ ├── application.ts │ │ │ │ │ ├── environment.ts │ │ │ │ │ ├── metadata.ts │ │ │ │ │ ├── pipeline.ts │ │ │ │ │ ├── plugin.ts │ │ │ │ │ ├── project.ts │ │ │ │ │ ├── reporting.ts │ │ │ │ │ ├── system.ts │ │ │ │ │ ├── traffic.ts │ │ │ │ │ └── user.ts │ │ │ │ ├── run.sh │ │ │ │ ├── sam-dev-template.yaml │ │ │ │ ├── service │ │ │ │ │ ├── application.ts │ │ │ │ │ ├── attribution.ts │ │ │ │ │ ├── display.ts │ │ │ │ │ ├── environment.ts │ │ │ │ │ ├── metadata.ts │ │ │ │ │ ├── pipeline.ts │ │ │ │ │ ├── plugin.ts │ │ │ │ │ ├── project.ts │ │ │ │ │ ├── quicksight │ │ │ │ │ │ ├── dashboard-ln.ts │ │ │ │ │ │ ├── reporting-check.ts │ │ │ │ │ │ ├── reporting-utils.ts │ │ │ │ │ │ ├── sql-builder-attribution.ts │ │ │ │ │ │ ├── sql-builder.ts │ │ │ │ │ │ └── templates │ │ │ │ │ │ │ ├── attribution-table-chart.json │ │ │ │ │ │ │ ├── dashboard.json │ │ │ │ │ │ │ ├── dataset-config.json │ │ │ │ │ │ │ ├── datetime-parameter.json │ │ │ │ │ │ │ ├── event-bar-chart-multiple.json │ │ │ │ │ │ │ ├── event-bar-chart.json │ │ │ │ │ │ │ ├── event-line-chart-multiple.json │ │ │ │ │ │ │ ├── event-line-chart.json │ │ │ │ │ │ │ ├── event-pivot-table-chart-multiple.json │ │ │ │ │ │ │ ├── event-pivot-table-chart.json │ │ │ │ │ │ │ ├── event-table-chart.json │ │ │ │ │ │ │ ├── filter-control-datetime.json │ │ │ │ │ │ │ ├── filter-control-relative-datetime.json │ │ │ │ │ │ │ ├── filter-group-relative.template │ │ │ │ │ │ │ ├── filter-group.template │ │ │ │ │ │ │ ├── funnel-bar-chart-multiple.json │ │ │ │ │ │ │ ├── funnel-bar-chart.json │ │ │ │ │ │ │ ├── funnel-funnel-chart.json │ │ │ │ │ │ │ ├── funnel-table-chart.json │ │ │ │ │ │ │ ├── layout-control.json │ │ │ │ │ │ │ ├── layout-visual.json │ │ │ │ │ │ │ ├── path-sankey-chart.json │ │ │ │ │ │ │ ├── percentage-column-config.json │ │ │ │ │ │ │ ├── retention-bar-chart-multiple.json │ │ │ │ │ │ │ ├── retention-bar-chart.json │ │ │ │ │ │ │ ├── retention-line-chart-multiple.json │ │ │ │ │ │ │ ├── retention-line-chart.json │ │ │ │ │ │ │ ├── retention-pivot-table-chart-multiple.json │ │ │ │ │ │ │ └── retention-pivot-table-chart.json │ │ │ │ │ ├── reporting.ts │ │ │ │ │ ├── stack.ts │ │ │ │ │ ├── system.ts │ │ │ │ │ ├── traffic.ts │ │ │ │ │ └── user.ts │ │ │ │ ├── store │ │ │ │ │ ├── aws │ │ │ │ │ │ ├── acm.ts │ │ │ │ │ │ ├── cloudformation.ts │ │ │ │ │ │ ├── cloudwatch.ts │ │ │ │ │ │ ├── ec2.ts │ │ │ │ │ │ ├── events.ts │ │ │ │ │ │ ├── iam.ts │ │ │ │ │ │ ├── kafka.ts │ │ │ │ │ │ ├── quicksight.ts │ │ │ │ │ │ ├── redshift.ts │ │ │ │ │ │ ├── route53.ts │ │ │ │ │ │ ├── s3.ts │ │ │ │ │ │ ├── secretsmanager.ts │ │ │ │ │ │ ├── sfn.ts │ │ │ │ │ │ ├── sns.ts │ │ │ │ │ │ └── sts.ts │ │ │ │ │ ├── click-stream-store.ts │ │ │ │ │ ├── dynamodb │ │ │ │ │ │ ├── dynamodb-metadata-store.ts │ │ │ │ │ │ └── dynamodb-store.ts │ │ │ │ │ └── metadata-store.ts │ │ │ │ ├── test │ │ │ │ │ └── api │ │ │ │ │ │ ├── analytics-dashboard.test.ts │ │ │ │ │ │ ├── app.test.ts │ │ │ │ │ │ ├── application.test.ts │ │ │ │ │ │ ├── attribution-sql-builder.test.ts │ │ │ │ │ │ ├── auth.test.ts │ │ │ │ │ │ ├── aws-sdk-mock-util.ts │ │ │ │ │ │ ├── ddb-mock.ts │ │ │ │ │ │ ├── env-bucket-policy.test.ts │ │ │ │ │ │ ├── env.test.ts │ │ │ │ │ │ ├── metadata-mock.ts │ │ │ │ │ │ ├── metadata-v3.test.ts │ │ │ │ │ │ ├── metadata.test.ts │ │ │ │ │ │ ├── pipeline-mock.ts │ │ │ │ │ │ ├── pipeline-status.test.ts │ │ │ │ │ │ ├── pipeline.test.ts │ │ │ │ │ │ ├── plugin.test.ts │ │ │ │ │ │ ├── project.test.ts │ │ │ │ │ │ ├── reporting-utils.test.ts │ │ │ │ │ │ ├── reporting.test.ts │ │ │ │ │ │ ├── request-id.test.ts │ │ │ │ │ │ ├── sql-builder.test.ts │ │ │ │ │ │ ├── store.test.ts │ │ │ │ │ │ ├── system.test.ts │ │ │ │ │ │ ├── traffic.test.ts │ │ │ │ │ │ ├── user.test.ts │ │ │ │ │ │ ├── utils.test.ts │ │ │ │ │ │ ├── workflow-mock.ts │ │ │ │ │ │ ├── workflow-version.test.ts │ │ │ │ │ │ └── workflow.test.ts │ │ │ │ ├── tsconfig.dev.json │ │ │ │ └── tsconfig.json │ │ │ ├── batch-insert-ddb │ │ │ │ └── index.ts │ │ │ ├── listen-stack-status │ │ │ │ ├── index.ts │ │ │ │ └── listen-tools.ts │ │ │ ├── listen-state-status │ │ │ │ └── index.ts │ │ │ ├── sfn-action │ │ │ │ └── index.ts │ │ │ └── sfn-workflow │ │ │ │ └── index.ts │ │ ├── layer │ │ │ └── lambda-web-adapter │ │ │ │ ├── Dockerfile │ │ │ │ ├── bootstrap │ │ │ │ ├── lambda-adapter │ │ │ │ └── layer.ts │ │ ├── stack-action-state-machine-construct.ts │ │ └── stack-workflow-state-machine-construct.ts │ ├── cloudfront-s3-portal.ts │ ├── frontend │ │ ├── Dockerfile │ │ └── config │ │ │ └── nginx.conf │ ├── local │ │ ├── backend.sh │ │ ├── frontend.sh │ │ └── start.sh │ └── private │ │ ├── constant.ts │ │ ├── nag.ts │ │ ├── solution-cognito.ts │ │ └── solution-config.ts ├── data-analytics-redshift-stack.ts ├── data-modeling-athena-stack.ts ├── data-pipeline-stack.ts ├── data-pipeline │ ├── data-pipeline.ts │ ├── etl-common │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── gradle │ │ │ ├── config │ │ │ │ ├── checkstyle │ │ │ │ │ └── sun_checks.xml │ │ │ │ └── scripts │ │ │ │ │ ├── coverage.gradle │ │ │ │ │ ├── idea.gradle │ │ │ │ │ └── style.gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── lombok.config │ │ ├── settings.gradle │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── software │ │ │ │ └── aws │ │ │ │ └── solution │ │ │ │ └── clickstream │ │ │ │ └── common │ │ │ │ ├── BaseEventParser.java │ │ │ │ ├── Cache.java │ │ │ │ ├── ClickstreamEventParser.java │ │ │ │ ├── Constant.java │ │ │ │ ├── EventParser.java │ │ │ │ ├── ExtraParams.java │ │ │ │ ├── ParseDataResult.java │ │ │ │ ├── ParseRowResult.java │ │ │ │ ├── RuleConfig.java │ │ │ │ ├── SafeBooleanDeserializer.java │ │ │ │ ├── SafeDoubleDeserializer.java │ │ │ │ ├── SafeIntegerDeserializer.java │ │ │ │ ├── SafeLongDeserializer.java │ │ │ │ ├── TransformConfig.java │ │ │ │ ├── Util.java │ │ │ │ ├── enrich │ │ │ │ ├── RuleBasedTrafficSourceHelper.java │ │ │ │ ├── UAEnrichHelper.java │ │ │ │ ├── UrlParseResult.java │ │ │ │ └── ts │ │ │ │ │ ├── CategoryTrafficSource.java │ │ │ │ │ ├── SourceMedium.java │ │ │ │ │ ├── TrafficSourceHelper.java │ │ │ │ │ ├── TrafficSourceUtm.java │ │ │ │ │ └── rule │ │ │ │ │ ├── CategoryItem.java │ │ │ │ │ ├── CategoryListEvaluator.java │ │ │ │ │ ├── ChannelListEvaluator.java │ │ │ │ │ ├── ChannelRule.java │ │ │ │ │ ├── ChannelRuleCondition.java │ │ │ │ │ ├── ChannelRuleConditionItem.java │ │ │ │ │ ├── ChannelRuleEvaluator.java │ │ │ │ │ ├── ChannelRuleEvaluatorInput.java │ │ │ │ │ ├── OpEnum.java │ │ │ │ │ └── SourceCategoryAndTerms.java │ │ │ │ ├── exception │ │ │ │ └── ExtractDataException.java │ │ │ │ ├── gtm │ │ │ │ ├── GTMEventParser.java │ │ │ │ └── event │ │ │ │ │ ├── Brand.java │ │ │ │ │ ├── ClientHints.java │ │ │ │ │ ├── EventLocation.java │ │ │ │ │ ├── GTMEvent.java │ │ │ │ │ ├── Item.java │ │ │ │ │ ├── SstSystemProperties.java │ │ │ │ │ ├── SystemProperties.java │ │ │ │ │ ├── UserProperties.java │ │ │ │ │ └── Version.java │ │ │ │ ├── ingest │ │ │ │ ├── ClickstreamIngestRow.java │ │ │ │ ├── Event.java │ │ │ │ ├── EventAttributes.java │ │ │ │ ├── Item.java │ │ │ │ ├── User.java │ │ │ │ ├── UserPropIntegerValue.java │ │ │ │ ├── UserPropLongValue.java │ │ │ │ ├── UserPropObjectValue.java │ │ │ │ └── UserPropStringValue.java │ │ │ │ ├── model │ │ │ │ ├── ClickstreamEvent.java │ │ │ │ ├── ClickstreamEventPropValue.java │ │ │ │ ├── ClickstreamEventPropValueSerializer.java │ │ │ │ ├── ClickstreamItem.java │ │ │ │ ├── ClickstreamUA.java │ │ │ │ ├── ClickstreamUser.java │ │ │ │ ├── ClickstreamUserPropValue.java │ │ │ │ ├── ClickstreamUserPropValueSerializer.java │ │ │ │ ├── TimeShiftInfo.java │ │ │ │ ├── ValueType.java │ │ │ │ └── ValueTypeSerializer.java │ │ │ │ └── sensors │ │ │ │ ├── SensorsEventParser.java │ │ │ │ └── event │ │ │ │ ├── Identities.java │ │ │ │ ├── Item.java │ │ │ │ ├── Lib.java │ │ │ │ ├── Properties.java │ │ │ │ └── SensorsEvent.java │ │ │ └── test │ │ │ ├── java │ │ │ └── software │ │ │ │ └── aws │ │ │ │ └── solution │ │ │ │ └── clickstream │ │ │ │ ├── BaseTest.java │ │ │ │ └── common │ │ │ │ ├── CacheTest.java │ │ │ │ ├── ClickstreamEventParserTest.java │ │ │ │ ├── RuleConfigTest.java │ │ │ │ ├── SafeBooleanDeserializerTest.java │ │ │ │ ├── SafeDoubleDeserializerTest.java │ │ │ │ ├── SafeIntegerDeserializerTest.java │ │ │ │ ├── TransformConfigTest.java │ │ │ │ ├── UAEnrichHelperTest.java │ │ │ │ ├── enrich │ │ │ │ ├── RuleBasedTrafficSourceHelperTest.java │ │ │ │ ├── UAEnrichHelperTest.java │ │ │ │ └── ts │ │ │ │ │ ├── CategoryTrafficSourceTest.java │ │ │ │ │ ├── TrafficSourceUtmTest.java │ │ │ │ │ └── rule │ │ │ │ │ ├── CategoryItemTest.java │ │ │ │ │ ├── CategoryListEvaluatorTest.java │ │ │ │ │ ├── ChannelListEvaluatorTest.java │ │ │ │ │ ├── ChannelRuleConditionItemTest.java │ │ │ │ │ ├── ChannelRuleEvaluatorInputTest.java │ │ │ │ │ ├── ChannelRuleEvaluatorTest.java │ │ │ │ │ ├── ChannelRuleTest.java │ │ │ │ │ └── SourceCategoryAndTermsTest.java │ │ │ │ ├── exception │ │ │ │ └── ExceptionTest.java │ │ │ │ ├── gtm │ │ │ │ └── GTMEventParserTest.java │ │ │ │ ├── ingest │ │ │ │ ├── UserPropIntegerValueTest.java │ │ │ │ ├── UserPropLongValueTest.java │ │ │ │ ├── UserPropObjectValueTest.java │ │ │ │ ├── UserPropStringValueTest.java │ │ │ │ └── UserTest.java │ │ │ │ ├── model │ │ │ │ ├── ClickstreamEventPropValueSerializerTest.java │ │ │ │ ├── ClickstreamEventTest.java │ │ │ │ └── ClickstreamUserPropValueSerializerTest.java │ │ │ │ ├── sensors │ │ │ │ └── SensorsEventParserTest.java │ │ │ │ └── util │ │ │ │ └── UtilTest.java │ │ │ └── resources │ │ │ ├── data_future_event_time.json │ │ │ ├── empty_latest_referrer.json │ │ │ ├── event_deser_input.json │ │ │ ├── expected │ │ │ ├── test_adjustFutureEventTime.json │ │ │ ├── test_parse_data.json │ │ │ ├── test_parse_line.json │ │ │ ├── test_parse_line_to_db_row_disable_time_shift.json │ │ │ ├── test_parse_line_to_db_row_event.json │ │ │ ├── test_parse_line_to_db_row_event_v2.json │ │ │ ├── test_parse_line_to_db_row_event_web.json │ │ │ ├── test_parse_line_to_db_row_item.json │ │ │ ├── test_parse_line_to_db_row_time_shift.json │ │ │ ├── test_parse_line_to_db_row_user.json │ │ │ ├── test_parse_line_to_db_row_zip_event_v2.json │ │ │ └── test_screen_view_engagement_time.json │ │ │ ├── gtm-server │ │ │ ├── expected │ │ │ │ ├── test_gtm_ingestDataToEvent.json │ │ │ │ ├── test_gtm_ingestLineToRow.json │ │ │ │ ├── test_gtm_parseLineToDBRow_event.json │ │ │ │ ├── test_gtm_parseLineToDBRow_item.json │ │ │ │ ├── test_gtm_parseLineToDBRow_user.json │ │ │ │ └── test_gtm_parseLineToDBRow_user_login.json │ │ │ ├── server-items.json │ │ │ ├── server-session-start.json │ │ │ ├── server-single.json │ │ │ └── server-user-props.json │ │ │ ├── one_line.json │ │ │ ├── original_data.json │ │ │ ├── original_data_nozip.json │ │ │ ├── original_data_nozip_client_time.json │ │ │ ├── original_data_nozip_upload_time.json │ │ │ ├── original_data_nozip_uri_upload.json │ │ │ ├── original_data_page_url.json │ │ │ ├── original_data_page_url_web.json │ │ │ ├── original_data_single.json │ │ │ ├── original_data_with_items.json │ │ │ ├── screen_view_engagement_time.json │ │ │ ├── sensors-data │ │ │ ├── expected │ │ │ │ ├── test_sensors_data_ingestDataToEvent.json │ │ │ │ ├── test_sensors_data_parseLineToDBRow_event.json │ │ │ │ ├── test_sensors_data_parseLineToDBRow_web_event.json │ │ │ │ ├── test_sensors_ingestLineToRow.json │ │ │ │ └── test_sensors_parseLineToDBRow_user.json │ │ │ ├── gzip-raw-data.json │ │ │ ├── sensors-web-data-empty.json │ │ │ ├── sensors-web-ua.json │ │ │ ├── track-signup-data.json │ │ │ ├── unzip-data.json │ │ │ ├── unzip-data2.json │ │ │ ├── unzip-item-data.json │ │ │ ├── web-non-gzip-data.json │ │ │ └── web-sdk-data.json │ │ │ └── ts │ │ │ ├── traffic_source_category_rule_test.json │ │ │ ├── traffic_source_category_rule_v0.json │ │ │ ├── traffic_source_channel_rule_test.json │ │ │ └── traffic_source_channel_rule_v0.json │ ├── lambda │ │ ├── copy-assets │ │ │ └── index.ts │ │ ├── emr-job-state-listener │ │ │ └── index.ts │ │ ├── emr-job-submitter │ │ │ ├── emr-client-util.ts │ │ │ ├── index.ts │ │ │ ├── traffic_source_category_rule_v1.ts │ │ │ └── traffic_source_channel_rule_v1.ts │ │ ├── emr-serverless-app │ │ │ └── index.ts │ │ └── partition-syncer │ │ │ ├── glue-client-util.ts │ │ │ └── index.ts │ ├── parameter.ts │ ├── spark-etl │ │ ├── .dockerignore │ │ ├── build.gradle │ │ ├── gradle.properties │ │ ├── gradle │ │ │ ├── config │ │ │ │ ├── checkstyle │ │ │ │ │ └── sun_checks.xml │ │ │ │ └── scripts │ │ │ │ │ ├── coverage.gradle │ │ │ │ │ ├── idea.gradle │ │ │ │ │ └── style.gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── lombok.config │ │ ├── settings.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── software │ │ │ │ │ └── aws │ │ │ │ │ └── solution │ │ │ │ │ └── clickstream │ │ │ │ │ ├── DataProcessor.java │ │ │ │ │ ├── ETLRunner.java │ │ │ │ │ ├── IPEnrichment.java │ │ │ │ │ ├── IPEnrichmentV2.java │ │ │ │ │ ├── Transformer.java │ │ │ │ │ ├── TransformerInterfaceV3.java │ │ │ │ │ ├── TransformerV2.java │ │ │ │ │ ├── TransformerV3.java │ │ │ │ │ ├── UAEnrichment.java │ │ │ │ │ ├── UAEnrichmentV2.java │ │ │ │ │ ├── exception │ │ │ │ │ └── ExecuteTransformerException.java │ │ │ │ │ ├── gtm │ │ │ │ │ ├── GTMServerDataTransformer.java │ │ │ │ │ ├── GTMServerDataTransformerV2.java │ │ │ │ │ ├── ServerDataConverter.java │ │ │ │ │ └── ServerDataConverterV2.java │ │ │ │ │ ├── model │ │ │ │ │ └── ModelV2.java │ │ │ │ │ ├── rowconv │ │ │ │ │ ├── EventGenericRowConverter.java │ │ │ │ │ ├── ItemGenericRowConverter.java │ │ │ │ │ └── UserGenericRowConverter.java │ │ │ │ │ ├── sensors │ │ │ │ │ ├── SensorsDataConverterV2.java │ │ │ │ │ └── SensorsDataTransformerV2.java │ │ │ │ │ ├── transformer │ │ │ │ │ ├── BaseThirdPartyTransformer.java │ │ │ │ │ ├── BaseTransformerV3.java │ │ │ │ │ ├── Cleaner.java │ │ │ │ │ ├── EventParamsConverter.java │ │ │ │ │ ├── KvConverter.java │ │ │ │ │ ├── MaxLengthTransformer.java │ │ │ │ │ ├── MaxLengthTransformerV2.java │ │ │ │ │ ├── TransformConfigurable.java │ │ │ │ │ ├── TransformerNameEnum.java │ │ │ │ │ └── UserPropertiesConverter.java │ │ │ │ │ ├── udfconverter │ │ │ │ │ ├── BaseDataConverter.java │ │ │ │ │ ├── ClickstreamDataConverterV3.java │ │ │ │ │ ├── DatasetConverter.java │ │ │ │ │ ├── EventParserFactory.java │ │ │ │ │ └── UDFHelper.java │ │ │ │ │ └── util │ │ │ │ │ ├── ContextUtil.java │ │ │ │ │ ├── DatasetUtil.java │ │ │ │ │ ├── ETLMetric.java │ │ │ │ │ ├── ETLRunnerConfig.java │ │ │ │ │ ├── PathContent.java │ │ │ │ │ └── TableName.java │ │ │ └── resources │ │ │ │ ├── data_schema.json │ │ │ │ └── data_schema_v2.json │ │ │ └── test │ │ │ ├── java │ │ │ └── software │ │ │ │ └── aws │ │ │ │ └── solution │ │ │ │ └── clickstream │ │ │ │ ├── BaseSparkTest.java │ │ │ │ ├── BaseTest.java │ │ │ │ ├── CleanerTest.java │ │ │ │ ├── ClickstreamDataConverterV3Test.java │ │ │ │ ├── ContextUtilTest.java │ │ │ │ ├── DataProcessorTest.java │ │ │ │ ├── ETLRunnerBaseTest.java │ │ │ │ ├── ETLRunnerForGtmV2Test.java │ │ │ │ ├── ETLRunnerForSensorsDataTest.java │ │ │ │ ├── ETLRunnerForTransformerV3Test.java │ │ │ │ ├── ETLRunnerTest.java │ │ │ │ ├── EventParamsConverterTest.java │ │ │ │ ├── ExceptionTest.java │ │ │ │ ├── IPEnrichmentTest.java │ │ │ │ ├── IPEnrichmentV2Test.java │ │ │ │ ├── MaxLengthTransformerTest.java │ │ │ │ ├── MaxLengthTransformerV2Test.java │ │ │ │ ├── SimpleEnrich.java │ │ │ │ ├── SimpleTransformer.java │ │ │ │ ├── TransformerTest.java │ │ │ │ ├── TransformerV2Test.java │ │ │ │ ├── TransformerV3Test.java │ │ │ │ ├── UAEnrichmentTest.java │ │ │ │ ├── UAEnrichmentV2Test.java │ │ │ │ ├── UserPropertiesConverterTest.java │ │ │ │ ├── gtm │ │ │ │ ├── GTMServerDataTransformerTest.java │ │ │ │ ├── GTMServerDataTransformerV2Test.java │ │ │ │ ├── ServerDataConverterTest.java │ │ │ │ └── ServerDataConverterV2Test.java │ │ │ │ ├── sensors │ │ │ │ └── SensorsDataTransformerV2Test.java │ │ │ │ └── transformer │ │ │ │ └── EventParserFactoryTest.java │ │ │ └── resources │ │ │ ├── data │ │ │ ├── final_data.json │ │ │ ├── sdk_data.json │ │ │ └── transformed_data.json │ │ │ ├── data_cleaned.json │ │ │ ├── data_cleaned_no_attributes.json │ │ │ ├── data_cleaned_no_user.json │ │ │ ├── empty_latest_referrer.json │ │ │ ├── event-data.json │ │ │ ├── event_deser_input.json │ │ │ ├── event_v2 │ │ │ ├── event_v2_session_dataset.json │ │ │ ├── event_v2_session_dataset2.json │ │ │ ├── expected │ │ │ │ ├── etl_runner_v3_event1.json │ │ │ │ ├── etl_runner_v3_event2.json │ │ │ │ ├── etl_user_v2_props_incremental_v1_schema.json │ │ │ │ ├── event_v2_parquet_schema.json │ │ │ │ ├── item_v2_parquet_schema.json │ │ │ │ ├── session_parquet_schema.json │ │ │ │ ├── test_enrich_UA_v2.json │ │ │ │ ├── test_enrich_ip_v2.json │ │ │ │ ├── test_extract_session_from_event.json │ │ │ │ ├── test_extract_session_from_event2.json │ │ │ │ ├── test_max_len_transform_v2.json │ │ │ │ ├── transform_v3_event.json │ │ │ │ ├── transform_v3_event_max_len2.json │ │ │ │ ├── transform_v3_gzip_event.json │ │ │ │ ├── transform_v3_item.json │ │ │ │ ├── transform_v3_item_max_len.json │ │ │ │ ├── transform_v3_traffic_source_event.json │ │ │ │ ├── transform_v3_traffic_source_session.json │ │ │ │ ├── transform_v3_traffic_source_user.json │ │ │ │ ├── transform_v3_user.json │ │ │ │ ├── transform_v3_user2.json │ │ │ │ ├── transform_v3_user_first_open.json │ │ │ │ ├── transform_v3_user_max_len.json │ │ │ │ └── user_v2_parquet_schema.json │ │ │ ├── max_len_input.json │ │ │ ├── transformed_data_event_bot_v2.json │ │ │ ├── transformed_data_event_v2.json │ │ │ └── transformed_data_ua_null.json │ │ │ ├── expected │ │ │ ├── empty_ip_enrich_data_v2.json │ │ │ ├── etl_runner_v2_event1.json │ │ │ ├── etl_runner_v2_event2.json │ │ │ ├── etl_runner_v2_event3.json │ │ │ ├── etl_runner_v2_event4.json │ │ │ ├── etl_runner_v2_event_parameter.json │ │ │ ├── etl_runner_v2_item.json │ │ │ ├── etl_runner_v2_user.json │ │ │ ├── ip_enrich_data_v2.json │ │ │ ├── schema-event.json │ │ │ ├── schema-event_parameter.json │ │ │ ├── schema-event_v2.json │ │ │ ├── schema-item.json │ │ │ ├── schema-item_v2.json │ │ │ ├── schema-session.json │ │ │ ├── schema-user.json │ │ │ ├── schema-user_v2.json │ │ │ ├── test_convert_data_v3.json │ │ │ ├── test_parse_data.json │ │ │ ├── test_parse_line.json │ │ │ ├── test_parse_line_to_db_row_disable_time_shift.json │ │ │ ├── test_parse_line_to_db_row_event_v2.json │ │ │ ├── test_parse_line_to_db_row_zip_event_v2.json │ │ │ ├── transform_v2_event.json │ │ │ ├── transform_v2_event_max_len.json │ │ │ ├── transform_v2_event_max_len2.json │ │ │ ├── transform_v2_event_params.json │ │ │ ├── transform_v2_event_params_max_len.json │ │ │ ├── transform_v2_item0.json │ │ │ ├── transform_v2_item_max_len.json │ │ │ ├── transform_v2_user.json │ │ │ ├── transform_v2_user2.json │ │ │ ├── transform_v2_user_app_start1.json │ │ │ ├── transform_v2_user_app_start2.json │ │ │ ├── transform_v2_user_first_open.json │ │ │ ├── transform_v2_user_ip.json │ │ │ ├── transform_v2_user_max_len.json │ │ │ ├── transform_v2_user_referrer.json │ │ │ ├── transform_v2_user_traffic_source.json │ │ │ └── ua_enrich_data_v2.json │ │ │ ├── gtm-server │ │ │ ├── etl_gtm_user_v2_props_incremental_v1 │ │ │ │ └── update_date=20240312 │ │ │ │ │ └── app_id=GMTServerLogin2 │ │ │ │ │ └── part-00000-8f5a4cb2-f6aa-4b4f-8c6a-08145a4e5b15.c000.snappy.parquet │ │ │ ├── expected │ │ │ │ ├── test_GTM_server_runner_v2_event_v2.json │ │ │ │ ├── test_GTM_server_runner_v2_item_v2.json │ │ │ │ ├── test_GTM_server_runner_v2_session.json │ │ │ │ ├── test_GTM_server_runner_v2_user_v2.json │ │ │ │ ├── test_convert_array_data.json │ │ │ │ ├── test_convert_array_data_v2.json │ │ │ │ ├── test_convert_items_data.json │ │ │ │ ├── test_convert_items_data_v2.json │ │ │ │ ├── test_convert_no_js_client_id.json │ │ │ │ ├── test_convert_no_js_client_id_v2.json │ │ │ │ ├── test_convert_session_start1.json │ │ │ │ ├── test_convert_session_start1_v2.json │ │ │ │ ├── test_convert_session_start2.json │ │ │ │ ├── test_convert_session_start2_v2.json │ │ │ │ ├── test_convert_single_data.json │ │ │ │ ├── test_convert_single_data_v2.json │ │ │ │ ├── test_convert_user_data.json │ │ │ │ ├── test_convert_user_data_user_props_v2.json │ │ │ │ ├── test_convert_user_data_v2.json │ │ │ │ ├── test_etl_runner_data_event.json │ │ │ │ ├── test_gtm_ingestDataToEvent.json │ │ │ │ ├── test_gtm_ingestLineToRow.json │ │ │ │ ├── test_gtm_parseLineToDBRow_event.json │ │ │ │ ├── test_gtm_parseLineToDBRow_item.json │ │ │ │ ├── test_gtm_parseLineToDBRow_user.json │ │ │ │ ├── test_transform_data_brand.json │ │ │ │ ├── test_transform_data_brand_v2.json │ │ │ │ ├── test_transform_data_event1.json │ │ │ │ ├── test_transform_data_event1_post.json │ │ │ │ ├── test_transform_data_event_param.json │ │ │ │ ├── test_transform_data_event_post_v2.json │ │ │ │ ├── test_transform_data_event_v2.json │ │ │ │ ├── test_transform_data_item.json │ │ │ │ ├── test_transform_data_item_v2.json │ │ │ │ ├── test_transform_data_session.json │ │ │ │ ├── test_transform_data_session_start.json │ │ │ │ ├── test_transform_data_session_start_param.json │ │ │ │ ├── test_transform_data_session_start_v2.json │ │ │ │ ├── test_transform_data_user.json │ │ │ │ ├── test_transform_data_user_incremental_v2.json │ │ │ │ ├── test_transform_data_user_login.json │ │ │ │ ├── test_transform_data_user_login2.json │ │ │ │ ├── test_transform_data_user_login2_v2.json │ │ │ │ ├── test_transform_data_user_login_v2.json │ │ │ │ ├── test_transform_data_user_v2.json │ │ │ │ └── test_transform_data_with_session_v2.json │ │ │ ├── server-all-with-ss.json │ │ │ ├── server-all.json │ │ │ ├── server-app1.json │ │ │ ├── server-array.json │ │ │ ├── server-brand.json │ │ │ ├── server-invalid-data.json │ │ │ ├── server-items.json │ │ │ ├── server-session-start.json │ │ │ ├── server-single.json │ │ │ ├── server-user-login.json │ │ │ ├── server-user-login2.json │ │ │ ├── server-user-props.json │ │ │ ├── server-user.json │ │ │ ├── test-convert-corrupt-v2.json │ │ │ ├── test-convert-corrupt.json │ │ │ ├── test-convert-fv-session.json │ │ │ ├── test-convert-no-js-client-id.json │ │ │ └── test-convert-session-start.json │ │ │ ├── ods_event.sql │ │ │ ├── original_data.json │ │ │ ├── original_data_event_name_not_filtered.json │ │ │ ├── original_data_nozip.json │ │ │ ├── original_data_nozip_client_time.json │ │ │ ├── original_data_nozip_item_max_len.json │ │ │ ├── original_data_nozip_max_len.json │ │ │ ├── original_data_nozip_upload_time.json │ │ │ ├── original_data_null.json │ │ │ ├── original_data_raw_json_error.json │ │ │ ├── original_data_raw_text_error.json │ │ │ ├── original_data_single.json │ │ │ ├── original_data_with_app1.json │ │ │ ├── original_data_with_app_start.json │ │ │ ├── original_data_with_empty_item.json │ │ │ ├── original_data_with_empty_user.json │ │ │ ├── original_data_with_error.json │ │ │ ├── original_data_with_first_open.json │ │ │ ├── original_data_with_future_time.json │ │ │ ├── original_data_with_items.json │ │ │ ├── original_data_with_traffic_source.json │ │ │ ├── original_data_with_traffic_source_v2.json │ │ │ ├── original_data_with_user_etl_runner.json │ │ │ ├── original_data_with_user_profile_set.json │ │ │ ├── original_data_with_user_profile_set2.json │ │ │ ├── original_data_with_user_profile_set3.json │ │ │ ├── original_data_with_user_referrer.json │ │ │ ├── original_data_without_seq_id.json │ │ │ ├── partition_data │ │ │ └── year=2022 │ │ │ │ └── month=11 │ │ │ │ ├── day=09 │ │ │ │ └── original_data.json │ │ │ │ ├── day=10 │ │ │ │ └── original_data.json │ │ │ │ └── day=11 │ │ │ │ └── corrupt_data.json │ │ │ ├── rule_config │ │ │ └── app1 │ │ │ │ ├── traffic_source_category_rule_v1.json │ │ │ │ └── traffic_source_channel_rule_v1.json │ │ │ ├── sensordata │ │ │ ├── expected │ │ │ │ └── test_sensors_data_runner.json │ │ │ ├── sensors-web-ua.json │ │ │ └── web_sdk_data.json │ │ │ ├── test_parse_line_to_db_row_event_v2.json │ │ │ ├── transformed_data.json │ │ │ ├── transformed_data_v2.json │ │ │ ├── transformed_data_v2_empty_ip.json │ │ │ ├── transformed_data_with_ip_error.json │ │ │ ├── transformed_data_with_ua_error.json │ │ │ ├── ts │ │ │ ├── traffic_source_category_rule_v0.json │ │ │ └── traffic_source_channel_rule_v0.json │ │ │ └── user_cn_chars.json │ ├── tables │ │ ├── event_v2-table.ts │ │ ├── ingestion-events-table.ts │ │ ├── item_v2-table.ts │ │ ├── session-table.ts │ │ └── user_v2-table.ts │ └── utils │ │ ├── custom-resource.ts │ │ ├── metrics.ts │ │ ├── s3-asset.ts │ │ ├── utils-common.ts │ │ ├── utils-glue.ts │ │ ├── utils-lambda.ts │ │ └── utils-role.ts ├── data-reporting-quicksight-stack.ts ├── ingestion-server-stack.ts ├── ingestion-server │ ├── custom-resource │ │ ├── delete-ecs-cluster.ts │ │ ├── delete-ecs-cluster │ │ │ └── index.ts │ │ ├── update-alb-rules.ts │ │ └── update-alb-rules │ │ │ └── index.ts │ ├── kafka-s3-connector │ │ ├── custom-resource.ts │ │ ├── custom-resource │ │ │ └── kafka-s3-sink-connector │ │ │ │ └── index.ts │ │ ├── iam.ts │ │ ├── kafka-s3-connector.ts │ │ └── metrics-connector.ts │ ├── kinesis-data-stream │ │ ├── kinesis-data-stream-nested-stack.ts │ │ ├── kinesis-to-s3-lambda │ │ │ └── index.ts │ │ ├── kinesis-to-s3.ts │ │ └── private │ │ │ ├── iam.ts │ │ │ ├── kinesis.ts │ │ │ ├── lambda.ts │ │ │ ├── metrics-kinesis.ts │ │ │ └── sg.ts │ └── server │ │ ├── condition.ts │ │ ├── images │ │ ├── nginx │ │ │ ├── Dockerfile │ │ │ └── config │ │ │ │ ├── docker-entrypoint.sh │ │ │ │ └── nginx.conf │ │ └── vector │ │ │ ├── Dockerfile │ │ │ └── config │ │ │ ├── entrypoint.sh │ │ │ ├── vector-global.toml │ │ │ ├── vector-kinesis-ack.toml │ │ │ ├── vector-kinesis-batch.toml │ │ │ ├── vector-msk-ack.toml │ │ │ ├── vector-msk-batch.toml │ │ │ ├── vector-s3.toml │ │ │ └── vector.toml │ │ ├── ingestion-server.ts │ │ ├── parameter.ts │ │ └── private │ │ ├── aga.ts │ │ ├── alb.ts │ │ ├── cfn-nag.ts │ │ ├── ecr.ts │ │ ├── ecs-cluster.ts │ │ ├── ecs-service.ts │ │ ├── iam.ts │ │ ├── metircs-kafka.ts │ │ ├── metircs-server.ts │ │ ├── route53.ts │ │ └── sg.ts ├── kafka-s3-connector-stack.ts ├── main.ts ├── metrics-stack.ts ├── metrics │ ├── add-sns-subscription.ts │ ├── custom-resource │ │ ├── add-sns-subscription │ │ │ └── index.ts │ │ ├── get-interval │ │ │ └── index.ts │ │ ├── put-dashboard │ │ │ ├── index.ts │ │ │ └── set-alarms-action.ts │ │ └── set-metrics-widgets │ │ │ └── index.ts │ ├── get-interval-custom-resource.ts │ ├── metrics-widgets-custom-resource.ts │ ├── metrics.ts │ ├── settings.ts │ └── util.ts ├── private │ └── function.ts ├── reporting │ ├── lambda │ │ └── custom-resource │ │ │ └── quicksight │ │ │ ├── index.ts │ │ │ └── network-interface-check.ts │ ├── network-interface-check-custom-resource.ts │ ├── parameter.ts │ ├── private │ │ ├── dashboard.ts │ │ ├── dataset-col-def.ts │ │ ├── iam.ts │ │ ├── template-def-spice.json │ │ └── template-def.json │ └── quicksight-custom-resource.ts └── service-catalog-appregistry-stack.ts ├── test ├── analytics │ ├── analytics-on-redshift │ │ ├── data-analytics-redshift-stack.test.ts │ │ ├── lambda │ │ │ ├── clear-expire-events │ │ │ │ ├── check-clear-status.test.ts │ │ │ │ └── clear-expire-events.test.ts │ │ │ ├── custom-resources │ │ │ │ ├── create-redshift-namespace.test.ts │ │ │ │ ├── create-redshift-user.test.ts │ │ │ │ ├── create-schemas.test.ts │ │ │ │ ├── get-source-prefix.test.ts │ │ │ │ └── redshift-associate-iam-role.test.ts │ │ │ ├── load-data │ │ │ │ ├── check-load-status.test.ts │ │ │ │ ├── create-manifest.test.ts │ │ │ │ ├── has-more-job-new.test.ts │ │ │ │ ├── load-manifest-to-redshift.test.ts │ │ │ │ ├── put-ods-source-to-store.test.ts │ │ │ │ ├── refresh-views.test.ts │ │ │ │ └── skip-running-workflow.test.ts │ │ │ ├── refresh-materialized-views-workflow │ │ │ │ ├── check-refresh-sp-status.test.ts │ │ │ │ ├── check-refresh-view-status.test.ts │ │ │ │ ├── check-refresh-workflow-start.test.ts │ │ │ │ ├── check-start-sp-refresh.test.ts │ │ │ │ ├── get-refresh-viewlist.test.ts │ │ │ │ ├── parse-timezone-with-appId-list.test.ts │ │ │ │ ├── refresh-basic-view.test.ts │ │ │ │ └── refresh-sp.test.ts │ │ │ ├── scan-metadata-workflow │ │ │ │ ├── check-scan-metadata-status.test.ts │ │ │ │ ├── check-workflow-start.test.ts │ │ │ │ ├── scan-metadata.test.ts │ │ │ │ ├── store-metadata-into-ddb.test.ts │ │ │ │ └── update-workflow-info.test.ts │ │ │ └── sql-execution-sfn │ │ │ │ ├── sql-execution-sfn-provisioned.test.ts │ │ │ │ └── sql-execution-sfn-serverless.test.ts │ │ └── workflow │ │ │ ├── load-data-stepfuncs.json │ │ │ ├── load-data-stepfuncs.test.ts │ │ │ ├── load-data-sub-stepfuncs.json │ │ │ ├── sql-execution-sfn.json │ │ │ └── sql-execution-sfn.test.ts │ └── modeling-on-athena │ │ └── data-modeliing-athena-stack.test.ts ├── common │ ├── alb.test.ts │ ├── jest.ts │ ├── lambda-context.ts │ ├── lambda-events.ts │ └── s3.test.ts ├── constants.ts ├── control-plane │ ├── alb-control-plane-stack.test.ts │ ├── alb-lambda-portal.test.ts │ ├── click-stream-api.test.ts │ ├── cloudfront-control-plane-stack.test.ts │ ├── cloudfront-s3-portal.test.ts │ ├── lambda │ │ ├── auth.test.ts │ │ ├── batch-insert-ddb.test.ts │ │ ├── listen-stack-status.test.ts │ │ ├── listen-state-status.test.ts │ │ ├── sfn-action.test.ts │ │ └── sfn-workflow.test.ts │ └── test-utils.ts ├── data-pipeline │ ├── copy-asset.test.ts │ ├── data-pipeline-stack.test.ts │ ├── emr-job-listener.test.ts │ ├── emr-job-submitter.test.ts │ ├── emr-serverless-app-cn.test.ts │ ├── emr-serverless-app.test.ts │ └── partition-syncer │ │ ├── glue-util.test.ts │ │ └── partition-syncer.test.ts ├── fs-utils.ts ├── ingestion-server │ ├── custom-resource │ │ ├── delete-ecs-cluster.test.ts │ │ └── update-alb-rules.test.ts │ ├── kafka-s3-connector │ │ ├── custom-resource │ │ │ └── kafka-s3-sink-connector.test.ts │ │ └── kafka-s3-connector-stack.test.ts │ ├── kinesis-data-stream │ │ └── kinesis-to-s3-lambda.test.ts │ └── server │ │ ├── TestTask.ts │ │ ├── ingestion-server-stack.test.ts │ │ ├── ingestion-server.test.ts │ │ └── tier-setting.ts ├── jestEnv.js ├── metrics │ ├── add-sns-subscription.test.ts │ ├── get-interval.test.ts │ ├── metrics-stack.test.ts │ ├── put-dashboard.test.ts │ └── set-metrics-widgets.test.ts ├── reporting │ └── quicksight │ │ ├── data-reporting-quicksight-stack.test.ts │ │ └── lambda │ │ ├── network-interface-check.test.ts │ │ └── quicksight-resource.test.ts ├── rules.ts ├── service-catalog-appregistry │ └── service-catalog-appregistry-stack.test.ts └── utils.ts ├── tsconfig.dev.json └── tsconfig.json /.cfnnag_global_suppress_list: -------------------------------------------------------------------------------- 1 | --- 2 | RulesToSuppress: 3 | - id: W58 4 | reason: False alarm on the Lambda function, which uses a custom role instead of having the ability to write CloudWatch logs. 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | cdk.out/ 2 | test* 3 | **/node_modules/ 4 | script 5 | coverage 6 | deployment/ 7 | docs/ 8 | coverage/ 9 | .git/ 10 | .gitlab/ 11 | .github/ -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: Report a bug 4 | title: "" 5 | labels: bug, needs-triage 6 | --- 7 | ## Summary 8 | 9 | 10 | 11 | ## Steps to reproduce 12 | 13 | 14 | 15 | ## What is the current bug behavior? 16 | 17 | 18 | 19 | ## What is the expected correct behavior? 20 | 21 | 22 | 23 | ## Relevant logs and/or screenshots 24 | 25 | 26 | 27 | ## Possible fixes 28 | 29 | 30 | 31 | --- 32 | 33 | This is :bug: Bug Report -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: Request a new feature 4 | title: "" 5 | labels: feature-request, needs-triage 6 | --- 7 | 8 | ## Describe the feature 9 | 10 | 11 | 12 | ## Use Case 13 | 14 | 15 | 16 | ## Proposed Solution - Optional 17 | 18 | 19 | 20 | ## Other Information 21 | 22 | 23 | 24 | **Notes**: 25 | 26 | - [ ] this feature has an UI update 27 | - [ ] this feature contains an implementation guide update 28 | 29 | --- 30 | 31 | This is a :rocket: Feature Request -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/general-issues.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U00002753 General Issue" 3 | about: Create a new issue 4 | title: "" 5 | labels: needs-triage, guidance 6 | --- 7 | 8 | ## :question: General Issue 9 | 10 | ### The Question 11 | 16 | 17 | ### Other information 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Basic `dependabot.yml` file with 2 | 3 | version: 2 4 | updates: 5 | # Enable version updates for github-actions 6 | - package-ecosystem: "github-actions" 7 | directory: "/" 8 | # Check the actions for updates every week 9 | schedule: 10 | interval: "weekly" -------------------------------------------------------------------------------- /.gitlab/ci-templates/cfn-nag.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - qa 5 | cfn-nag: 6 | tags: 7 | - arch:amd64 8 | image: 9 | name: stelligent/cfn_nag 10 | entrypoint: 11 | - /bin/sh 12 | - -c 13 | stage: qa 14 | script: 15 | - set -x; cfn_nag -f -b .cfnnag_global_suppress_list build/cdk.out/*.template.json 16 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/data-pipeline-spark-etl.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | etl-unit-test: 6 | stage: build 7 | image: 8 | name: public.ecr.aws/docker/library/gradle:7.6-jdk17 9 | variables: {} 10 | rules: 11 | - if: $CI_MERGE_REQUEST_IID 12 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 13 | before_script: 14 | - cd ./src/data-pipeline/spark-etl 15 | script: 16 | - gradle clean build 17 | - gradle jacocoAggregatedReport 18 | artifacts: 19 | reports: 20 | junit: ./src/data-pipeline/spark-etl/build/test-results/test/TEST-*.xml 21 | coverage_report: 22 | coverage_format: cobertura 23 | path: ./src/data-pipeline/spark-etl/build/reports/jacoco/jacocoAggregatedReport/jacocoAggregatedReport.xml 24 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/etl-plugin-samples.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | etl-plugin-samples-unit-test: 6 | stage: build 7 | image: 8 | name: public.ecr.aws/docker/library/gradle:7.6-jdk17 9 | variables: {} 10 | rules: 11 | - if: $CI_MERGE_REQUEST_IID 12 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 13 | before_script: 14 | - cd ./examples/custom-plugins 15 | script: 16 | - ./build.sh 17 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/git-secrets-scan.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | secrets-scan: 6 | stage: build 7 | image: 8 | name: public.ecr.aws/docker/library/debian:bullseye 9 | before_script: 10 | - apt update 11 | - apt install -y git git-secrets 12 | script: 13 | - git secrets --register-aws 14 | - git secrets --scan 15 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/license-check.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | license header: 6 | stage: build 7 | image: 8 | name: public.ecr.aws/amazonlinux/amazonlinux:2 9 | rules: 10 | - if: $CI_MERGE_REQUEST_IID 11 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 12 | variables: 13 | LICENSE_FILE: /tmp/license-check/license-header.txt 14 | before_script: 15 | - yum install -y tar gzip git 16 | - mkdir -p /tmp/license-check && curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash -s -- -b /tmp/license-check 17 | - sed -n 1,12p .projenrc.js > $LICENSE_FILE && cat $LICENSE_FILE 18 | script: 19 | - /tmp/license-check/license-header-checker -a -r -i node_modules,cdk.out,coverage $LICENSE_FILE . ts tsx js java && ([[ -z `git status -s` ]] || (echo "Found files violdate with license header" && exit 1)) 20 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/mutations-check.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | mutations-check: 6 | stage: build 7 | rules: 8 | - if: $CI_MERGE_REQUEST_IID 9 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 10 | script: 11 | - npm install -g pnpm@9.15.3 12 | - pnpm install 13 | - pnpm projen 14 | - pnpm projen eslint 15 | - git add . 16 | - git diff --staged --patch --exit-code > .repo.patch || export mutations_happened=true 17 | - if [ "$mutations_happened" = "true" ]; then cat .repo.patch && exit 1; fi; 18 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/prlint.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | lint: 6 | rules: 7 | - if: $CI_PIPELINE_SOURCE == "merge_request_event" 8 | stage: build 9 | script: 10 | - '[[ "$CI_MERGE_REQUEST_TITLE" =~ ^(feat|fix|chore|docs|tests|ci): ]] || (echo "no commit type is specified in merge request title" && exit 1)' 11 | -------------------------------------------------------------------------------- /.gitlab/ci-templates/semgrep.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | stages: 4 | - build 5 | semgrep: 6 | tags: 7 | - arch:amd64 8 | stage: build 9 | image: 10 | name: returntocorp/semgrep 11 | rules: 12 | - if: $CI_MERGE_REQUEST_IID 13 | - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 14 | variables: 15 | SEMGREP_RULES: p/default 16 | SEMGREP_GITLAB_JSON: "1" 17 | script: 18 | - semgrep ci --gitlab-sast > gl-sast-report.json || true 19 | artifacts: 20 | reports: 21 | sast: gl-sast-report.json 22 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/bugs.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | (Summarize the bug encountered concisely) 4 | 5 | ## Steps to reproduce 6 | 7 | (How one can reproduce the issue - this is very important) 8 | 9 | ## What is the current bug behavior? 10 | 11 | (What actually happens) 12 | 13 | ## What is the expected correct behavior? 14 | 15 | (What you should see instead) 16 | 17 | ## Relevant logs and/or screenshots 18 | 19 | (Paste any relevant logs - use code blocks (```) to format console output, logs, and code, as 20 | it's very hard to read otherwise.) 21 | 22 | ## Possible fixes 23 | 24 | (If you can, link to the line of code that might be responsible for the problem) 25 | 26 | /label ~Bug -------------------------------------------------------------------------------- /.gitlab/issue_templates/features.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | (describe what the feature does) 4 | 5 | ## What is the expected correct behavior? 6 | 7 | (What you should see instead) 8 | 9 | /label ~Feature 10 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | queue_rules: 4 | - name: default 5 | update_method: merge 6 | conditions: 7 | - "#approved-reviews-by>=1" 8 | - -label~=(do-not-merge) 9 | - status-success=build 10 | merge_method: squash 11 | commit_message_template: |- 12 | {{ title }} (#{{ number }}) 13 | 14 | {{ body }} 15 | pull_request_rules: 16 | - name: Automatic merge on approval and successful build 17 | actions: 18 | delete_head_branch: {} 19 | queue: 20 | name: default 21 | conditions: 22 | - "#approved-reviews-by>=1" 23 | - -label~=(do-not-merge) 24 | - status-success=build 25 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | permissions-backup.acl 7 | /.mergify.yml 8 | /test/ 9 | /tsconfig.dev.json 10 | /src/ 11 | !/lib/ 12 | !/lib/**/*.js 13 | !/lib/**/*.d.ts 14 | dist 15 | /tsconfig.json 16 | /.github/ 17 | /.vscode/ 18 | /.idea/ 19 | /.projenrc.js 20 | tsconfig.tsbuildinfo 21 | /.eslintrc.json 22 | !/assets/ 23 | cdk.out/ 24 | .cdk.staging/ 25 | /.gitattributes 26 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | resolution-mode=highest 4 | auto-install-peers=true 5 | -------------------------------------------------------------------------------- /.semgrepignore: -------------------------------------------------------------------------------- 1 | .github/workflows/build.yml -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | Reporting Security Issues 2 | ---------------------------------------------------------------------------------------------------------- 3 | We take all security reports seriously. When we receive such reports, we will investigate and 4 | subsequently address any potential vulnerabilities as quickly as possible. If you discover a potential 5 | security issue in this project, please notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or 6 | directly via email to [AWS Security](mailto:aws-security@amazon.com). Please do not create a public GitHub issue in this project. -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node -P tsconfig.json --prefer-ts-exts src/main.ts", 3 | "output": "cdk.out", 4 | "build": "pnpm dlx projen bundle", 5 | "watch": { 6 | "include": [ 7 | "src/**/*.ts", 8 | "test/**/*.ts" 9 | ], 10 | "exclude": [ 11 | "README.md", 12 | "cdk*.json", 13 | "**/*.d.ts", 14 | "**/*.js", 15 | "tsconfig.json", 16 | "package*.json", 17 | "yarn.lock", 18 | "node_modules" 19 | ] 20 | }, 21 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"pnpm dlx projen\"." 22 | } 23 | -------------------------------------------------------------------------------- /deployment/cdk-solution-helper/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk-solution-helper", 3 | "version": "0.1.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "cdk-solution-helper", 9 | "version": "0.1.0", 10 | "license": "Apache-2.0", 11 | "dependencies": { 12 | "fs": "0.0.1-security" 13 | }, 14 | "devDependencies": { 15 | "fs": "0.0.1-security" 16 | } 17 | }, 18 | "node_modules/fs": { 19 | "version": "0.0.1-security", 20 | "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", 21 | "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=", 22 | "dev": true 23 | } 24 | }, 25 | "dependencies": { 26 | "fs": { 27 | "version": "0.0.1-security", 28 | "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", 29 | "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=", 30 | "dev": true 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /deployment/cdk-solution-helper/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk-solution-helper", 3 | "version": "0.1.0", 4 | "license": "Apache-2.0", 5 | "description": "cdk solution helper", 6 | "author": {"name": "Amazon Web Services", "url": "https://aws.amazon.com/solutions"}, 7 | "devDependencies": { 8 | "fs": "0.0.1-security" 9 | }, 10 | "dependencies": { 11 | "fs": "0.0.1-security" 12 | } 13 | } -------------------------------------------------------------------------------- /deployment/exclude-open-source-list.txt: -------------------------------------------------------------------------------- 1 | 2 | Config 3 | .nightswatch 4 | .crux_template.md 5 | test/functional_tests 6 | 7 | -------------------------------------------------------------------------------- /deployment/solution_config: -------------------------------------------------------------------------------- 1 | SOLUTION_ID='SO0219' 2 | 3 | SOLUTION_ECR_ACCOUNT=366590864501 4 | SOLUTION_ECR_REPO_NAME='clickstream-analytics-on-aws' 5 | SOLUTION_ECR_BUILD_VERSION=$(curl -s https://s3.amazonaws.com/aws-gcr-solutions/clickstream-analytics-on-aws/latest/version) 6 | SOLUTION_CN_TEMPLATES=( 7 | https://aws-gcr-solutions.s3.cn-north-1.amazonaws.com.cn/clickstream-analytics-on-aws/latest/cn/cloudfront-s3-control-plane-stack-cn.template.json 8 | https://aws-gcr-solutions.s3.cn-north-1.amazonaws.com.cn/clickstream-analytics-on-aws/latest/cn/private-exist-vpc-control-plane-stack.template.json 9 | ) 10 | -------------------------------------------------------------------------------- /deployment/test-build-dist-1.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | export SOLUTION_NAME='test-clickstream-001' 6 | export BUILD_VERSION=v1.1.0-ci-latest 7 | 8 | export BSS_IMAGE_ASSET_REPOSITORY_NAME=clickstream 9 | export AWS_ASSET_ACCOUNT_ID=555555555555 10 | export AWS_CN_ASSET_ACCOUNT_ID=444455556666 11 | export REGIONS='us-west-1,us-west-2' 12 | export AWS_ASSET_PUBLISH_ROLE='arn:aws:iam::555555555555:role/cross-account-publishing-role' 13 | export GLOBAL_ASSETS='default/' 14 | export CN_ASSETS='cn/' 15 | export TARGET='feature-rel/test' 16 | export OUT_TAG_IMAGE_SHELL_FILE='test-tag-images.sh' 17 | 18 | export SOLUTION_ECR_ACCOUNT=$AWS_ASSET_ACCOUNT_ID 19 | export SOLUTION_ECR_REPO_NAME=$SOLUTION_NAME 20 | 21 | # in gcr pipeline 22 | export IS_IN_GCR_PIPELINE=1 23 | export DIST_OUTPUT_BUCKET='aws-gcr-solutions' 24 | ./deployment/build-s3-dist-1.sh solution-bucket $SOLUTION_NAME $BUILD_VERSION 25 | node ./deployment/test/index.js 26 | -------------------------------------------------------------------------------- /deployment/test-build-dist.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | export SOLUTION_NAME='test-clickstream-001' 6 | export BUILD_VERSION=ci-latest-001 7 | 8 | # in global pipeline 9 | export CODEBUILD_SRC_DIR=`pwd` 10 | cd deployment/ 11 | export IS_IN_GCR_PIPELINE=0 12 | export DIST_OUTPUT_BUCKET='aws-test-solutions' 13 | ./build-s3-dist.sh solution-bucket $SOLUTION_NAME $BUILD_VERSION 14 | source ./solution_config 15 | export SOLUTION_ID 16 | export SOLUTION_NAME 17 | export SOLUTION_TRADEMARKEDNAME 18 | export SOLUTION_ECR_BUILD_VERSION 19 | export SOLUTION_ECR_ACCOUNT 20 | export SOLUTION_ECR_REPO_NAME 21 | node ./test/index.js 22 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ -------------------------------------------------------------------------------- /docs/en/analytics/data-mgmt/index.md: -------------------------------------------------------------------------------- 1 | # Data management 2 | Data Management helps you manage your clickstream data. This module provides the following features: 3 | 4 | 1. Metadata management. The solution automatically scans the data in your clickstream database to generate metadata, allowing you to view names and descriptions of events, event parameters, and user attributes, as well as update their display names, descriptions, and data dictionaries. [Learn more](../data-mgmt/metadata-mgmt.md) 5 | 2. Traffic source configuration. You can update the traffic source categorization rules in this tab. [Learn more](../data-mgmt/traffic-source.md) 6 | 7 | ## Accessing Data Management 8 | To access Data Management module, follow below steps: 9 | 10 | 1. Go to **Clickstream Analytics on AWS Console**, in the **Navigation Bar**, click on "**Analytics Studio**", a new tab will be opened in your browser. 11 | 2. In the Analytics Studio page, click the **Data Management** icon in the left navigation panel. -------------------------------------------------------------------------------- /docs/en/deployment/common-parameters.md: -------------------------------------------------------------------------------- 1 | | Admin User Email (`Email`) | `` | Specify the email of the Administrator. This email address will receive a temporary password to access the {{ solution_name }} web console. You can create more users directly in the provisioned Cognito User Pool after launching the solution. | 2 | | IAM Role Prefix (`IamRolePrefix`) | `` | Specify the prefix for the name of IAM roles created in the solution. | 3 | | IAM Role Boundary ARN (`IamRoleBoundaryArn`) | `` | Specify [the permissions boundary for the IAM roles][iam-boundary] created in the solution. | 4 | 5 | [iam-boundary]: https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html -------------------------------------------------------------------------------- /docs/en/deployment/domain-parameters.md: -------------------------------------------------------------------------------- 1 | | Parameter | Default | Description | 2 | | ---------- | ---------------- | ------------------------------------------------------------ | 3 | | Host Zone ID (`HostedZoneId`) | `` | Choose the public hosted zone ID of Amazon Route 53. | 4 | | Host Zone Name (`HostedZoneName`) | `` | The domain name of the public hosted zone, for example, `example.com`. | 5 | | Record Name (`RecordName`) | `` | The sub name (as known as record name in R53) of the domain name of console. For example, enter `clickstream`, if you want to use custom domain `clickstream.example.com` for the console. | 6 | -------------------------------------------------------------------------------- /docs/en/deployment/oidc-parameters.md: -------------------------------------------------------------------------------- 1 | | OpenID Connector Client Id (`OIDCClientId`) | `` | OpenID Connect client Id. | 2 | | OpenID Connector Issuer (`OIDCProvider`) | `` | OpenID Connect provider issuer. The issuer must begin with `https://` | 3 | {% 4 | include-markdown "./common-parameters.md" 5 | comments=false 6 | %} -------------------------------------------------------------------------------- /docs/en/deployment/tls-note.md: -------------------------------------------------------------------------------- 1 | By default, this deployment uses TLSv1.0 and TLSv1.1 in CloudFront. However, we recommend that you manually configure CloudFront to use the more secure TLSv1.2/TLSv1.3 and apply for a certificate and custom domain to enable this. We highly recommend that you update your TLS configuration and cipher suite selection according to the following recommendations: 2 | 3 | - **Transport Layer Security Protocol**: Upgrade to TLSv1.2 or higher 4 | - **Key Exchange**: ECDHE 5 | - **Block Cipher Mode of Operation**: GCM 6 | - **Authentication**: ECDSA 7 | - **Encryption Cipher**: AES256 8 | - **Message Authentication**: SHA(256/384/any hash function except for SHA1) 9 | 10 | Such as TLSv1.2_2021 can meet the above recommendations. -------------------------------------------------------------------------------- /docs/en/images/OIDC/adfs-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/adfs-9.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/authing-authorization-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/authing-authorization-configuration.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/authing-endpoint-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/authing-endpoint-info.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/cognito-hostUI-new.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/cognito-hostUI-new.jpeg -------------------------------------------------------------------------------- /docs/en/images/OIDC/cognito-new-console-clientID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/cognito-new-console-clientID.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/cognito-new-console-userpoolID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/cognito-new-console-userpoolID.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/keycloak-OIDC-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/keycloak-OIDC-config.png -------------------------------------------------------------------------------- /docs/en/images/OIDC/keycloak-example-realm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/OIDC/keycloak-example-realm.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/acquisition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/acquisition.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/activity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/activity.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/details.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/device.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/device.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/engagement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/engagement.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/path.png -------------------------------------------------------------------------------- /docs/en/images/analytics/dashboard/retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/dashboard/retention.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/access-studio-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/access-studio-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/access-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/access-studio.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/acquisition-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/acquisition-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/activity-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/activity-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-add-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-add-dataset.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-adjust-pivot-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-adjust-pivot-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-adjust-pivot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-adjust-pivot.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-calculated-field-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-calculated-field-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-calculated-field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-calculated-field.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyze-new-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyze-new-dataset.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyzes-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyzes-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/analyzes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/analyzes.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/attribution-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/attribution-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/crash-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/crash-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/custom-dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/custom-dashboard-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/custom-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/custom-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/dashboard-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/engagement-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/engagement-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-event-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-event-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-event.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-event.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-funnel-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-funnel-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-funnel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-funnel.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-path-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-path-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-path.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-retention-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-retention-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore-retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore-retention.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/explore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/explore.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/metadata-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/metadata-en.png -------------------------------------------------------------------------------- /docs/en/images/analytics/explore/metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/analytics/explore/metadata.png -------------------------------------------------------------------------------- /docs/en/images/architecture/01-architecture-end-to-end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/01-architecture-end-to-end.png -------------------------------------------------------------------------------- /docs/en/images/architecture/02-architecture-ingestion-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/02-architecture-ingestion-module.png -------------------------------------------------------------------------------- /docs/en/images/architecture/03-architecture-data-processing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/03-architecture-data-processing.png -------------------------------------------------------------------------------- /docs/en/images/architecture/04-architecture-data-modeling-redshift.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/04-architecture-data-modeling-redshift.png -------------------------------------------------------------------------------- /docs/en/images/architecture/05-architecture-data-modeling-scan-metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/05-architecture-data-modeling-scan-metadata.png -------------------------------------------------------------------------------- /docs/en/images/architecture/06-architecture-data-modeling-athena.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/06-architecture-data-modeling-athena.png -------------------------------------------------------------------------------- /docs/en/images/architecture/07-architecture-reporting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/07-architecture-reporting.png -------------------------------------------------------------------------------- /docs/en/images/architecture/08-analytics-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/architecture/08-analytics-studio.png -------------------------------------------------------------------------------- /docs/en/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/favicon.ico -------------------------------------------------------------------------------- /docs/en/images/get-started/explore-dashboard-zh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/get-started/explore-dashboard-zh.png -------------------------------------------------------------------------------- /docs/en/images/get-started/explore-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/get-started/explore-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/get-started/qs-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/get-started/qs-dashboard.png -------------------------------------------------------------------------------- /docs/en/images/launch-stack.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/launch-stack.webp -------------------------------------------------------------------------------- /docs/en/images/pipe-mgmt/concepts-v11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/pipe-mgmt/concepts-v11.png -------------------------------------------------------------------------------- /docs/en/images/pipe-mgmt/concepts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/pipe-mgmt/concepts.png -------------------------------------------------------------------------------- /docs/en/images/pipe-mgmt/table_relationship.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/pipe-mgmt/table_relationship.png -------------------------------------------------------------------------------- /docs/en/images/sdk-manual/android_raw_folder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/sdk-manual/android_raw_folder.png -------------------------------------------------------------------------------- /docs/en/images/sdk-manual/replace-in-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/sdk-manual/replace-in-file.png -------------------------------------------------------------------------------- /docs/en/images/sdk-manual/swift_add_amplify_config_json_file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/sdk-manual/swift_add_amplify_config_json_file.png -------------------------------------------------------------------------------- /docs/en/images/sdk-manual/swift_add_package.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/sdk-manual/swift_add_package.png -------------------------------------------------------------------------------- /docs/en/images/sdk-manual/swift_add_package_url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/sdk-manual/swift_add_package_url.png -------------------------------------------------------------------------------- /docs/en/images/solution-overview.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/en/images/solution-overview.webp -------------------------------------------------------------------------------- /docs/en/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | -------------------------------------------------------------------------------- /docs/en/notices.md: -------------------------------------------------------------------------------- 1 | Customers are responsible for making their own independent assessment of the information in this document. This document: (a) is for informational purposes only, (b) represents Amazon Web Services current product offerings and practices, which are subject to change without notice, and (c) does not create any commitments or assurances from Amazon Web Services and its affiliates, suppliers or licensors. Amazon Web Services products or services are provided "as is" without warranties, representations, or conditions of any kind, whether express or implied. Amazon Web Services responsibilities and liabilities to its customers are controlled by Amazon Web Services agreements, and this document is not part of, nor does it modify, any agreement between Amazon Web Services and its customers. 2 | 3 | {{solution_name}} is licensed under the terms of the Apache License Version 2.0 available at [The Apache Software Foundation][foundation]. 4 | 5 | [foundation]: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- /docs/en/pipeline-mgmt/data-processing/index.md: -------------------------------------------------------------------------------- 1 | # Data Processing 2 | {{solution_name}} provides an inbuilt data schema to parse and model the raw event data sent from your web and mobile apps, which makes it easy for you to analyze the data in analytics engines (such as Redshift and Athena). 3 | 4 | Data Processing module includes two functionalities: 5 | 6 | - **Transformation**: Extract the data from files sank by ingestion module, then parse each event data and transform them to solution data model. 7 | - **Enrichment**: Add additional dimensions/fields to event data. 8 | 9 | 10 | This chapter includes: 11 | 12 | - [Data schema](./data-schema.md) 13 | - [Configure execution parameters](./configure-execution-para.md) 14 | - [Configure ETL custom plugins](./configure-plugin.md) 15 | -------------------------------------------------------------------------------- /docs/en/source.md: -------------------------------------------------------------------------------- 1 | Visit our [GitHub repository][source] to download the source code for this solution. The {{ solution_name }} template is generated using the [AWS Cloud Development Kit (CDK)][cdk]. Refer to the [README.md][readme] file for additional information. 2 | 3 | [source]: {{ source_repo }} 4 | [cdk]: http://aws.amazon.com/cdk/ 5 | [readme]: {{ source_repo }}/blob/main/README.md 6 | -------------------------------------------------------------------------------- /docs/en/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .icon_check { 2 | color: green; 3 | } 4 | 5 | .icon_cross { 6 | color: red; 7 | } 8 | 9 | /* Maximum space for text block */ 10 | .md-grid { 11 | max-width: 100%; /* or 100%, if you want to stretch to full-width */ 12 | } -------------------------------------------------------------------------------- /docs/images/OIDC/adfs-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/adfs-9.png -------------------------------------------------------------------------------- /docs/images/OIDC/authing-authorization-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/authing-authorization-configuration.png -------------------------------------------------------------------------------- /docs/images/OIDC/authing-endpoint-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/authing-endpoint-info.png -------------------------------------------------------------------------------- /docs/images/OIDC/cognito-hostUI-new.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/cognito-hostUI-new.jpeg -------------------------------------------------------------------------------- /docs/images/OIDC/cognito-new-console-clientID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/cognito-new-console-clientID.png -------------------------------------------------------------------------------- /docs/images/OIDC/cognito-new-console-userpoolID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/cognito-new-console-userpoolID.png -------------------------------------------------------------------------------- /docs/images/OIDC/keycloak-OIDC-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/keycloak-OIDC-config.png -------------------------------------------------------------------------------- /docs/images/OIDC/keycloak-example-realm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/OIDC/keycloak-example-realm.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/acquisition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/acquisition.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/activity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/activity.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/details.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/device.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/device.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/engagement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/engagement.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/path.png -------------------------------------------------------------------------------- /docs/images/analytics/dashboard/retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/dashboard/retention.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/access-studio-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/access-studio-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/access-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/access-studio.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/acquisition-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/acquisition-dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/activity-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/activity-dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-add-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-add-dataset.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-adjust-pivot-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-adjust-pivot-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-adjust-pivot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-adjust-pivot.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-calculated-field-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-calculated-field-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-calculated-field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-calculated-field.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyze-new-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyze-new-dataset.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyzes-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyzes-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/analyzes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/analyzes.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/attribution-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/attribution-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/crash-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/crash-dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/custom-dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/custom-dashboard-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/custom-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/custom-dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/dashboard-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/engagement-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/engagement-dashboard.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-event-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-event-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-event.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-event.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-funnel-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-funnel-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-funnel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-funnel.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-path-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-path-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-path.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-retention-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-retention-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore-retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore-retention.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/explore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/explore.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/metadata-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/metadata-en.png -------------------------------------------------------------------------------- /docs/images/analytics/explore/metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/analytics/explore/metadata.png -------------------------------------------------------------------------------- /docs/images/architecture/01-architecture-end-to-end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/01-architecture-end-to-end.png -------------------------------------------------------------------------------- /docs/images/architecture/02-architecture-ingestion-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/02-architecture-ingestion-module.png -------------------------------------------------------------------------------- /docs/images/architecture/03-architecture-data-processing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/03-architecture-data-processing.png -------------------------------------------------------------------------------- /docs/images/architecture/04-architecture-data-modeling-redshift.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/04-architecture-data-modeling-redshift.png -------------------------------------------------------------------------------- /docs/images/architecture/05-architecture-data-modeling-scan-metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/05-architecture-data-modeling-scan-metadata.png -------------------------------------------------------------------------------- /docs/images/architecture/06-architecture-data-modeling-athena.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/06-architecture-data-modeling-athena.png -------------------------------------------------------------------------------- /docs/images/architecture/07-architecture-reporting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/07-architecture-reporting.png -------------------------------------------------------------------------------- /docs/images/architecture/08-analytics-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/architecture/08-analytics-studio.png -------------------------------------------------------------------------------- /docs/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/favicon.ico -------------------------------------------------------------------------------- /docs/images/get-started/explore-dashboard-zh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/get-started/explore-dashboard-zh.png -------------------------------------------------------------------------------- /docs/images/get-started/explore-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/get-started/explore-dashboard.png -------------------------------------------------------------------------------- /docs/images/get-started/qs-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/get-started/qs-dashboard.png -------------------------------------------------------------------------------- /docs/images/launch-stack.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/launch-stack.webp -------------------------------------------------------------------------------- /docs/images/pipe-mgmt/concepts-v11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/pipe-mgmt/concepts-v11.png -------------------------------------------------------------------------------- /docs/images/pipe-mgmt/concepts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/pipe-mgmt/concepts.png -------------------------------------------------------------------------------- /docs/images/pipe-mgmt/table_relationship.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/pipe-mgmt/table_relationship.png -------------------------------------------------------------------------------- /docs/images/sdk-manual/android_raw_folder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/sdk-manual/android_raw_folder.png -------------------------------------------------------------------------------- /docs/images/sdk-manual/replace-in-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/sdk-manual/replace-in-file.png -------------------------------------------------------------------------------- /docs/images/sdk-manual/swift_add_amplify_config_json_file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/sdk-manual/swift_add_amplify_config_json_file.png -------------------------------------------------------------------------------- /docs/images/sdk-manual/swift_add_package.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/sdk-manual/swift_add_package.png -------------------------------------------------------------------------------- /docs/images/sdk-manual/swift_add_package_url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/sdk-manual/swift_add_package_url.png -------------------------------------------------------------------------------- /docs/images/solution-overview.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/images/solution-overview.webp -------------------------------------------------------------------------------- /docs/overrides/partials/integrations/analytics/custom.html: -------------------------------------------------------------------------------- 1 | 2 | 11 | -------------------------------------------------------------------------------- /docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .icon_check { 2 | color: green; 3 | } 4 | 5 | .icon_cross { 6 | color: red; 7 | } 8 | 9 | /* Maximum space for text block */ 10 | .md-grid { 11 | max-width: 100%; /* or 100%, if you want to stretch to full-width */ 12 | } -------------------------------------------------------------------------------- /docs/zh/analytics/data-mgmt/index.md: -------------------------------------------------------------------------------- 1 | # 数据管理 2 | 数据管理帮助您管理点击流数据。该模块提供以下功能: 3 | 4 | 1. 元数据管理。该解决方案会自动扫描您点击流数据库中的数据以生成元数据，使您能够查看事件、事件参数和用户属性的名称和描述，并更新它们的显示名称、描述和数据字典。[了解更多](../data-mgmt/metadata-mgmt.md) 5 | 2. 流量源配置。您可以在此更新流量源分类规则。[了解更多](../data-mgmt/traffic-source.md) 6 | 7 | ## 访问数据管理 8 | 要访问数据管理模块,请执行以下步骤: 9 | 10 | 1. 打开**亚马逊云科技点击流分析解决方案的控制台**，在左侧导航栏中单击 **分析工作坊**，浏览器将打开一个新标签页。 11 | 2. 在**分析工作坊**页面中,点击左侧导航面板中的**数据管理**图标。 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/zh/analytics/index.md: -------------------------------------------------------------------------------- 1 | # 分析工作坊 2 | 分析工作坊（Analytics Studio）是供业务分析师或数据分析师查看和创建仪表板、查询和探索点击流数据以及管理元数据的统一网页界面。 3 | 4 | ## 模块 5 | 分析工作坊了包含以下模块。 6 | 7 | - [仪表板](./dashboard/index.md)。查看现成的仪表板和自定义仪表板。 8 | - [探索](./explore/index.md)。用户用于发现超越现成仪表板的洞见的高级分析模型。 9 | - [分析](./analyzes/index.md)。用于创建和修改仪表板，以及管理数据集的作者界面。 10 | - [数据管理](./data-mgmt/index.md)。查看和管理点击流数据的元数据。 11 | 12 | ## 术语表 13 | 以下是分析工作坊中使用的术语和概念，当您使用分析工作坊时，熟悉这些术语将会很有帮助。 14 | 15 | - **事件**。事件指的是用户在应用程序或网站内执行的操作生成的点击流数据（例如点击、访问页面等）。 16 | - **预设事件**。由点击流 SDK 自动收集的事件，通常以下划线 '_' 开头的名称。 17 | - **自定义事件**。由应用程序所有者定义和收集的事件，事件的收集时机和业务目的因应用而异。 18 | - **事件参数**。'事件参数' 用于描述事件发生时的各种信息维度。事件参数分为两类：公共参数，即所有事件都包含的参数，例如用户信息（user_id）、设备信息（应用程序版本、设备型号等）；私有参数，即某些事件特有的参数，例如用户定制的参数。 19 | - **公共参数**。所有事件都包含的参数，例如用户信息（user_id）、设备信息（应用程序版本、设备型号等）。 20 | - **私有参数**。某些事件特有的参数，例如用户定制的参数。 21 | - **用户属性**。用户属性用于记录用户的属性。用户属性分为两类：预设属性，即由 SDK 预设收集的属性，例如 '_first_visit_date'；自定义属性，即用户自己报告的用户属性，例如 'email_address'。 22 | -------------------------------------------------------------------------------- /docs/zh/deployment/common-parameters.md: -------------------------------------------------------------------------------- 1 | | Admin User Email (`Email`) | `<需要输入>` | 指定管理员的电子邮件地址。此电子邮件地址将收到一个临时密码，用于访问 {{ solution_name }} Web 控制台。您可以在启动解决方案后直接在配置的 Cognito 用户池中创建更多用户。 | 2 | | IAM Role Prefix (`IamRolePrefix`) | `<可选输入>` | 指定在解决方案中创建的 IAM 角色名称的前缀。 | 3 | | IAM Role Boundary ARN (`IamRoleBoundaryArn`) | `<可选输入>` | 指定在解决方案中创建的[IAM 角色的权限边界][iam-boundary]。 | 4 | 5 | [iam-boundary]: https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html -------------------------------------------------------------------------------- /docs/zh/deployment/domain-parameters.md: -------------------------------------------------------------------------------- 1 | | 参数 | 默认值 | 描述 | 2 | | ---------- | ---------------- | ------------------------------------------------------------ | 3 | | Host Zone ID (`HostedZoneId`) | `<需要输入>` | 选择 Amazon Route 53 的公共托管区域 ID。 | 4 | | Host Zone Name (`HostedZoneName`) | `<需要输入>` | 公共托管区域的域名，例如 `example.com`。 | 5 | | Record Name (`RecordName`) | `<需要输入>` | 控制台的域名的子名称（在 R53 中称为记录名称）。例如，如果您要为控制台使用自定义域 `clickstream.example.com`，请输入 `clickstream`。 | 6 | -------------------------------------------------------------------------------- /docs/zh/deployment/oidc-parameters.md: -------------------------------------------------------------------------------- 1 | | OpenID Connector Client Id (`OIDCClientId`) | `<输入>` | OpenID Connect 客户端 ID。 | 2 | | OpenID Connector Issuer (`OIDCProvider`) | `<输入>` | OpenID Connect 提供程序发行方。发行方必须以 `https://` 开头 | 3 | {% 4 | include-markdown "./common-parameters.md" 5 | comments=false 6 | %} -------------------------------------------------------------------------------- /docs/zh/deployment/tls-note.md: -------------------------------------------------------------------------------- 1 | 默认情况下，此部署在CloudFront中使用TLSv1.0和TLSv1.1。但是，我们建议您手动配置CloudFront以使用更安全的TLSv1.2/TLSv1.3，并申请证书和自定义域来启用此功能。我们强烈建议您根据以下建议更新您的TLS配置和密码套件选择： 2 | 3 | - **传输层安全协议**：升级到TLSv1.2或更高版本 4 | - **密钥交换**：ECDHE 5 | - **块密码操作模式**：GCM 6 | - **身份验证**：ECDSA 7 | - **加密密码**：AES256 8 | - **消息鉴别**：SHA（256/384/除SHA1以外的任何哈希函数） 9 | 10 | 例如，TLSv1.2_2021 可以满足上述建议。 -------------------------------------------------------------------------------- /docs/zh/getting-started/1.create-project.md: -------------------------------------------------------------------------------- 1 | # 第一步 - 创建项目 2 | 3 | 要开始使用{{ solution_name }}解决方案，首先需要在解决方案控制台中创建一个项目。项目类似于一个容器，用于集中管理收集和分析点击流数据的所有相关AWS资源。 4 | 5 | ## 先决条件 6 | 确保您已部署{{ solution_name }}解决方案。如果尚未部署，请参阅[部署指南](../deployment/index.md)。 7 | 8 | ## 步骤 9 | 按照以下步骤创建一个项目。 10 | 11 | 1. 登录到**AWS上的点击流分析控制台**。 12 | 2. 在**主页**上，选择**创建项目**。 13 | 3. 在弹出的窗口中，输入一个项目名称，例如`quickstart`。 14 | 4. （可选）自定义由解决方案自动创建的项目ID。要这样做，点击`编辑`图标，根据您的需求更新项目ID。 15 | 5. 为您的项目提供一个描述，例如`这是一个演示项目`。 16 | 6. 选择**下一步**。 17 | 7. 提供一个电子邮件地址，用于接收与该项目相关的通知，例如`email@example.com`，然后选择**下一步**。 18 | 8. 为该项目指定一个环境类型。在本例中，选择`开发`。 19 | 9. 选择**创建**。等待项目创建完成，您将被重定向到**项目**页面。 20 | 21 | 我们已完成创建项目的所有步骤。 22 | 23 | ## 下一步 24 | 25 | * [配置数据管道](./2.config-pipeline.md) -------------------------------------------------------------------------------- /docs/zh/getting-started/4.view-dashboard.md: -------------------------------------------------------------------------------- 1 | # 第四步 - 查看仪表板 2 | 在您的应用程序发送数据（或者发送示例数据）到流水线之后，您可以在分析工作坊中查看由解决方案创建的开箱即用的仪表板和查询数据。 3 | 4 | ## 步骤 5 | 6 | 1. 登录到 **AWS控制台上的Clickstream Analytics**。 7 | 2. 在左侧导航窗格中，选择 **分析工作坊**，一个新网页界面应该在浏览器中打开。 8 | 3. 在 Analytics Studio 页面中，在网页顶部的下拉列表中选择您刚才创建的项目和应用程序 9 | 4. 默认情况下，你应该进入仪表板页面，如果没有，请点击左侧导航面板中的第一个图标（仪表板）。 10 | ![qs-dashboard](../images/get-started/explore-dashboard-zh.png) 11 | 5. 单击 “用户生命周期仪表板-默认”，您将能够看到解决方案创建的仪表板。 12 | 6. 点击左侧导航面板中的第二个图标（探索），您将能够使用探索性分析模型查询点击流数据。 13 | 14 | 恭喜！您已完成入门教程。您可以试用使用分析工作坊的功能查询分析数据，或者稍后继续了解更多关于这个解决方案的内容。 15 | 16 | ## 下一步 17 | 18 | * [管道管理](../pipeline-mgmt/index.md) -------------------------------------------------------------------------------- /docs/zh/getting-started/index.md: -------------------------------------------------------------------------------- 1 | # 入门指南 2 | 3 | 在[部署解决方案](../deployment/index.md)后，参考本节内容快速了解如何利用{{ solution_name }}解决方案收集和分析应用程序的事件数据。本章将向您展示如何创建一个无服务器数据管道来收集来自应用程序的数据，并使用分析工作坊查看内置的分析仪表板和探索分析点击流数据。 4 | 5 | ## 步骤 6 | 7 | - [步骤 1：创建项目](./1.create-project.md)。创建一个项目。 8 | - [步骤 2：配置数据管道](./2.config-pipeline.md)。使用无服务器基础架构配置数据管道。 9 | - [步骤 3：集成 SDK](./3.integrate-sdk.md)。将SDK集成到您的应用程序中，以自动收集数据并将数据发送到管道。 10 | - [步骤 4：访问内置仪表板](./4.view-dashboard.md)。查看基于自动收集的应用程序数据的内置仪表板。 -------------------------------------------------------------------------------- /docs/zh/images/OIDC/adfs-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/adfs-9.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/authing-authorization-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/authing-authorization-configuration.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/authing-endpoint-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/authing-endpoint-info.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/cognito-hostUI-new.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/cognito-hostUI-new.jpeg -------------------------------------------------------------------------------- /docs/zh/images/OIDC/cognito-new-console-clientID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/cognito-new-console-clientID.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/cognito-new-console-userpoolID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/cognito-new-console-userpoolID.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/keycloak-OIDC-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/keycloak-OIDC-config.png -------------------------------------------------------------------------------- /docs/zh/images/OIDC/keycloak-example-realm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/OIDC/keycloak-example-realm.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/acquisition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/acquisition.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/activity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/activity.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/details.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/device.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/device.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/engagement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/engagement.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/path.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/dashboard/retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/dashboard/retention.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/access-studio-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/access-studio-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/access-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/access-studio.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/acquisition-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/acquisition-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/activity-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/activity-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-add-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-add-dataset.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-adjust-pivot-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-adjust-pivot-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-adjust-pivot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-adjust-pivot.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-calculated-field-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-calculated-field-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-calculated-field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-calculated-field.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyze-new-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyze-new-dataset.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyzes-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyzes-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/analyzes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/analyzes.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/attribution-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/attribution-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/crash-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/crash-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/custom-dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/custom-dashboard-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/custom-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/custom-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/dashboard-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/dashboard-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/engagement-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/engagement-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-event-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-event-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-event.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-event.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-funnel-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-funnel-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-funnel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-funnel.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-path-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-path-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-path.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-retention-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-retention-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore-retention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore-retention.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/explore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/explore.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/metadata-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/metadata-en.png -------------------------------------------------------------------------------- /docs/zh/images/analytics/explore/metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/analytics/explore/metadata.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/01-architecture-end-to-end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/01-architecture-end-to-end.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/02-architecture-ingestion-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/02-architecture-ingestion-module.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/03-architecture-data-processing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/03-architecture-data-processing.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/04-architecture-data-modeling-redshift.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/04-architecture-data-modeling-redshift.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/05-architecture-data-modeling-scan-metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/05-architecture-data-modeling-scan-metadata.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/06-architecture-data-modeling-athena.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/06-architecture-data-modeling-athena.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/07-architecture-reporting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/07-architecture-reporting.png -------------------------------------------------------------------------------- /docs/zh/images/architecture/08-analytics-studio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/architecture/08-analytics-studio.png -------------------------------------------------------------------------------- /docs/zh/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/favicon.ico -------------------------------------------------------------------------------- /docs/zh/images/get-started/explore-dashboard-zh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/get-started/explore-dashboard-zh.png -------------------------------------------------------------------------------- /docs/zh/images/get-started/explore-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/get-started/explore-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/get-started/qs-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/get-started/qs-dashboard.png -------------------------------------------------------------------------------- /docs/zh/images/launch-stack.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/launch-stack.webp -------------------------------------------------------------------------------- /docs/zh/images/pipe-mgmt/concepts-v11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/pipe-mgmt/concepts-v11.png -------------------------------------------------------------------------------- /docs/zh/images/pipe-mgmt/concepts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/pipe-mgmt/concepts.png -------------------------------------------------------------------------------- /docs/zh/images/pipe-mgmt/table_relationship.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/pipe-mgmt/table_relationship.png -------------------------------------------------------------------------------- /docs/zh/images/sdk-manual/android_raw_folder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/sdk-manual/android_raw_folder.png -------------------------------------------------------------------------------- /docs/zh/images/sdk-manual/replace-in-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/sdk-manual/replace-in-file.png -------------------------------------------------------------------------------- /docs/zh/images/sdk-manual/swift_add_amplify_config_json_file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/sdk-manual/swift_add_amplify_config_json_file.png -------------------------------------------------------------------------------- /docs/zh/images/sdk-manual/swift_add_package.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/sdk-manual/swift_add_package.png -------------------------------------------------------------------------------- /docs/zh/images/sdk-manual/swift_add_package_url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/sdk-manual/swift_add_package_url.png -------------------------------------------------------------------------------- /docs/zh/images/solution-overview.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/docs/zh/images/solution-overview.webp -------------------------------------------------------------------------------- /docs/zh/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | -------------------------------------------------------------------------------- /docs/zh/notices.md: -------------------------------------------------------------------------------- 1 | 顾客有责任对本文档中的信息进行独立评估。本文档：(a) 仅用于信息目的，(b) 代表亚马逊网络服务的当前产品和实践，这些内容可能会在不事先通知的情况下发生变化，以及 (c) 不会对亚马逊网络服务及其关联公司、供应商或许可方产生任何承诺或保证。亚马逊网络服务的产品或服务按原样提供，不附带任何明示或暗示的保证、陈述或条件。亚马逊网络服务对其客户的责任和责任受亚马逊网络服务的协议控制，本文档不是亚马逊网络服务与其客户之间的任何协议的一部分，也不修改任何协议。 2 | 3 | {{solution_name}}根据 Apache 许可证第 2.0 版的条款获得许可，该许可证可在 [Apache Software Foundation][foundation] 上获得。 4 | 5 | [foundation]: https://www.apache.org/licenses/LICENSE-2.0 6 | -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/app-registration.md: -------------------------------------------------------------------------------- 1 | # 应用注册 2 | 3 | 一旦数据管道转换为 `Active` 状态，就需要将您的应用程序注册到管道中，以便接收点击流数据。在添加应用程序之后，您可以将 SDK 集成到您的应用程序中，以将数据发送到管道中。以下是注册应用程序到数据管道的步骤。 4 | 5 | ## 步骤 6 | 7 | 1. 登录到 **AWS 控制台上的 Clickstream Analytics**。 8 | 2. 在左侧导航窗格中，选择 **项目**，然后选择要注册应用程序的项目，单击其标题，将您带到项目页面。 9 | 3. 选择 **+ 添加应用程序** 来开始将应用程序添加到管道中。 10 | 4. 通过填写以下字段来完成表单： 11 | * 应用程序名称：为您的应用程序提供一个名称。 12 | * 应用程序 ID：系统将基于名称生成一个 ID，如果需要，您可以进行自定义。 13 | * 应用程序报告时区：指定应用程序的报告时区；预设的仪表板使用此时区来计算每日指标。 14 | * 描述：为您的应用程序提供描述。 15 | * Android 包名称：如果您的应用程序具有 Android 客户端，请提供包名称。 16 | * 应用程序 Bundle ID：如果您的应用程序具有 iOS 客户端，请提供 Bundle ID。 17 | 18 | 5. 选择 **注册应用程序并生成 SDK 指南**，并等待完成注册。 19 | 20 | 6. 您现在应该看到针对不同平台的 SDK 集成选项卡。单击相关选项卡以查看如何将 SDK 添加到您的应用程序的详细说明。按照提供的步骤集成 SDK。对于某些平台，您可以单击 **下载配置 JSON 文件** 按钮以获取配置文件。 21 | 22 | 新添加的应用程序更新管道可能需要大约 3 到 5 分钟。一旦管道状态返回为 `Active`，即可准备好从您的应用程序接收数据。 23 | -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/data-processing/index.md: -------------------------------------------------------------------------------- 1 | # 数据处理 2 | {{solution_name}} 提供了内置的数据模式，用于解析和建模从您的 Web 和移动应用程序发送的原始事件数据，使您能够在分析引擎（如 RedShift 和 Athena）中轻松分析数据。 3 | 4 | 数据处理模块包括两个功能： 5 | 6 | - **转换**: 从摄取模块沉降的文件中提取数据，然后解析每个事件数据并将其转换为解决方案数据模型。 7 | - **丰富**: 将附加维度/字段添加到事件数据中。 8 | 9 | 本章包括： 10 | 11 | - [数据模式](./data-schema.md) 12 | - [配置执行参数](./configure-execution-para.md) 13 | - [配置 ETL 自定义插件](./configure-plugin.md) -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/ingestion/create-data-sink-w-kinesis.md: -------------------------------------------------------------------------------- 1 | # Amazon Kinesis Data Stream 2 | 该数据宿将从摄取端点收集的点击流数据流式传输到 KDS。该解决方案将根据您的规格在您的 AWS 账户中创建一个 KDS。 3 | 4 | ## 配置模式 5 | 提供两种模式：**按需**和**预配置** 6 | 7 | * **按需**：在此模式下，KDS 分片是根据研讨会自动配置的。按需模式适用于具有不可预测和高度可变流量模式的工作负载。 8 | 9 | * **预配置**：在此模式下，KDS 分片在创建时设置。预配置模式适用于具有容量需求易于预测的可预测流量。如果您想对数据在分片之间的分布方式进行精细控制，也可以使用预配置模式。 10 | * 分片数：在预配置模式下，您必须指定数据流的分片数量。有关分片的更多信息，请参阅[预配置模式](https://docs.aws.amazon.com/streams/latest/dev/how-do-i-size-a-stream.html#provisionedmode)。 11 | 12 | ## 其他设置 13 | * **接收器最大间隔：** 使用此配置，您可以指定记录在流式传输到 AWS 服务之前应缓冲的最长时间（以秒为单位）。 14 | * **批处理大小：** 使用此配置，您可以指定单个批次中传递的记录数的最大值。 15 | -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/ingestion/create-data-sink-w-s3.md: -------------------------------------------------------------------------------- 1 | # Amazon S3 2 | 在此选项中，点击流数据在摄取服务器的内存中进行缓冲，然后汇入到 S3 存储桶中。如果不需要实时数据消耗，则此选项提供了最佳的性价比。 3 | 4 | !!! info "注意" 5 | 与 Kafka 和 KDS 数据宿不同，此选项在摄取服务器中缓冲数据，并在数据汇入 S3 前向 SDK 客户端响应 200 状态码，因此在摄取服务器失败且自动扩展机器正在创建过程中，数据可能会丢失。但值得注意的是，由于解决方案具有高可用性设计，这种可能性非常低。 6 | 7 | * **缓存大小：** 指定发送到 Amazon S3 之前要缓冲的数据大小。更高的缓冲大小可能具有更低的成本和更高的延迟，而较低的缓冲大小将在更短的交付时间内提供更快的交付速度但成本更高。最小值：1 MiB，最大值：50 MiB。 8 | * **缓存间隔：** 指定将缓存保存到 S3 的最大时间间隔（秒）。更高的间隔允许更多时间来收集数据，数据大小可能更大。较低的间隔更频繁地发送数据，并且在查看较短周期的数据活动时可能更有优势。最小值：60 秒，最大值：3600 秒。 9 | -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/ingestion/index.md: -------------------------------------------------------------------------------- 1 | # 数据摄取 2 | 3 | 数据摄取模块包含一个 Web 服务，它提供通过 HTTP/HTTPS 请求收集数据的端点，主要由 Amazon 应用程序负载均衡器和 Amazon 弹性容器服务组成。它还支持将数据直接汇入到流服务或 S3。 4 | 5 | 您可以使用以下设置创建一个数据摄取模块： 6 | 7 | * [摄取端点设置](./configure-ingestion-endpoint.md)：创建一个 Web 服务作为摄取端点，以收集从 SDK 发送的数据。 8 | 9 | * 数据宿设置：配置解决方案如何为下游消费者接收数据。目前，解决方案支持三种类型的数据宿： 10 | - [Apache Kafka](./create-data-sink-w-kafka.md) 11 | - [Amazon S3](./create-data-sink-w-s3.md) 12 | - [Amazon Kinesis Data Stream (KDS)](./create-data-sink-w-kinesis.md) 13 | 14 | ## 限流 15 | 目前，本方案没有内置的限流功能。如果您需要限流功能，您可以通过配置AWS WAF来实现限流的功能。请参考[WAF][waf]文档。 16 | 17 | [waf]: https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-rate-based.html -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/quicksight/configure-quicksight.md: -------------------------------------------------------------------------------- 1 | # 配置报表 2 | 3 | ## QuickSight 4 | 一旦数据由数据管道进行处理和建模，解决方案支持在 QuickSight 中创建开箱即用的仪表板。 5 | 6 | !!! info "注意" 7 | 要启用此模块，您的 AWS 帐户需要订阅 QuickSight 企业版本。如果没有，请按照此[注册 Amazon QuickSight](https://docs.aws.amazon.com/quicksight/latest/user/signing-up.html)创建订阅。 8 | 9 | 您可以通过以下设置配置报告。 10 | 11 | * **在 QuickSight 中创建示例报表**： 启用此功能将允许解决方案在您的 QuickSight 帐户中创建示例仪表板。 12 | 13 | * **QuickSight 用户**: 为解决方案选择一个管理员用户以创建 QuickSight 资源 (只有亚马逊云科技中国区需要此配置）。 14 | 15 | -------------------------------------------------------------------------------- /docs/zh/pipeline-mgmt/vpc-prerequisites.md: -------------------------------------------------------------------------------- 1 | - 在 VPC 中至少有两个公共子网跨越两个可用区。 2 | - 在 VPC 中至少有两个私有子网（带 NAT 网关或实例）跨越两个可用区，或者在 VPC 中至少有两个隔离子网跨越两个可用区。如果您想将解决方案资源部署在隔离子网中，则必须为以下 AWS 服务创建 [VPC 终端][vpc-endpoints]， 3 | - `s3`、`logs`、`ecr.api`、`ecr.dkr`、`ecs`、`ecs-agent`、`ecs-telemetry`。 4 | - 如果您在摄取模块中使用 KDS 作为缓冲器，则为 `kinesis-streams`。 5 | - 如果您启用数据处理模块，则为 `emr-serverless`、`glue`。 6 | - 如果您在数据建模模块中启用 Redshift 作为分析引擎，则为 `redshift-data`、`sts`、`dynamodb`(必须是[网关类型的 DynamoDB 端点][gateway-endppint])、`states` 和 `lambda`。 7 | 8 | [vpc-endpoints]: https://docs.aws.amazon.com/zh_cn/whitepapers/latest/aws-privatelink/what-are-vpc-endpoints.html 9 | [gateway-endppint]: https://docs.aws.amazon.com/vpc/latest/privatelink/vpc-endpoints-ddb.html 10 | -------------------------------------------------------------------------------- /docs/zh/revisions.md: -------------------------------------------------------------------------------- 1 | | 日期 | 版本发布说明 | 2 | | :----------- | :--------------- | 3 | | 2023年07月 | 首次发布 | 4 | | 2023年09月 | 新增 Web SDK | 5 | | 2023年12月 |
  • 新增分析工作坊
  • 新增 Flutter SDK
  • 新增身份管理
  • 新增升级解决方案
  • 更新成本、入门指南、管道管理
    • | 6 | | 2024年5月 |
  • 优化数据模式,以提高在探索、分析和临时查询中的查询性能
  • 支持自动流量源检测、分类和仪表板
  • 优化预置仪表板
  • 在预置仪表板中支持时区
  • 在事件分析中支持更多指标类型
  • 在路径分析中支持"显示所有节点"
  • 在探索中添加新的分析模型 - 归因分析
  • 支持使用内部负载均衡器作为数据接收端点
  • 提高将数据加载到Redshift的稳定性
  • 添加React Native SDK
    • | 7 | | 2024年9月 |
  • 使用 EC2 启动模板替代已弃用的启动配置。有关升级后操作的详细信息，请参阅升级文档。
  • 一些稳定性改进
    • | -------------------------------------------------------------------------------- /docs/zh/sdk-manual/index.md: -------------------------------------------------------------------------------- 1 | {{ solution_name }} 提供不同的客户端SDK，可帮助您更轻松地向解决方案中创建的数据管道报告事件。目前，该解决方案支持以下平台： 2 | 3 | - [Android](./android.md) 4 | - [Swift](./swift.md) 5 | - [Web](./web.md) 6 | - [Flutter](./flutter.md) 7 | - [React Native](./react-native.md) 8 | - [WeChat Miniprogram](./wechat.md) 9 | 10 | 另外，我们还提供了 [HTTP API](./http-api.md)，让您可以通过http请求来收集其他平台（例如：服务端）的数据。 11 | 12 | ## 主要功能和优势 13 | 14 | - **自动数据收集**。Clickstream SDK提供内置功能，可自动收集常见事件，例如屏幕视图、会话和用户参与度，因此您只需要专注于记录业务特定事件。 15 | - **易于使用**。Clickstream SDK提供多个API和配置选项，以简化事件报告和属性设置过程。 16 | - **跨平台分析**。Clickstream SDK在事件数据结构、属性验证规则和事件发送机制上保持一致，因此数据可以以相同的结构规范化，以进行跨平台分析。 17 | 18 | !!! info "注意" 19 | 20 | 所有Clickstream SDK均在[GitHub][github]上以Apache 2.0许可证开源。如果需要，您可以自定义SDK。 21 | 22 | [github]: https://github.com/awslabs/?q=clickstream&type=all&language=&sort= 23 | -------------------------------------------------------------------------------- /docs/zh/solution-overview/features-and-benefits.md: -------------------------------------------------------------------------------- 1 | 该解决方案包括以下主要功能： 2 | 3 | - **可视化数据管道生成器**。 您只需通过基于 Web 的用户界面控制台定义数据管道即可。该解决方案将负责底层基础设施的创建、所需的安全设置和数据集成。每个流水线模块都具有各种功能，并且设计为松散耦合，因此您可以灵活地针对特定用例进行自定义。 4 | - **专门构建的SDK**。 这些 SDK 针对从 Android、iOS 和 JavaScript 平台收集数据进行了优化，这些平台会自动收集常见事件（例如，首次访问、屏幕查看），支持内置本地缓存、重试和验证机制，以确保数据传输的完整性。 5 | - **开箱即用的仪表板**。 该解决方案提供了十几个内置的可视化图表（例如用户获取、用户参与度、留存率和用户统计信息）和探索性报告模板（例如转化分析、路径分析、事件分析，和留存分析），为用户行为分析、营销分析和产品分析等各种关键业务分析用例提供支持。 6 | 7 | -------------------------------------------------------------------------------- /docs/zh/solution-overview/use-cases.md: -------------------------------------------------------------------------------- 1 | 点击流数据在许多在线业务分析场景中起着举足轻重的作用，{{solution_name}} 可以应用于以下场景： 2 | 3 | - **用户行为分析**: 用户行为分析中的点击流数据可以深入了解网站或应用程序上用户互动的顺序和时间序列模式，帮助企业了解用户导航、偏好和参与度，从而增强整体产品体验并推动产品创新。 4 | - **客户数据平台**: 点击流数据以及其他业务数据源（例如订单历史记录和用户个人资料）使客户能够创建自有客户数据平台，该平台提供用户的完整视图，使企业能够个性化客户体验、优化客户交互并提供有针对性的营销信息。 5 | - **营销分析**: 营销分析中的点击流数据提供有关用户流量来源，点击路径和与营销活动互动的详细信息，使企业能够衡量营销活动的有效性，优化营销策略并提高转化率。 6 | -------------------------------------------------------------------------------- /docs/zh/source.md: -------------------------------------------------------------------------------- 1 | 访问我们的[GitHub代码库][source]以下载此解决方案的源代码。{{ solution_name }}模板是使用[AWS Cloud Development Kit (CDK)][cdk]生成的。请参阅[README.md][readme]文件获取其他信息。 2 | 3 | [source]: {{ source_repo }} 4 | [cdk]: http://aws.amazon.com/cdk/ 5 | [readme]: {{ source_repo }}/blob/main/README.md 6 | -------------------------------------------------------------------------------- /docs/zh/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .icon_check { 2 | color: green; 3 | } 4 | 5 | .icon_cross { 6 | color: red; 7 | } 8 | 9 | /* Maximum space for text block */ 10 | .md-grid { 11 | max-width: 100%; /* or 100%, if you want to stretch to full-width */ 12 | } -------------------------------------------------------------------------------- /docs/zh/uninstall.md: -------------------------------------------------------------------------------- 1 | # 卸载Clickstream Analytics on AWS 2 | 3 | !!! warning "警告" 4 | 如果你在删除{{ solution_name }}主堆栈之前，删除项目相关的堆栈，你会遇到IAM角色丢失的错误。{{ solution_name }}控制台为Clickstream数据管道启动了额外的CloudFormation堆栈。 5 | 我们建议你在卸载解决方案之前删除项目。 6 | 7 | ## 步骤1. 删除项目 8 | 9 | 1. 转到{{solution_name}}控制台。 10 | 2. 在左边的侧边栏，选择**项目**。 11 | 3. 选择要删除的项目。 12 | 4. 选择右上角的**删除**按钮。 13 | 5. 重复步骤3和4，删除所有项目。 14 | 15 | ## 步骤2. 删除{{ solution_name }}主堆栈 16 | 17 | 1. 转到[CloudFormation控制台][cloudformation]。 18 | 2. 找到解决方案的CloudFormation堆栈。 19 | 3. 删除该解决方案的CloudFormation堆栈。 20 | 4. （可选）删除该解决方案创建的S3桶。 21 | 1. 选择解决方案的CloudFormation堆栈，并选择**资源**标签。 22 | 2. 在搜索栏中，输入 "DataBucket"。它显示所有由解决方案创建的名称为 "DataBucket "的资源。你可以找到资源类型**AWS::S3::Bucket**，**Physical ID**字段是S3桶的名称。 23 | 3. 进入S3控制台，找到S3桶的名称。**空**并**删除**该S3桶。 24 | 25 | [cloudformation]: https://console.aws.amazon.com/cloudfromation/ 26 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | ## Clickstream samples 2 | 3 | ### Custom plug-in samples 4 | 5 | - [Transformer plug-in](./custom-plugins/custom-sdk-transformer/README.md) 6 | - [Enrichment plug-in](./custom-plugins/custom-enrich/README.md) 7 | 8 | ### Simulator client 9 | It's a Python application to simulate a cross-platform (Web, Android, and iOS) **Shopping** application to send user behavior events. See [this](./standalone-data-generator/README.md) for the detailed usage. 10 | 11 | ### SDK samples 12 | See [this repo](https://github.com/aws-samples/clickstream-sdk-samples) for different kinds of SDK samples, including iOS, Android, Web, and React Native. -------------------------------------------------------------------------------- /examples/custom-plugins/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | curr_dir=$(pwd) 5 | 6 | echo $curr_dir 7 | 8 | echo "Build custom-enrich ...." 9 | 10 | cd $curr_dir/custom-enrich 11 | 12 | ./gradlew clean build 13 | 14 | ls -l ./build/libs/*.jar 15 | 16 | echo "" 17 | echo "Build custom-sdk-transformer ...." 18 | 19 | cd $curr_dir/custom-sdk-transformer 20 | 21 | ./gradlew clean build 22 | 23 | ls -l ./build/libs/*.jar 24 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | bin 3 | build 4 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project 2 | group=com.example 3 | projectVersion=1.1.0 4 | 5 | # Dependencies 6 | lombokVersion=1.18.30 7 | junitVersion=5.10.1 8 | junitBomVersion=5.10.1 9 | assertJVersion=3.24.2 10 | mockitoVersion=5.8.0 11 | jacocoVersion=0.8.11 12 | jacocoLogVersion=3.1.0 13 | checkstyleVersion=9.3 14 | guavaVersion=32.1.2-jre 15 | sparkVersion=3.4.1 16 | uapJavaVersion=1.5.3 17 | maxmindDbVersion=3.1.0 18 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/gradle/config/scripts/idea.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'idea' 2 | 3 | rootProject.idea { 4 | workspace.iws.withXml { provider -> 5 | def junitDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 6 | it.@type == 'JUnit' 7 | } 8 | junitDefaults.option.find { it.@name == 'WORKING_DIRECTORY' }.@value = '$MODULE_DIR$' 9 | 10 | def applicationDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 11 | it.@default == 'true' && it.@type == 'Application' 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/gradle/config/scripts/style.gradle: -------------------------------------------------------------------------------- 1 | class StyleExtension { 2 | List excludePackages; 3 | List excludeClasses; 4 | } 5 | 6 | project.extensions.create('style', StyleExtension) 7 | 8 | apply plugin: 'checkstyle' 9 | 10 | checkstyle { 11 | configDirectory = file("$rootDir/gradle/config/checkstyle") 12 | configFile = file("$rootDir/gradle/config/checkstyle/sun_checks.xml") 13 | toolVersion = checkstyleVersion 14 | } 15 | 16 | checkstyleTest.enabled = false 17 | 18 | checkstyleMain.doFirst { 19 | style.excludePackages.each() { 20 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '/*'}" 21 | } 22 | 23 | style.excludeClasses.each() { 24 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '.java'}" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/examples/custom-plugins/custom-enrich/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.setter.flagUsage=error 2 | lombok.data.flagUsage=error -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'custom-enrich' 2 | //include('app') 3 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-enrich/src/test/resources/test.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/examples/custom-plugins/custom-enrich/src/test/resources/test.snappy.parquet -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | bin 3 | build 4 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project 2 | group=com.example 3 | projectVersion=1.1.0 4 | 5 | # Dependencies 6 | lombokVersion=1.18.30 7 | junitVersion=5.10.1 8 | junitBomVersion=5.10.1 9 | assertJVersion=3.24.2 10 | mockitoVersion=5.8.0 11 | jacocoVersion=0.8.11 12 | jacocoLogVersion=3.1.0 13 | checkstyleVersion=9.3 14 | guavaVersion=32.1.2-jre 15 | sparkVersion=3.4.1 16 | uapJavaVersion=1.5.3 17 | maxmindDbVersion=3.1.0 18 | 19 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/gradle/config/scripts/idea.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'idea' 2 | 3 | rootProject.idea { 4 | workspace.iws.withXml { provider -> 5 | def junitDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 6 | it.@type == 'JUnit' 7 | } 8 | junitDefaults.option.find { it.@name == 'WORKING_DIRECTORY' }.@value = '$MODULE_DIR$' 9 | 10 | def applicationDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 11 | it.@default == 'true' && it.@type == 'Application' 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/gradle/config/scripts/style.gradle: -------------------------------------------------------------------------------- 1 | class StyleExtension { 2 | List excludePackages; 3 | List excludeClasses; 4 | } 5 | 6 | project.extensions.create('style', StyleExtension) 7 | 8 | apply plugin: 'checkstyle' 9 | 10 | checkstyle { 11 | configDirectory = file("$rootDir/gradle/config/checkstyle") 12 | configFile = file("$rootDir/gradle/config/checkstyle/sun_checks.xml") 13 | toolVersion = checkstyleVersion 14 | } 15 | 16 | checkstyleTest.enabled = false 17 | 18 | checkstyleMain.doFirst { 19 | style.excludePackages.each() { 20 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '/*'}" 21 | } 22 | 23 | style.excludeClasses.each() { 24 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '.java'}" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/examples/custom-plugins/custom-sdk-transformer/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.data.flagUsage=error -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'custom-sdk-transformer' 2 | //include('app') 3 | -------------------------------------------------------------------------------- /examples/custom-plugins/custom-sdk-transformer/src/main/java/com/example/clickstream/transformer/v1/Constants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package com.example.clickstream.transformer.v1; 15 | 16 | public class Constants { 17 | public static final String DEBUG_LOCAL_PATH = "/tmp/"; 18 | } 19 | -------------------------------------------------------------------------------- /examples/standalone-data-generator/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | 6 | # env files 7 | .env 8 | .venv 9 | # PyCharm 10 | .idea/ 11 | # PyBuilder 12 | target/ 13 | # Unit test / coverage reports 14 | htmlcov/ 15 | .tox/ 16 | .coverage 17 | .cache 18 | nosetests.xml 19 | coverage.xml 20 | 21 | 22 | data/ 23 | venv/ 24 | *.mmdb 25 | 26 | amplifyconfiguration.json 27 | -------------------------------------------------------------------------------- /examples/standalone-data-generator/weighted/weighted.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | with the License. A copy of the License is located at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | and limitations under the License. 12 | """ 13 | import random 14 | 15 | 16 | class WeightedArray: 17 | def __init__(self, items_with_weights): 18 | self.items, self.weights = zip(*items_with_weights) 19 | 20 | def get_random_item(self): 21 | return random.choices(self.items, weights=self.weights)[0] 22 | -------------------------------------------------------------------------------- /frontend/.eslintignore: -------------------------------------------------------------------------------- 1 | *-ln.ts 2 | react-app-env.d.ts 3 | setupProxy.js -------------------------------------------------------------------------------- /frontend/.gitattributes: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | * text=auto eol=lf 4 | /.gitattributes linguist-generated 5 | /.gitignore linguist-generated 6 | /.npmignore linguist-generated 7 | /.npmrc linguist-generated 8 | /.projen/** linguist-generated 9 | /.projen/deps.json linguist-generated 10 | /.projen/files.json linguist-generated 11 | /.projen/tasks.json linguist-generated 12 | /LICENSE linguist-generated 13 | /package.json linguist-generated 14 | /pnpm-lock.yaml linguist-generated 15 | /tsconfig.dev.json linguist-generated 16 | /tsconfig.json linguist-generated -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | !/.gitattributes 3 | !/.projen/tasks.json 4 | !/.projen/deps.json 5 | !/.projen/files.json 6 | !/package.json 7 | !/LICENSE 8 | !/.npmignore 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | lib-cov 21 | coverage 22 | *.lcov 23 | .nyc_output 24 | build/Release 25 | node_modules/ 26 | jspm_packages/ 27 | *.tsbuildinfo 28 | .eslintcache 29 | *.tgz 30 | .yarn-integrity 31 | .cache 32 | aws-exports.json 33 | src/setupProxy.js 34 | build/ 35 | /test-reports/ 36 | junit.xml 37 | /coverage/ 38 | !/.npmrc 39 | !/test/ 40 | !/tsconfig.json 41 | !/tsconfig.dev.json 42 | !/src/ 43 | /lib 44 | /dist/ 45 | -------------------------------------------------------------------------------- /frontend/.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | /test/ 7 | /tsconfig.dev.json 8 | /src/ 9 | !/lib/ 10 | !/lib/**/*.js 11 | !/lib/**/*.d.ts 12 | dist 13 | /tsconfig.json 14 | /.github/ 15 | /.vscode/ 16 | /.idea/ 17 | /.projenrc.js 18 | tsconfig.tsbuildinfo 19 | /.gitattributes 20 | -------------------------------------------------------------------------------- /frontend/.npmrc: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | resolution-mode=highest 4 | -------------------------------------------------------------------------------- /frontend/.prettierignore: -------------------------------------------------------------------------------- 1 | *-ln.ts 2 | react-app-env.d.ts 3 | setupProxy.js -------------------------------------------------------------------------------- /frontend/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 2, 3 | "singleQuote": true 4 | } -------------------------------------------------------------------------------- /frontend/.projen/files.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | ".gitattributes", 4 | ".gitignore", 5 | ".npmignore", 6 | ".npmrc", 7 | ".projen/deps.json", 8 | ".projen/files.json", 9 | ".projen/tasks.json", 10 | "LICENSE", 11 | "tsconfig.dev.json", 12 | "tsconfig.json" 13 | ], 14 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"pnpm dlx projen\"." 15 | } 16 | -------------------------------------------------------------------------------- /frontend/config/webpack/persistentCache/createEnvironmentHash.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | 'use strict'; 15 | const { createHash } = require('crypto'); 16 | 17 | module.exports = env => { 18 | const hash = createHash('md5'); 19 | hash.update(JSON.stringify(env)); 20 | 21 | return hash.digest('hex'); 22 | }; 23 | -------------------------------------------------------------------------------- /frontend/public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/public/apple-touch-icon.png -------------------------------------------------------------------------------- /frontend/public/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/public/favicon-16x16.png -------------------------------------------------------------------------------- /frontend/public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/public/favicon-32x32.png -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: / 4 | 5 | -------------------------------------------------------------------------------- /frontend/src/apis/system.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import { apiRequest } from 'ts/request'; 15 | 16 | export const getSystemInfo = async () => { 17 | const result: any = await apiRequest('get', `/system/info`); 18 | return result; 19 | }; 20 | -------------------------------------------------------------------------------- /frontend/src/assets/images/android.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/src/assets/images/android.webp -------------------------------------------------------------------------------- /frontend/src/assets/images/howitworks.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/src/assets/images/howitworks.webp -------------------------------------------------------------------------------- /frontend/src/assets/images/ios.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/src/assets/images/ios.webp -------------------------------------------------------------------------------- /frontend/src/assets/images/pipelineArch.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/frontend/src/assets/images/pipelineArch.webp -------------------------------------------------------------------------------- /frontend/src/components/common/alert/style.scss: -------------------------------------------------------------------------------- 1 | .common-alert { 2 | width: fit-content; 3 | position: fixed; 4 | bottom: 30px; 5 | left: 0; 6 | right: 0; 7 | margin: auto; 8 | z-index: 9999; 9 | opacity: 1; 10 | transition: all ease-out 0.8s; 11 | } 12 | 13 | .common-alert-hide { 14 | opacity: 0; 15 | } 16 | -------------------------------------------------------------------------------- /frontend/src/context/SystemInfoContext.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import { createContext } from 'react'; 15 | export const SystemInfoContext = createContext({ 16 | version: '', 17 | remoteVersion: '', 18 | templateUrl: '', 19 | stackId: '', 20 | hasUpdate: false, 21 | }); 22 | -------------------------------------------------------------------------------- /frontend/src/context/UserContext.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import { createContext } from 'react'; 15 | import { INIT_USER_DATA } from 'ts/init'; 16 | export const UserContext = createContext(INIT_USER_DATA); 17 | -------------------------------------------------------------------------------- /frontend/src/pages/analytics/metadata/table/table-config.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | import { XSS_PATTERN } from '@aws/clickstream-base-lib'; 14 | 15 | export const displayNameRegex = new RegExp(XSS_PATTERN); 16 | export const descriptionRegex = new RegExp(XSS_PATTERN); 17 | -------------------------------------------------------------------------------- /frontend/src/types/system.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export {}; 15 | declare global { 16 | interface SystemInfo { 17 | readonly version: string; 18 | readonly remoteVersion: string; 19 | readonly templateUrl: string; 20 | readonly stackId: string; 21 | readonly hasUpdate: boolean; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /frontend/test/app.test.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import App from 'App'; 15 | import { createRoot } from 'react-dom/client'; 16 | 17 | it('renders without crashing', () => { 18 | const div = document.createElement('div'); 19 | const root = createRoot(div); 20 | root.render(); 21 | }); 22 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | packages: 4 | - src/base-lib 5 | - frontend 6 | - src/control-plane/backend/lambda/api 7 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/athena/clickstream-item-query.sql: -------------------------------------------------------------------------------- 1 | -- run following command to load latest partition 2 | -- msck repair table {{database}}.event_v2; 3 | -- msck repair table {{database}}.user_v2; 4 | -- msck repair table {{database}}.item_v2; 5 | -- msck repair table {{database}}.session; 6 | 7 | SELECT 8 | event_timestamp, 9 | event_id, 10 | event_name, 11 | platform, 12 | user_pseudo_id, 13 | user_id, 14 | item_id, 15 | name, 16 | brand, 17 | currency, 18 | price, 19 | quantity, 20 | creative_name, 21 | creative_slot, 22 | location_id, 23 | category, 24 | category2, 25 | category3, 26 | category4, 27 | category5, 28 | custom_parameters_json_str, 29 | custom_parameters 30 | FROM 31 | {{database}}.item_v2 32 | where partition_app = ? 33 | and partition_year >= ? 34 | and partition_month >= ? 35 | and partition_day >= ? 36 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/athena/clickstream-session-query.sql: -------------------------------------------------------------------------------- 1 | -- run following command to load latest partition 2 | -- msck repair table {{database}}.event_v2; 3 | -- msck repair table {{database}}.user_v2; 4 | -- msck repair table {{database}}.item_v2; 5 | -- msck repair table {{database}}.session; 6 | 7 | SELECT 8 | event_timestamp, 9 | user_pseudo_id, 10 | session_id, 11 | user_id, 12 | session_number, 13 | session_start_time_msec, 14 | session_source, 15 | session_medium, 16 | session_campaign, 17 | session_content, 18 | session_term, 19 | session_campaign_id, 20 | session_clid_platform, 21 | session_clid, 22 | session_channel_group, 23 | session_source_category 24 | FROM 25 | {{database}}.session 26 | where partition_app = ? 27 | and partition_year >= ? 28 | and partition_month >= ? 29 | and partition_day >= ? -------------------------------------------------------------------------------- /src/analytics/private/sqls/athena/clickstream-user-query.sql: -------------------------------------------------------------------------------- 1 | -- run following command to load latest partition 2 | -- msck repair table {{database}}.event_v2; 3 | -- msck repair table {{database}}.user_v2; 4 | -- msck repair table {{database}}.item_v2; 5 | -- msck repair table {{database}}.session; 6 | 7 | SELECT 8 | user_pseudo_id, 9 | user_id, 10 | event_timestamp, 11 | first_touch_time_msec, 12 | first_visit_date, 13 | first_referrer, 14 | first_traffic_source, 15 | first_traffic_medium, 16 | first_traffic_campaign, 17 | first_traffic_content, 18 | first_traffic_term, 19 | first_traffic_campaign_id, 20 | first_traffic_clid_platform, 21 | first_traffic_clid, 22 | first_traffic_channel_group, 23 | first_traffic_category, 24 | first_app_install_source, 25 | user_properties, 26 | user_properties_json_str 27 | FROM 28 | {{database}}.user_v2 29 | where partition_app = ? 30 | and partition_year >= ? 31 | and partition_month >= ? 32 | and partition_day >= ? -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/clickstream-log-v2.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{schema}}.{{table_clickstream_log}} ( 2 | id BIGINT IDENTITY(1,1), 3 | log_name varchar(256), 4 | log_level varchar(32), 5 | log_msg varchar(65535), 6 | log_date TIMESTAMP default getdate() 7 | ); 8 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_acquisition_country_new_user.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | geo_country varchar(255), 5 | geo_city varchar(255), 6 | user_count bigint, 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) 11 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_acquisition_day_traffic_source_user.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | aggregation_type varchar(255), 5 | aggregation_dim varchar(65535), 6 | user_id varchar(255), 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_acquisition_day_user_acquisition.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | aggregation_type varchar(255), 4 | aggregation_dim varchar(65535), 5 | platform varchar(255), 6 | user_id varchar(255), 7 | new_user_count bigint, 8 | session_count bigint, 9 | engagement_session_count bigint, 10 | engagement_rate real, 11 | total_user_engagement_time_seconds real, 12 | avg_user_engagement_time_seconds real, 13 | event_count bigint, 14 | update_timestamp timestamp DEFAULT getdate() 15 | ) 16 | BACKUP YES 17 | SORTKEY(event_date) 18 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_acquisition_day_user_view_cnt.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | platform varchar(255), 4 | active_users varchar(255), 5 | new_users varchar(255), 6 | view_count bigint 7 | ) 8 | BACKUP YES 9 | SORTKEY(event_date) 10 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_acquisition_intra_day_user_mv.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW {{database_name}}.{{schema}}.{{viewName}} 2 | BACKUP YES 3 | SORTKEY(event_date, platform) 4 | AUTO REFRESH NO 5 | AS 6 | SELECT 7 | DATE_TRUNC('hour', CONVERT_TIMEZONE('{{{timezone}}}', e.event_timestamp)) as event_date, 8 | e.platform, 9 | COALESCE(u.user_id, e.user_pseudo_id) as active_users, 10 | CASE WHEN event_name = '_first_open' THEN COALESCE(u.user_id, e.user_pseudo_id) ELSE NULL END as new_users 11 | FROM {{database_name}}.{{schema}}.clickstream_event_base_view as e 12 | join {{database_name}}.{{schema}}.user_v2 as u using(user_pseudo_id) 13 | GROUP BY 1,2,3,4 14 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_device_crash_rate.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | platform varchar(255), 4 | app_version varchar(255), 5 | merged_user_id varchar(255), 6 | crashed_user_id varchar(255), 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_device_user_device.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | user_id varchar(255), 4 | platform varchar(255), 5 | device varchar(255), 6 | app_version varchar(65535), 7 | operating_system varchar(65535), 8 | operating_system_version varchar(65535), 9 | device_ua_browser varchar(65535), 10 | device_screen_resolution varchar(65535), 11 | device_ua_device varchar(65535), 12 | device_ua_device_category varchar(65535), 13 | event_count bigint, 14 | update_timestamp timestamp DEFAULT getdate() 15 | ) 16 | BACKUP YES 17 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_day_event_view.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | event_count bigint, 5 | view_count bigint, 6 | update_timestamp timestamp DEFAULT getdate() 7 | ) 8 | BACKUP YES 9 | SORTKEY(event_date) 10 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_entrance.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | aggregation_type varchar(255), 5 | aggregation_dim varchar(65535), 6 | entrance_count bigint, 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) 11 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_event_name.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | platform varchar(255), 4 | user_id varchar(255), 5 | event_name varchar(255), 6 | event_value real, 7 | event_count bigint, 8 | update_timestamp timestamp DEFAULT getdate() 9 | ) 10 | BACKUP YES 11 | SORTKEY(event_date) 12 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_exit.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | aggregation_type varchar(255), 5 | aggregation_dim varchar(65535), 6 | exit_count bigint, 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) 11 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_kpi.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 4 | event_date date, 5 | platform varchar(255), 6 | avg_engaged_session_per_user real, 7 | avg_engagement_time_per_session_seconds real, 8 | avg_engagement_time_per_user_seconds real, 9 | update_timestamp timestamp DEFAULT getdate() 10 | ) 11 | BACKUP YES 12 | SORTKEY(event_date) 13 | ; 14 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_page_screen_view.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | aggregation_type varchar(255), 5 | aggregation_dim varchar(65535), 6 | view_count bigint, 7 | update_timestamp timestamp DEFAULT getdate() 8 | ) 9 | BACKUP YES 10 | SORTKEY(event_date) 11 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_engagement_page_screen_view_detail.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | aggregation_type varchar(255), 5 | aggregation_dim varchar(65535), 6 | user_id varchar(255), 7 | user_engagement_time_seconds real, 8 | event_count bigint, 9 | view_count bigint, 10 | session_count bigint, 11 | update_timestamp timestamp DEFAULT getdate() 12 | ) 13 | BACKUP YES 14 | SORTKEY(event_date) 15 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_lifecycle_view_v2.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW {{database_name}}.{{schema}}.{{viewName}} 2 | BACKUP YES 3 | AUTO REFRESH NO 4 | AS 5 | select 6 | user_pseudo_id, 7 | platform, 8 | DATE_TRUNC('week', CONVERT_TIMEZONE('{{{timezone}}}', event_timestamp)) as time_period_week 9 | from {{database_name}}.{{schema}}.event_v2 10 | where event_name = '_session_start' 11 | group by 1,2,3 12 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_retention_base_view.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW {{database_name}}.{{schema}}.{{viewName}} 2 | BACKUP YES 3 | SORTKEY(first_date) 4 | AUTO REFRESH NO 5 | AS 6 | select 7 | e.user_pseudo_id, 8 | e.platform, 9 | DATE(CONVERT_TIMEZONE('{{{timezone}}}', TIMESTAMP 'epoch' + u.first_touch_time_msec/1000 * INTERVAL '1 second')) as first_date, 10 | DATE_DIFF('day', DATE(CONVERT_TIMEZONE('{{{timezone}}}', TIMESTAMP 'epoch' + u.first_touch_time_msec/1000 * INTERVAL '1 second')), DATE(CONVERT_TIMEZONE('{{{timezone}}}', e.event_timestamp))) as day_diff 11 | from {{database_name}}.{{schema}}.clickstream_event_base_view e 12 | join {{database_name}}.{{schema}}.user_v2 u using(user_pseudo_id) 13 | where e.event_timestamp >= TIMESTAMP 'epoch' + u.first_touch_time_msec/1000 * INTERVAL '1 second' 14 | and u.first_touch_time_msec is not null 15 | and day_diff <= 42 16 | group by 1,2,3,4 17 | ; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_retention_dau_wau.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | platform varchar(255), 4 | merged_user_id varchar(255), 5 | update_timestamp timestamp DEFAULT getdate() 6 | ) 7 | BACKUP YES 8 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_retention_event_overtime.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}}( 2 | event_date date, 3 | platform varchar(255), 4 | event_count bigint, 5 | update_timestamp timestamp DEFAULT getdate() 6 | ) 7 | BACKUP YES 8 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/dashboard/clickstream_retention_user_new_return.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{database_name}}.{{schema}}.{{viewName}} ( 2 | event_date date, 3 | platform varchar(255), 4 | user_type varchar(50), 5 | user_count bigint, 6 | update_timestamp timestamp DEFAULT getdate() 7 | ) 8 | BACKUP YES 9 | SORTKEY(event_date) -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/grant-permissions-to-bi-user-1.sql: -------------------------------------------------------------------------------- 1 | GRANT USAGE ON SCHEMA {{schema}} TO {{user_bi}}; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/grant-permissions-to-bi-user-2.sql: -------------------------------------------------------------------------------- 1 | ALTER DEFAULT PRIVILEGES IN SCHEMA {{schema}} GRANT SELECT ON TABLES TO {{user_bi}}; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/item-v2.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{schema}}.item_v2 ( 2 | event_timestamp timestamp not null, 3 | event_id varchar(255) not null, 4 | event_name varchar(255) not null, 5 | platform varchar(255), 6 | user_pseudo_id varchar(255) not null, 7 | user_id varchar(255), 8 | item_id varchar(255) not null, 9 | name varchar(255), 10 | brand varchar(255), 11 | currency varchar(255), 12 | price DOUBLE PRECISION, 13 | quantity DOUBLE PRECISION, 14 | creative_name varchar(255), 15 | creative_slot varchar(255), 16 | location_id varchar(255), 17 | category varchar(255), 18 | category2 varchar(255), 19 | category3 varchar(255), 20 | category4 varchar(255), 21 | category5 varchar(255), 22 | custom_parameters_json_str varchar(65535), 23 | custom_parameters super, 24 | process_info super, 25 | created_time timestamp DEFAULT getdate() 26 | ) BACKUP YES DISTSTYLE EVEN SORTKEY (event_timestamp); -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/migrate/fn-combine-json-list.sql: -------------------------------------------------------------------------------- 1 | create or replace function {{schema}}.combine_json_list(s varchar(65535)) 2 | returns varchar(65535) 3 | /* 4 | this function combines a list of json objects to a json object 5 | e.g. input: [{"a":1},{"b":2}] => output: {"a":1,"b":2} 6 | */ 7 | stable 8 | as $$ 9 | 10 | import json 11 | list_of_dicts = json.loads(s) 12 | combined_dict = {} 13 | for d in list_of_dicts: 14 | combined_dict.update(d) 15 | 16 | return json.dumps(combined_dict) 17 | 18 | $$ language plpythonu; 19 | 20 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/migrate/fn-rm-object-props.sql: -------------------------------------------------------------------------------- 1 | create or replace function {{schema}}.rm_object_props(o varchar(65535), props_list varchar(65535)) 2 | returns varchar(65535) 3 | stable 4 | as $$ 5 | 6 | import json 7 | d = json.loads(o) 8 | for prop in props_list.split(","): 9 | if prop in d: 10 | del d[prop] 11 | return json.dumps(d) 12 | 13 | $$ language plpythonu; 14 | 15 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/refresh-mv-sp-status.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS {{schema}}.{{table_refresh_mv_sp_status}} ( 2 | refresh_name varchar(255), 3 | refresh_type varchar(10), 4 | refresh_date date, 5 | triggerred_by varchar(50), 6 | log_date TIMESTAMP default getdate() 7 | ); 8 | -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/sp-clickstream-log-non-atomic.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE PROCEDURE {{schema}}.{{sp_clickstream_log_non_atomic}}(name in varchar(50), level in varchar(10), msg in varchar(256)) 2 | NONATOMIC AS 3 | $$ 4 | BEGIN 5 | INSERT INTO {{schema}}.{{table_clickstream_log}} (log_name, log_level, log_msg) VALUES(name, level, msg); 6 | EXCEPTION WHEN OTHERS THEN 7 | RAISE INFO 'error message: %', SQLERRM; 8 | END; 9 | $$ LANGUAGE plpgsql; -------------------------------------------------------------------------------- /src/analytics/private/sqls/redshift/sp-clickstream-log.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE PROCEDURE {{schema}}.{{sp_clickstream_log}}(name in varchar(50), level in varchar(10), msg in varchar(256)) 2 | AS 3 | $$ 4 | BEGIN 5 | INSERT INTO {{schema}}.{{table_clickstream_log}} (log_name, log_level, log_msg) VALUES(name, level, msg); 6 | EXCEPTION WHEN OTHERS THEN 7 | RAISE INFO 'error message: %', SQLERRM; 8 | END; 9 | $$ LANGUAGE plpgsql; -------------------------------------------------------------------------------- /src/base-lib/.gitattributes: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | * text=auto eol=lf 4 | /.gitattributes linguist-generated 5 | /.gitignore linguist-generated 6 | /.npmignore linguist-generated 7 | /.npmrc linguist-generated 8 | /.projen/** linguist-generated 9 | /.projen/deps.json linguist-generated 10 | /.projen/files.json linguist-generated 11 | /.projen/tasks.json linguist-generated 12 | /LICENSE linguist-generated 13 | /package.json linguist-generated 14 | /pnpm-lock.yaml linguist-generated 15 | /tsconfig.dev.json linguist-generated 16 | /tsconfig.json linguist-generated -------------------------------------------------------------------------------- /src/base-lib/.gitignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | !/.gitattributes 3 | !/.projen/tasks.json 4 | !/.projen/deps.json 5 | !/.projen/files.json 6 | !/package.json 7 | !/LICENSE 8 | !/.npmignore 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | lib-cov 21 | coverage 22 | *.lcov 23 | .nyc_output 24 | build/Release 25 | node_modules/ 26 | jspm_packages/ 27 | *.tsbuildinfo 28 | .eslintcache 29 | *.tgz 30 | .yarn-integrity 31 | .cache 32 | /test-reports/ 33 | junit.xml 34 | /coverage/ 35 | !/.npmrc 36 | !/test/ 37 | !/tsconfig.json 38 | !/tsconfig.dev.json 39 | !/src/ 40 | /lib 41 | /dist/ 42 | -------------------------------------------------------------------------------- /src/base-lib/.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | /test/ 7 | /tsconfig.dev.json 8 | /src/ 9 | !/lib/ 10 | !/lib/**/*.js 11 | !/lib/**/*.d.ts 12 | dist 13 | /tsconfig.json 14 | /.github/ 15 | /.vscode/ 16 | /.idea/ 17 | /.projenrc.js 18 | tsconfig.tsbuildinfo 19 | /.gitattributes 20 | -------------------------------------------------------------------------------- /src/base-lib/.npmrc: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | resolution-mode=highest 4 | -------------------------------------------------------------------------------- /src/base-lib/.projen/files.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | ".gitattributes", 4 | ".gitignore", 5 | ".npmignore", 6 | ".npmrc", 7 | ".projen/deps.json", 8 | ".projen/files.json", 9 | ".projen/tasks.json", 10 | "LICENSE", 11 | "tsconfig.dev.json", 12 | "tsconfig.json" 13 | ], 14 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"pnpm dlx projen\"." 15 | } 16 | -------------------------------------------------------------------------------- /src/base-lib/README.md: -------------------------------------------------------------------------------- 1 | # @clickstream/base-lib 2 | 3 | @clickstream/base-lib is a foundational library that integrates common methods, constants, and regular expressions, designed to simplify clickstream data processing and analysis, supporting code reuse across projects. 4 | -------------------------------------------------------------------------------- /src/base-lib/src/auth/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export * from './authorizer'; -------------------------------------------------------------------------------- /src/base-lib/src/common/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export * from './fetch'; 15 | export * from './powertools'; 16 | export * from './sdk-client-config'; 17 | export * from './solution-info'; 18 | export * from './utils'; 19 | -------------------------------------------------------------------------------- /src/base-lib/src/constant/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | export * from './constant'; 14 | export * from './explore-types'; 15 | export * from './dashboard'; 16 | -------------------------------------------------------------------------------- /src/base-lib/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export * from './constant'; 15 | export * from './model'; 16 | export * from './common'; 17 | export * from './auth'; -------------------------------------------------------------------------------- /src/base-lib/src/model/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export * from './metadata'; 15 | -------------------------------------------------------------------------------- /src/base-lib/tsconfig.json: -------------------------------------------------------------------------------- 1 | // ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | { 3 | "compilerOptions": { 4 | "rootDir": "src", 5 | "outDir": "lib", 6 | "alwaysStrict": true, 7 | "declaration": true, 8 | "esModuleInterop": true, 9 | "experimentalDecorators": true, 10 | "inlineSourceMap": true, 11 | "inlineSources": true, 12 | "lib": [ 13 | "es2019" 14 | ], 15 | "module": "CommonJS", 16 | "noEmitOnError": false, 17 | "noFallthroughCasesInSwitch": true, 18 | "noImplicitAny": true, 19 | "noImplicitReturns": true, 20 | "noImplicitThis": true, 21 | "noUnusedLocals": true, 22 | "noUnusedParameters": true, 23 | "resolveJsonModule": true, 24 | "strict": true, 25 | "strictNullChecks": true, 26 | "strictPropertyInitialization": true, 27 | "stripInternal": true, 28 | "target": "ES2019" 29 | }, 30 | "include": [ 31 | "src/**/*.ts" 32 | ], 33 | "exclude": [] 34 | } 35 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .aws-sam 3 | dist 4 | node_modules -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.gitattributes: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | * text=auto eol=lf 4 | /.gitattributes linguist-generated 5 | /.gitignore linguist-generated 6 | /.npmignore linguist-generated 7 | /.npmrc linguist-generated 8 | /.projen/** linguist-generated 9 | /.projen/deps.json linguist-generated 10 | /.projen/files.json linguist-generated 11 | /.projen/tasks.json linguist-generated 12 | /LICENSE linguist-generated 13 | /package.json linguist-generated 14 | /pnpm-lock.yaml linguist-generated 15 | /tsconfig.dev.json linguist-generated 16 | /tsconfig.json linguist-generated -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.gitignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | !/.gitattributes 3 | !/.projen/tasks.json 4 | !/.projen/deps.json 5 | !/.projen/files.json 6 | !/package.json 7 | !/LICENSE 8 | !/.npmignore 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | lib-cov 21 | coverage 22 | *.lcov 23 | .nyc_output 24 | build/Release 25 | node_modules/ 26 | jspm_packages/ 27 | *.tsbuildinfo 28 | .eslintcache 29 | *.tgz 30 | .yarn-integrity 31 | .cache 32 | src/aws-exports.js 33 | build/ 34 | ncc/ 35 | sqls/ 36 | /test-reports/ 37 | junit.xml 38 | /coverage/ 39 | !/.npmrc 40 | !/test// 41 | !/tsconfig.json 42 | !/tsconfig.dev.json 43 | !/.// 44 | /dist/ 45 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.npmignore: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | /.projen/ 3 | /test-reports/ 4 | junit.xml 5 | /coverage/ 6 | /test// 7 | /tsconfig.dev.json 8 | /.// 9 | !/dist// 10 | !/dist//**/*.js 11 | !/dist//**/*.d.ts 12 | dist 13 | /tsconfig.json 14 | /.github/ 15 | /.vscode/ 16 | /.idea/ 17 | /.projenrc.js 18 | tsconfig.tsbuildinfo 19 | /.gitattributes 20 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.npmrc: -------------------------------------------------------------------------------- 1 | # ~~ Generated by projen. To modify, edit .projenrc.js and run "pnpm dlx projen". 2 | 3 | resolution-mode=highest 4 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/.projen/files.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | ".gitattributes", 4 | ".gitignore", 5 | ".npmignore", 6 | ".npmrc", 7 | ".projen/deps.json", 8 | ".projen/files.json", 9 | ".projen/tasks.json", 10 | "LICENSE", 11 | "tsconfig.dev.json", 12 | "tsconfig.json" 13 | ], 14 | "//": "~~ Generated by projen. To modify, edit .projenrc.js and run \"pnpm dlx projen\"." 15 | } 16 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/common/sfn.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import { aws_sdk_client_common_config } from '@aws/clickstream-base-lib'; 15 | import { 16 | SFNClient, 17 | } from '@aws-sdk/client-sfn'; 18 | 19 | // Create SFN Client 20 | const sfnClient = new SFNClient({ 21 | ...aws_sdk_client_common_config, 22 | }); 23 | 24 | export { 25 | sfnClient, 26 | }; 27 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/locales/en-US.json: -------------------------------------------------------------------------------- 1 | { 2 | "dashboard": { 3 | "title": { 4 | "funnelAnalysis": "Funnel analysis", 5 | "eventAnalysis": "Event analysis", 6 | "pathAnalysis": "Path analysis", 7 | "retentionAnalysis": "Retention analysis", 8 | "attributionAnalysis": "Attribution analysis", 9 | "tableChart": "Detail information" 10 | }, 11 | "filter": { 12 | "scope": "Time filter scope is based on saved settings. Data out of scope can not be fetched." 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/locales/zh-CN.json: -------------------------------------------------------------------------------- 1 | { 2 | "dashboard": { 3 | "title": { 4 | "funnelAnalysis": "漏斗分析", 5 | "eventAnalysis": "事件分析", 6 | "pathAnalysis": "路径分析", 7 | "retentionAnalysis": "留存分析", 8 | "attributionAnalysis": "归因分析", 9 | "tableChart": "详细信息" 10 | }, 11 | "filter": { 12 | "scope": "该时间筛选器的作用范围基于此分析保存时的选择。例如保存此分析时选择的时间范围为过去7天则无法选择7天之前的数据。" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/model/dictionary.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export interface IDictionary { 15 | name: string; 16 | data: any; 17 | } 18 | -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | node ncc/index.js -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/dataset-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Placeholder": "", 3 | "DataSetSchema": { 4 | "ColumnSchemaList": [ 5 | ] 6 | }, 7 | "ColumnGroupSchemaList": [] 8 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/datetime-parameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "DateTimeParameterDeclaration": { 3 | "Name": "", 4 | "DefaultValues": { 5 | "StaticValues": [], 6 | "RollingDate": { 7 | "Expression": "truncDate('DD', now())" 8 | } 9 | }, 10 | "TimeGranularity": "DAY" 11 | } 12 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/filter-control-datetime.json: -------------------------------------------------------------------------------- 1 | { 2 | "DateTimePicker": { 3 | "FilterControlId": "", 4 | "Title": "", 5 | "SourceFilterId": "", 6 | "Type": "DATE_RANGE", 7 | "DisplayOptions": { 8 | "TitleOptions": { 9 | "Visibility": "VISIBLE", 10 | "FontConfiguration": { 11 | "FontSize": { 12 | "Relative": "MEDIUM" 13 | } 14 | } 15 | }, 16 | "DateTimeFormat": "YYYY/MM/DD HH:mm:ss", 17 | "InfoIconLabelOptions": { 18 | "Visibility": "VISIBLE", 19 | "InfoIconText": "" 20 | } 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/filter-control-relative-datetime.json: -------------------------------------------------------------------------------- 1 | { 2 | "RelativeDateTime": { 3 | "FilterControlId": "", 4 | "Title": "", 5 | "SourceFilterId": "", 6 | "DisplayOptions": { 7 | "TitleOptions": { 8 | "Visibility": "VISIBLE", 9 | "FontConfiguration": { 10 | "FontSize": { 11 | "Relative": "MEDIUM" 12 | } 13 | } 14 | }, 15 | "DateTimeFormat": "YYYY/MM/DD HH:mm:ss", 16 | "InfoIconLabelOptions": { 17 | "Visibility": "VISIBLE", 18 | "InfoIconText": "" 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/layout-control.json: -------------------------------------------------------------------------------- 1 | { 2 | "ElementId": "", 3 | "ElementType": "FILTER_CONTROL", 4 | "ColumnIndex": 0, 5 | "ColumnSpan": 8, 6 | "RowIndex": 0, 7 | "RowSpan": 4 8 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/layout-visual.json: -------------------------------------------------------------------------------- 1 | { 2 | "ElementId": "", 3 | "ElementType": "VISUAL", 4 | "ColumnIndex": 0, 5 | "ColumnSpan": 36, 6 | "RowIndex": 4, 7 | "RowSpan": 20 8 | } -------------------------------------------------------------------------------- /src/control-plane/backend/lambda/api/service/quicksight/templates/percentage-column-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Column": { 3 | "DataSetIdentifier": "", 4 | "ColumnName": "" 5 | }, 6 | "FormatConfiguration": { 7 | "NumberFormatConfiguration": { 8 | "FormatConfiguration": { 9 | "PercentageDisplayFormatConfiguration": { 10 | "Suffix": "%", 11 | "SeparatorConfiguration": { 12 | "DecimalSeparator": "DOT", 13 | "ThousandsSeparator": { 14 | "Symbol": "COMMA", 15 | "Visibility": "VISIBLE" 16 | } 17 | }, 18 | "NegativeValueConfiguration": { 19 | "DisplayMode": "NEGATIVE" 20 | } 21 | } 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /src/control-plane/backend/layer/lambda-web-adapter/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec -- "${LAMBDA_TASK_ROOT}/${_HANDLER}" -------------------------------------------------------------------------------- /src/control-plane/backend/layer/lambda-web-adapter/lambda-adapter: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | set -euo pipefail 6 | 7 | OWN_FILENAME="$(basename $0)" 8 | LAMBDA_EXTENSION_NAME="$OWN_FILENAME" # (external) extension name has to match the filename 9 | echo "[${LAMBDA_EXTENSION_NAME}] Initialization" 10 | 11 | ARCH=`uname -m` 12 | CMD="/opt/libs/${ARCH}/lambda-adapter" 13 | 14 | echo "[${LAMBDA_EXTENSION_NAME}] Running on ${ARCH}, forwarding to corresponding adapter ${CMD}" 15 | 16 | "$CMD" 17 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project 2 | group=software.aws.solution.clickstream.common 3 | projectVersion=1.1.9 4 | 5 | # Dependencies 6 | lombokVersion=1.18.30 7 | junitVersion=5.10.1 8 | junitBomVersion=5.10.1 9 | assertJVersion=3.24.2 10 | mockitoVersion=5.2.0 11 | jacocoVersion=0.8.11 12 | jacocoLogVersion=3.1.0 13 | checkstyleVersion=9.3 14 | guavaVersion=32.1.2-jre 15 | uapJavaVersion=1.5.3 16 | maxmindDbVersion=3.1.0 17 | fasterxmlJacksonAnnotationsVersion=2.14.2 18 | fasterxmlJacksonCoreVersion=2.14.2 19 | fasterxmlJacksonDatabindVersion=2.14.2 20 | log4j2Version=2.17.1 21 | slf4jVersion=2.0.6 22 | caffeineVersion=3.1.8 23 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/gradle/config/scripts/idea.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'idea' 2 | 3 | rootProject.idea { 4 | workspace.iws.withXml { provider -> 5 | def junitDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 6 | it.@type == 'JUnit' 7 | } 8 | junitDefaults.option.find { it.@name == 'WORKING_DIRECTORY' }.@value = '$MODULE_DIR$' 9 | 10 | def applicationDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 11 | it.@default == 'true' && it.@type == 'Application' 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/gradle/config/scripts/style.gradle: -------------------------------------------------------------------------------- 1 | class StyleExtension { 2 | List excludePackages; 3 | List excludeClasses; 4 | } 5 | 6 | project.extensions.create('style', StyleExtension) 7 | 8 | apply plugin: 'checkstyle' 9 | 10 | checkstyle { 11 | configDirectory = file("$rootDir/gradle/config/checkstyle") 12 | configFile = file("$rootDir/gradle/config/checkstyle/sun_checks.xml") 13 | toolVersion = checkstyleVersion 14 | } 15 | 16 | checkstyleTest.enabled = false 17 | 18 | checkstyleMain.doFirst { 19 | style.excludePackages.each() { 20 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '/*'}" 21 | } 22 | 23 | style.excludeClasses.each() { 24 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '.java'}" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/src/data-pipeline/etl-common/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.data.flagUsage=error -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'etl-common' 2 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/enrich/ts/SourceMedium.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.common.enrich.ts; 15 | 16 | import lombok.AllArgsConstructor; 17 | import lombok.Getter; 18 | 19 | @Getter 20 | @AllArgsConstructor 21 | public class SourceMedium { 22 | String source; 23 | String medium; 24 | } 25 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/exception/ExtractDataException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.common.exception; 15 | 16 | public class ExtractDataException extends RuntimeException{ 17 | public ExtractDataException(final Exception e) { 18 | super(e); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/expected/test_parse_line_to_db_row_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : 1682319109447, 3 | "event_id" : "1fcd7f5b-9529-4977-a303-e8c7e39db7b898", 4 | "event_name" : "Click3", 5 | "platform" : "ANDROID", 6 | "user_pseudo_id" : "83e45133-ddaf-4ac6-9844-fe457807715198", 7 | "user_id" : "312121", 8 | "item_id" : "item_id1", 9 | "name" : "French Press1", 10 | "brand" : "Brand1", 11 | "currency" : null, 12 | "price" : 52.99, 13 | "quantity" : 42.0, 14 | "creative_name" : "Creative Name", 15 | "creative_slot" : "Creative Slot", 16 | "location_id" : "Location#001", 17 | "category" : "housewares", 18 | "category2" : "Category-2", 19 | "category3" : "Category-3", 20 | "category4" : "Category-4", 21 | "category5" : "Category-5", 22 | "custom_parameters" : { }, 23 | "process_info" : null 24 | } 25 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/gtm-server/expected/test_gtm_parseLineToDBRow_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : 1695261266828, 3 | "event_id" : "25aefd4cb653fd0fcbac33e24fd3f0ba-0-1695261065-1", 4 | "event_name" : "select_item", 5 | "platform" : "Windows", 6 | "user_pseudo_id" : "oEzstqb1SUkULygCXFzTFuOZPnG9GItyNJIo8t9tCi0=.1695261066", 7 | "user_id" : null, 8 | "item_id" : "CTF-28015231-16005642", 9 | "name" : "Apple Watch Metallic Impact Case with Screen Protector 49mm - Orange", 10 | "brand" : null, 11 | "currency" : null, 12 | "price" : 35.6638714628, 13 | "quantity" : null, 14 | "creative_name" : null, 15 | "creative_slot" : null, 16 | "location_id" : null, 17 | "category" : null, 18 | "category2" : null, 19 | "category3" : null, 20 | "category4" : null, 21 | "category5" : null, 22 | "custom_parameters" : { 23 | "region" : "US" 24 | }, 25 | "process_info" : null 26 | } -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/gtm-server/expected/test_gtm_parseLineToDBRow_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : 1704867230000, 3 | "user_pseudo_id" : "cuPHQ5qyIqHl8QiSr3aZXrAR+jGYJKhuyWzJcFI39ds=.1703490225", 4 | "user_id" : null, 5 | "user_properties" : null, 6 | "first_touch_time_msec" : 1704867230837, 7 | "first_visit_date" : 1704867230837, 8 | "first_referrer" : null, 9 | "first_traffic_source" : "Direct", 10 | "first_traffic_medium" : "None", 11 | "first_traffic_campaign" : "Direct", 12 | "first_traffic_content" : null, 13 | "first_traffic_term" : null, 14 | "first_traffic_campaign_id" : null, 15 | "first_traffic_clid_platform" : null, 16 | "first_traffic_clid" : null, 17 | "first_traffic_channel_group" : "Direct", 18 | "first_traffic_category" : "Direct", 19 | "first_app_install_source" : null, 20 | "process_info" : null 21 | } -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/sensors-data/expected/test_sensors_parseLineToDBRow_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : 1690249851000, 3 | "user_pseudo_id" : "9113946abd821d52", 4 | "user_id" : "9113946abd821d52", 5 | "user_properties" : { 6 | "$x_test_user" : { 7 | "set_time_msec" : null, 8 | "value" : "Test Name" 9 | } 10 | }, 11 | "first_touch_time_msec" : 1690249850031, 12 | "first_visit_date" : 1690249850031, 13 | "first_referrer" : null, 14 | "first_traffic_source" : "Direct", 15 | "first_traffic_medium" : "None", 16 | "first_traffic_campaign" : "Direct", 17 | "first_traffic_content" : null, 18 | "first_traffic_term" : null, 19 | "first_traffic_campaign_id" : null, 20 | "first_traffic_clid_platform" : null, 21 | "first_traffic_clid" : null, 22 | "first_traffic_channel_group" : "Direct", 23 | "first_traffic_category" : "Direct", 24 | "first_app_install_source" : null, 25 | "process_info" : null 26 | } -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/sensors-data/sensors-web-data-empty.json: -------------------------------------------------------------------------------- 1 | {"date":"2024-04-24T08:13:56+00:00","ingest_time":1713946436000,"data":"","method":"GET","ip":"43.207.138.168","source_type":"http_server","fakeIp":null,"rid":"7e09de543165a6e66a8704f50e808b9c","ua":"node-fetch/1.0 (+https://github.com/bitinn/node-fetch)","uri":"/collect","platform":null,"path":"/collect","appId":null,"server_ingest_time":1713946436000,"compression":null,"timestamp":"2024-04-24T08:13:56.802235016Z"} 2 | {"date":"2024-04-24T08:14:08+00:00","ingest_time":1713946448000,"data":"","method":"GET","ip":"43.207.138.168","source_type":"http_server","fakeIp":null,"rid":"6e428d2e1a5c536c4ed0fb285d60ca97","ua":"node-fetch/1.0 (+https://github.com/bitinn/node-fetch)","uri":"/collect","platform":null,"path":"/collect","appId":null,"server_ingest_time":1713946448000,"compression":null,"timestamp":"2024-04-24T08:14:08.528966436Z"} 3 | -------------------------------------------------------------------------------- /src/data-pipeline/etl-common/src/test/resources/ts/traffic_source_category_rule_test.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "url": "search.comcast.net", 4 | "source": "Comcast", 5 | "category": "Search", 6 | "params": [ 7 | "q", 8 | "query" 9 | ] 10 | }, 11 | { 12 | "url": "wechat.com/search", 13 | "source": "wechat search", 14 | "category": "Social", 15 | "params": [] 16 | }, 17 | { 18 | "url": "za.pinterest.com", 19 | "source": "za.pinterest", 20 | "category": "Social", 21 | "params": [] 22 | } 23 | ] -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | bin 3 | build 4 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project 2 | group=software.aws.solution 3 | projectVersion=1.1.9 4 | 5 | # Dependencies 6 | lombokVersion=1.18.30 7 | junitVersion=5.10.1 8 | junitBomVersion=5.10.1 9 | assertJVersion=3.24.2 10 | mockitoVersion=5.8.0 11 | jacocoVersion=0.8.11 12 | jacocoLogVersion=3.1.0 13 | checkstyleVersion=9.3 14 | guavaVersion=32.1.2-jre 15 | sparkVersion=3.4.1 16 | uapJavaVersion=1.5.3 17 | maxmindDbVersion=3.1.0 18 | caffeineVersion=3.1.8 19 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/gradle/config/scripts/idea.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'idea' 2 | 3 | rootProject.idea { 4 | workspace.iws.withXml { provider -> 5 | def junitDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 6 | it.@type == 'JUnit' 7 | } 8 | junitDefaults.option.find { it.@name == 'WORKING_DIRECTORY' }.@value = '$MODULE_DIR$' 9 | 10 | def applicationDefaults = provider.node.component.find { it.@name == 'RunManager' }.configuration.find { 11 | it.@default == 'true' && it.@type == 'Application' 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/gradle/config/scripts/style.gradle: -------------------------------------------------------------------------------- 1 | class StyleExtension { 2 | List excludePackages; 3 | List excludeClasses; 4 | } 5 | 6 | project.extensions.create('style', StyleExtension) 7 | 8 | apply plugin: 'checkstyle' 9 | 10 | checkstyle { 11 | configDirectory = file("$rootDir/gradle/config/checkstyle") 12 | configFile = file("$rootDir/gradle/config/checkstyle/sun_checks.xml") 13 | toolVersion = checkstyleVersion 14 | } 15 | 16 | checkstyleTest.enabled = false 17 | 18 | checkstyleMain.doFirst { 19 | style.excludePackages.each() { 20 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '/*'}" 21 | } 22 | 23 | style.excludeClasses.each() { 24 | checkstyleMain.exclude "${'**/' + it.replaceAll('\\.', '/') + '.java'}" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/src/data-pipeline/spark-etl/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/lombok.config: -------------------------------------------------------------------------------- 1 | lombok.data.flagUsage=error -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spark-etl' 2 | //include('app') 3 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/main/java/software/aws/solution/clickstream/transformer/TransformConfigurable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.transformer; 15 | 16 | import software.aws.solution.clickstream.common.TransformConfig; 17 | 18 | public interface TransformConfigurable { 19 | TransformConfig getTransformConfig(); 20 | } 21 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/main/java/software/aws/solution/clickstream/transformer/TransformerNameEnum.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.transformer; 15 | 16 | public enum TransformerNameEnum { 17 | CLICKSTREAM, GTM_SERVER_DATA, SENSORS_DATA; 18 | } 19 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/main/java/software/aws/solution/clickstream/udfconverter/DatasetConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.udfconverter; 15 | 16 | import org.apache.spark.sql.Dataset; 17 | import org.apache.spark.sql.Row; 18 | 19 | 20 | public interface DatasetConverter { 21 | Dataset transform(Dataset dataset); 22 | } 23 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/main/java/software/aws/solution/clickstream/util/PathContent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | package software.aws.solution.clickstream.util; 15 | 16 | import lombok.Getter; 17 | import lombok.Setter; 18 | 19 | @Getter 20 | @Setter 21 | public class PathContent { 22 | private String path; // NOSONAR 23 | private String content; // NOSONAR 24 | } 25 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/etl_runner_v3_event2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/etl_runner_v3_event2.json -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/test_enrich_UA_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "device_ua_browser" : "Mobile Safari", 3 | "device_ua_browser_version" : "16.6", 4 | "device_ua_os" : "iOS", 5 | "device_ua_os_version" : "16.6.1", 6 | "device_ua_device" : "iPhone", 7 | "device_ua_device_category" : "Mobile", 8 | "device_ua" : { 9 | "device" : "{\"family\":\"iPhone\"}", 10 | "os" : "{\"family\":\"iOS\",\"major\":\"16\",\"minor\":\"6\",\"patch\":\"1\",\"patchMinor\":null}", 11 | "string" : "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1", 12 | "userAgent" : "{\"family\":\"Mobile Safari\",\"major\":\"16\",\"minor\":\"6\",\"patch\":null}" 13 | } 14 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/test_enrich_ip_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "ip" : "70.123.241.83", 3 | "geo_city" : "Schertz", 4 | "geo_continent" : "North America", 5 | "geo_country" : "United States", 6 | "geo_metro" : null, 7 | "geo_region" : "US-TX", 8 | "geo_sub_continent" : null, 9 | "geo_locale" : null 10 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/test_extract_session_from_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : "2024-05-16 19:50:01", 3 | "user_pseudo_id" : "412f30bf-a662-4cf8-8548-770816852f55", 4 | "session_id" : "16852f55-20240516-195001418", 5 | "user_id" : "NULL", 6 | "session_number" : null, 7 | "session_start_time_msec" : null, 8 | "session_source" : "Google", 9 | "session_medium" : "Organic", 10 | "session_campaign" : "NULL", 11 | "session_content" : "NULL", 12 | "session_term" : "NULL", 13 | "session_campaign_id" : "NULL", 14 | "session_clid_platform" : "NULL", 15 | "session_clid" : "NULL", 16 | "session_channel_group" : "Organic Search", 17 | "session_source_category" : "Search", 18 | "process_info" : { 19 | "event_name" : "_session_start", 20 | "process_job_id" : "test-job", 21 | "process_time" : "_PROCESS_TIME_" 22 | }, 23 | "app_id" : "blog", 24 | "created_time" : "_CREATED_TIME_" 25 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/event_v2/expected/test_extract_session_from_event2.json: -------------------------------------------------------------------------------- 1 | { 2 | "event_timestamp" : "2024-05-16 20:05:02", 3 | "user_pseudo_id" : "412f30bf-a662-4cf8-8548-770816852f55", 4 | "session_id" : "16852f55-20240516-1950014182", 5 | "user_id" : "NULL", 6 | "session_number" : null, 7 | "session_start_time_msec" : null, 8 | "session_source" : "Direct", 9 | "session_medium" : "Referral", 10 | "session_campaign" : "Direct", 11 | "session_content" : "NULL", 12 | "session_term" : "NULL", 13 | "session_campaign_id" : "NULL", 14 | "session_clid_platform" : "NULL", 15 | "session_clid" : "NULL", 16 | "session_channel_group" : "Internal", 17 | "session_source_category" : "Direct", 18 | "process_info" : { 19 | "event_name" : "_user_engagement", 20 | "process_job_id" : "test-job", 21 | "process_time" : "_PROCESS_TIME_" 22 | }, 23 | "app_id" : "blog", 24 | "created_time" : "_CREATED_TIME_" 25 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/expected/transform_v2_user_app_start1.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "appStartTest", 3 | "event_date" : "2023-04-24", 4 | "event_timestamp" : 1682319109440, 5 | "user_id" : null, 6 | "user_pseudo_id" : "uuid1", 7 | "user_first_touch_timestamp" : 1682319109440, 8 | "user_properties" : null, 9 | "user_ltv" : null, 10 | "_first_visit_date" : "2023-04-24", 11 | "_first_referer" : null, 12 | "_first_traffic_source_type" : null, 13 | "_first_traffic_medium" : null, 14 | "_first_traffic_source" : null, 15 | "device_id_list" : [ "deviceid-1682319109447" ], 16 | "_channel" : null 17 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/expected/transform_v2_user_traffic_source.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "uba-app", 3 | "event_date" : "2023-04-24", 4 | "event_timestamp" : 1682319109442, 5 | "user_id" : null, 6 | "user_pseudo_id" : "uu_p3121211_ts1", 7 | "user_first_touch_timestamp" : 1667877267895, 8 | "user_properties" : null, 9 | "user_ltv" : null, 10 | "_first_visit_date" : "2022-11-08", 11 | "_first_referer" : null, 12 | "_first_traffic_source_type" : "TSN1", 13 | "_first_traffic_medium" : "TSM1", 14 | "_first_traffic_source" : "TSS1", 15 | "device_id_list" : null, 16 | "_channel" : null 17 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/etl_gtm_user_v2_props_incremental_v1/update_date=20240312/app_id=GMTServerLogin2/part-00000-8f5a4cb2-f6aa-4b4f-8c6a-08145a4e5b15.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/src/data-pipeline/spark-etl/src/test/resources/gtm-server/etl_gtm_user_v2_props_incremental_v1/update_date=20240312/app_id=GMTServerLogin2/part-00000-8f5a4cb2-f6aa-4b4f-8c6a-08145a4e5b15.c000.snappy.parquet -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_GTM_server_runner_v2_item_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "created_time" : "_CREATED_TIME_", 3 | "event_id" : "25aefd4cb653fd0fcbac33e24fd3f0ba-0-1695261065-1", 4 | "event_name" : "select_item", 5 | "event_timestamp" : "2023-09-21T01:54:26.828Z", 6 | "item_id" : "CTF-28015231-16005642", 7 | "name" : "Apple Watch Metallic Impact Case with Screen Protector 49mm - Orange", 8 | "platform" : "Windows", 9 | "price" : 35.6638714628, 10 | "process_info" : { 11 | "process_job_id" : "test-job", 12 | "process_time" : "_PROCESS_TIME_" 13 | }, 14 | "user_pseudo_id" : "oEzstqb1SUkULygCXFzTFuOZPnG9GItyNJIo8t9tCi0=.1695261066", 15 | "partition_app" : "testAllAppId1", 16 | "partition_year" : 2023, 17 | "partition_month" : 9, 18 | "partition_day" : 21 19 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_gtm_parseLineToDBRow_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventTimestamp" : 1695261266828, 3 | "eventId" : "25aefd4cb653fd0fcbac33e24fd3f0ba-0-1695261065-1", 4 | "eventName" : "select_item", 5 | "platform" : "Web", 6 | "userPseudoId" : "oEzstqb1SUkULygCXFzTFuOZPnG9GItyNJIo8t9tCi0=.1695261066", 7 | "userId" : null, 8 | "itemId" : "CTF-28015231-16005642", 9 | "name" : "Apple Watch Metallic Impact Case with Screen Protector 49mm - Orange", 10 | "brand" : null, 11 | "currency" : null, 12 | "price" : 35.6638714628, 13 | "quantity" : null, 14 | "creativeName" : null, 15 | "creativeSlot" : null, 16 | "locationId" : null, 17 | "category" : null, 18 | "category2" : null, 19 | "category3" : null, 20 | "category4" : null, 21 | "category5" : null, 22 | "customParameters" : null, 23 | "processInfo" : null, 24 | "appId" : "example_clickstream_analytics_aws" 25 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_gtm_parseLineToDBRow_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventTimestamp" : 1704867230000, 3 | "userPseudoId" : "cuPHQ5qyIqHl8QiSr3aZXrAR+jGYJKhuyWzJcFI39ds=.1703490225", 4 | "userId" : null, 5 | "userProperties" : null, 6 | "firstTouchTimeMsec" : 1704867230837, 7 | "firstVisitDate" : 1704867230837, 8 | "firstReferrer" : null, 9 | "firstTrafficSource" : "direct", 10 | "firstTrafficMedium" : null, 11 | "firstTrafficCampaign" : "direct", 12 | "firstTrafficContent" : null, 13 | "firstTrafficTerm" : null, 14 | "firstTrafficCampaignId" : null, 15 | "firstTrafficClidPlatform" : null, 16 | "firstTrafficClid" : null, 17 | "firstTrafficChannelGroup" : null, 18 | "firstTrafficCategory" : null, 19 | "firstAppInstallSource" : null, 20 | "processInfo" : null, 21 | "appId" : "sessionStartAppId1", 22 | "eventName" : "_user_engagement" 23 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_transform_data_event_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "testAllAppId1", 3 | "event_date" : "2023-09-21", 4 | "event_timestamp" : 1695261256542, 5 | "event_id" : "4a31fde2533e11dd2b0e7800720f6f86-0-1695260713-0", 6 | "event_name" : "PDP Clicks", 7 | "event_param_key" : "x-ga-tfd", 8 | "event_param_double_value" : null, 9 | "event_param_float_value" : null, 10 | "event_param_int_value" : 32152, 11 | "event_param_string_value" : null 12 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_transform_data_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "testAllAppId1", 3 | "event_date" : "2023-09-21", 4 | "event_timestamp" : 1695261261382, 5 | "id" : "CTF-28015231-16005642", 6 | "properties" : [ { 7 | "key" : "item_name", 8 | "value" : { 9 | "double_value" : null, 10 | "float_value" : null, 11 | "int_value" : null, 12 | "string_value" : "Apple Watch Metallic Impact Case with Screen Protector 49mm - Orange" 13 | } 14 | }, { 15 | "key" : "price", 16 | "value" : { 17 | "double_value" : 35.6638714628, 18 | "float_value" : null, 19 | "int_value" : null, 20 | "string_value" : null 21 | } 22 | }, { 23 | "key" : "item_list_name", 24 | "value" : { 25 | "double_value" : null, 26 | "float_value" : null, 27 | "int_value" : null, 28 | "string_value" : "" 29 | } 30 | } ] 31 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_transform_data_session_start_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "sessionStartAppId1", 3 | "event_date" : "2024-01-10", 4 | "event_timestamp" : 1704867230837, 5 | "event_id" : "55384aaeda62c5b20a462076ec59f3f5-0-1704867229-0-1", 6 | "event_name" : "_session_start", 7 | "event_param_key" : "_session_duration", 8 | "event_param_double_value" : null, 9 | "event_param_float_value" : null, 10 | "event_param_int_value" : 0, 11 | "event_param_string_value" : null 12 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/gtm-server/expected/test_transform_data_user_login2.json: -------------------------------------------------------------------------------- 1 | { 2 | "app_id" : "GMTServerLogin2", 3 | "event_date" : "2023-08-29", 4 | "event_timestamp" : 1693281865635, 5 | "user_id" : "x-0eb41e46-2373-4883-8daf-e1975ccb3821", 6 | "user_pseudo_id" : "X-login-1690769179", 7 | "user_first_touch_timestamp" : 1693281865635, 8 | "user_properties" : null, 9 | "user_ltv" : null, 10 | "_first_visit_date" : "2023-08-29", 11 | "_first_referer" : null, 12 | "_first_traffic_source_type" : null, 13 | "_first_traffic_medium" : null, 14 | "_first_traffic_source" : null, 15 | "device_id_list" : null, 16 | "_channel" : null 17 | } -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/original_data_null.json: -------------------------------------------------------------------------------- 1 | {"date":"2023-04-24T05:54:57.838Z", "data": "", "ua":"Apache-HttpClient/4.5.12 (Java/11.0.15)","m":"POST","uri":"/collect?loadTestData=1&appId=uba-app&compression=gzip&event_bundle_sequence_id=111","platform":"ANDROID","path":"/collect","appId":"uba-app","compression":"gzip","ingest_time":1682315697838,"timestamp":"2023-04-24T05:54:57.838Z"} 2 | {"date":"2023-04-24T05:54:57.838Z", "data": null, "ua":"Apache-HttpClient/4.5.12 (Java/11.0.15)","m":"POST","uri":"/collect?loadTestData=1&appId=uba-app&compression=gzip&event_bundle_sequence_id=111","platform":"ANDROID","path":"/collect","appId":"uba-app","compression":"gzip","ingest_time":1682315697838,"timestamp":"2023-04-24T05:54:57.838Z"} 3 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/original_data_raw_json_error.json: -------------------------------------------------------------------------------- 1 | {"date":"2023-04-24T05:54:57.838Z","data":"[\"test data json array\"]","ip":"13.212.229.59","source_type":"http","rid":"07157da84281bb7234ba6f8613df7d32","ua":"Apache-HttpClient/4.5.12 (Java/11.0.15)","m":"POST","uri":"/collect?loadTestData=1&appId=uba-app&compression=gzip&event_bundle_sequence_id=111","platform":"ANDROID","path":"/collect","appId":"uba-app","compression":"gzip","ingest_time":1682315697838,"timestamp":"2023-04-24T05:54:57.838Z"} 2 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/original_data_raw_text_error.json: -------------------------------------------------------------------------------- 1 | {"date":"2023-04-24T05:54:57.838Z","data":"test data text","ip":"13.212.229.59","source_type":"http","rid":"07157da84281bb7234ba6f8613df7d32","ua":"Apache-HttpClient/4.5.12 (Java/11.0.15)","m":"POST","uri":"/collect?loadTestData=1&appId=uba-app&compression=gzip&event_bundle_sequence_id=111","platform":"ANDROID","path":"/collect","appId":"uba-app","compression":"gzip","ingest_time":1682315697838,"timestamp":"2023-04-24T05:54:57.838Z"} 2 | -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/partition_data/year=2022/month=11/day=11/corrupt_data.json: -------------------------------------------------------------------------------- 1 | "abc" -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/test_parse_line_to_db_row_event_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions/clickstream-analytics-on-aws/fcede5072ce23202ca96aa883220ffe798061b5c/src/data-pipeline/spark-etl/src/test/resources/test_parse_line_to_db_row_event_v2.json -------------------------------------------------------------------------------- /src/data-pipeline/spark-etl/src/test/resources/user_cn_chars.json: -------------------------------------------------------------------------------- 1 | {"name": "刘先生", "sex": "M", "age": 18, "address": "北京市海淀区", "phone": "13888888888"} 2 | {"name": "李先生", "sex": "M", "age": 19, "address": "陕西省西安市雁塔区128号", "phone": "13888888889"} -------------------------------------------------------------------------------- /src/ingestion-server/kinesis-data-stream/private/iam.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | import { Construct } from 'constructs'; 15 | import { createLambdaRole } from '../../../common/lambda'; 16 | 17 | export function createKinesisToS3LambdaRole(scope: Construct) { 18 | return createLambdaRole(scope, 'kinesisToS3LambdaRole', true, []); 19 | } -------------------------------------------------------------------------------- /src/ingestion-server/server/images/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PLATFORM_ARG 2 | FROM --platform=$PLATFORM_ARG public.ecr.aws/docker/library/nginx:1.27 3 | 4 | # default nginx path 5 | ENV SERVER_ENDPOINT_PATH='/collect' 6 | ENV PING_ENDPOINT_PATH='/ping' 7 | ENV SERVER_CORS_ORIGIN='*' 8 | ENV NGINX_WORKER_CONNECTIONS='1024' 9 | 10 | COPY ./config/nginx.conf /etc/nginx/nginx.conf 11 | COPY ./config/docker-entrypoint.sh / 12 | 13 | RUN rm -rf /etc/nginx/conf.d/default.conf 14 | 15 | RUN chown -R nginx /etc/nginx \ 16 | && chown -R nginx /docker-entrypoint.d \ 17 | && chown -R nginx /usr/share/nginx \ 18 | && chown -R nginx /run \ 19 | && chown -R nginx /var/cache/nginx \ 20 | && chown nginx ./docker-entrypoint.sh 21 | RUN chmod u+x ./docker-entrypoint.sh && chmod u+rw /etc/nginx/nginx.conf 22 | 23 | USER nginx 24 | 25 | HEALTHCHECK --interval=2m --timeout=5s \ 26 | CMD curl -f http://127.0.0.1:8088/health || exit 1 27 | 28 | EXPOSE 8088 29 | -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PLATFORM_ARG 2 | FROM --platform=$PLATFORM_ARG timberio/vector:0.43.0-debian 3 | 4 | ENV AWS_REGION='__NOT_SET__' 5 | ENV AWS_MSK_BROKERS='__NOT_SET__' 6 | ENV AWS_MSK_TOPIC='__NOT_SET__' 7 | ENV AWS_S3_BUCKET='__NOT_SET__' 8 | ENV AWS_S3_PREFIX='s3-data' 9 | ENV S3_BATCH_MAX_BYTES=30000000 10 | ENV S3_BATCH_TIMEOUT_SECS=300 11 | ENV AWS_KINESIS_STREAM_NAME='__NOT_SET__' 12 | 13 | ENV VECTOR_REQUIRE_HEALTHY='false' 14 | ENV STREAM_ACK_ENABLE='true' 15 | ENV WORKER_THREADS_NUM='-1' 16 | 17 | COPY ./config/vector*.toml /etc/vector/ 18 | COPY ./config/entrypoint.sh / 19 | 20 | RUN groupadd -r vector && useradd -r -g vector vector 21 | 22 | RUN chown vector /entrypoint.sh \ 23 | && chown -R vector /etc/vector \ 24 | && chown -R vector /var/lib/vector 25 | RUN chmod u+x /entrypoint.sh && chmod u+wr /etc/vector/*.toml 26 | 27 | USER vector 28 | 29 | ENTRYPOINT [ "/entrypoint.sh" ] 30 | -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-global.toml: -------------------------------------------------------------------------------- 1 | data_dir = "/var/lib/vector/" 2 | timezone = "UTC" 3 | healthchecks.enabled = true 4 | acknowledgements.enabled = false 5 | 6 | [api] 7 | enabled = true 8 | address = "0.0.0.0:8686" 9 | 10 | -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-kinesis-ack.toml: -------------------------------------------------------------------------------- 1 | 2 | [sinks.kinesis_sink] 3 | type = "aws_kinesis_streams" 4 | inputs = ["json_parser"] 5 | partition_key_field = "timestamp" 6 | compression = "none" 7 | region = "%%AWS_REGION%%" 8 | stream_name = "%%AWS_KINESIS_STREAM_NAME%%" 9 | 10 | acknowledgements.enabled = true 11 | request_retry_partial = true 12 | request.retry_attempts = 4 13 | request.retry_max_duration_secs = 3 14 | 15 | [sinks.kinesis_sink.encoding] 16 | codec = "json" -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-kinesis-batch.toml: -------------------------------------------------------------------------------- 1 | 2 | [sinks.kinesis_sink] 3 | type = "aws_kinesis_streams" 4 | inputs = ["json_parser"] 5 | partition_key_field = "timestamp" 6 | compression = "none" 7 | region = "%%AWS_REGION%%" 8 | stream_name = "%%AWS_KINESIS_STREAM_NAME%%" 9 | # Each PutRecords request can support up to 500 records. Each record in the request can be as large as 1 MiB, up to a limit of 5 MiB for the entire request, 10 | batch.max_events = 500 11 | batch.timeout_secs = 3 12 | acknowledgements.enabled = false 13 | [sinks.kinesis_sink.encoding] 14 | codec = "json" -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-msk-ack.toml: -------------------------------------------------------------------------------- 1 | # https://vector.dev/docs/about/under-the-hood/architecture/buffering-model/ 2 | 3 | [sinks.msk_sink] 4 | type = "kafka" 5 | inputs = ["json_parser"] 6 | bootstrap_servers = "%%AWS_MSK_BROKERS%%" 7 | #key_field = "ip" 8 | topic = "%%AWS_MSK_TOPIC%%" 9 | compression = "none" 10 | acknowledgements.enabled = true 11 | [sinks.msk_sink.encoding] 12 | codec = "json" 13 | -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-msk-batch.toml: -------------------------------------------------------------------------------- 1 | # https://vector.dev/docs/about/under-the-hood/architecture/buffering-model/ 2 | 3 | [sinks.msk_sink] 4 | type = "kafka" 5 | inputs = ["json_parser"] 6 | bootstrap_servers = "%%AWS_MSK_BROKERS%%" 7 | #key_field = "ip" 8 | topic = "%%AWS_MSK_TOPIC%%" 9 | compression = "none" 10 | batch.max_events = 1000 11 | batch.timeout_secs = 3 12 | acknowledgements.enabled = false 13 | [sinks.msk_sink.encoding] 14 | codec = "json" 15 | -------------------------------------------------------------------------------- /src/ingestion-server/server/images/vector/config/vector-s3.toml: -------------------------------------------------------------------------------- 1 | [sinks.s3_archives] 2 | inputs = ["json_parser"] 3 | type = "aws_s3" 4 | region = "%%AWS_REGION%%" 5 | bucket = "%%AWS_S3_BUCKET%%" 6 | key_prefix = "%%AWS_S3_PREFIX%%year=%Y/month=%m/day=%d/hour=%H/" 7 | compression = "gzip" # compress final objects 8 | framing.method = "newline_delimited" # new line delimited... 9 | encoding.codec = "json" # ...JSON 10 | batch.max_bytes = %%S3_BATCH_MAX_BYTES%% 11 | batch.timeout_secs = %%S3_BATCH_TIMEOUT_SECS%% 12 | acknowledgements.enabled = false 13 | -------------------------------------------------------------------------------- /test/constants.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance 5 | * with the License. A copy of the License is located at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES 10 | * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions 11 | * and limitations under the License. 12 | */ 13 | 14 | export enum CFN_FN { 15 | AND='Fn::And', 16 | NOT='Fn::Not', 17 | EQUALS='Fn::Equals', 18 | GET_ATT='Fn::GetAtt', 19 | } --------------------------------------------------------------------------------