├── .github └── workflows │ ├── ci.yaml │ └── commitlint.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README-Hacking.md ├── README.md ├── build.gradle ├── config └── codenarc │ ├── codenarcRules.groovy │ └── ignored-field-names.txt ├── dataeng ├── jobs │ ├── analytics │ │ ├── AggregateDailyTrackingLogs.groovy │ │ ├── AnalyticsEmailOptin.groovy │ │ ├── AnalyticsExporter.groovy │ │ ├── DBTDocs.groovy │ │ ├── DBTRun.groovy │ │ ├── DatabaseExportCoursewareStudentmodule.groovy │ │ ├── DeployCluster.groovy │ │ ├── EmrCostReporter.groovy │ │ ├── EventExportIncremental.groovy │ │ ├── EventExportIncrementalLarge.groovy │ │ ├── ExpireVerticaPassword.groovy │ │ ├── JenkinsBackup.groovy │ │ ├── ModelTransfers.groovy │ │ ├── PipelineAcceptanceTestManual.groovy │ │ ├── PipelineAcceptanceTestMaster.groovy │ │ ├── PrefectFlowsDeployment.groovy │ │ ├── ReadReplicaExportToS3.groovy │ │ ├── RetirementJobEdxTriggers.groovy │ │ ├── RetirementJobs.groovy │ │ ├── SnowflakeCollectMetrics.groovy │ │ ├── SnowflakeExpirePasswords.groovy │ │ ├── SnowflakePublicGrantsCleaner.groovy │ │ ├── SnowflakeRefreshSnowpipe.groovy │ │ ├── SnowflakeReplicaImportFromS3.groovy │ │ ├── SnowflakeSchemaBuilder.groovy │ │ ├── SnowflakeUserRetirementStatusCleanup.groovy │ │ ├── TerminateCluster.groovy │ │ ├── UpdateUsers.groovy │ │ ├── WarehouseTransforms.groovy │ │ ├── WarehouseTransformsCI.groovy │ │ ├── WarehouseTransformsCIManual.groovy │ │ └── WarehouseTransformsCIMasterMerges.groovy │ ├── createJobs.groovy │ └── createJobsNew.groovy └── resources │ ├── aggregate-daily-tracking-logs.sh │ ├── database-export-courseware-studentmodule.sh │ ├── datadog_job_end.sh │ ├── datadog_job_start.sh │ ├── dbt-docs.sh │ ├── dbt-run.sh │ ├── deploy-cluster.sh │ ├── email-optin-worker.sh │ ├── emr-cost-reporter.sh │ ├── event-export-incremental.sh │ ├── expire-vertica-password.sh │ ├── jenkins-backup.sh │ ├── model-transfers.sh │ ├── opsgenie-enable-heartbeat.sh │ ├── org-exporter-worker.sh │ ├── prefect-flows-deployment-identify.sh │ ├── prefect-flows-deployment.sh │ ├── read-replica-export.sh │ ├── remote-config.sh │ ├── retirement-partner-report-cleanup.sh │ ├── retirement-partner-reporter.sh │ ├── run-course-exporter.sh │ ├── run-pipeline-acceptance-test.sh │ ├── secrets-manager-setup.sh │ ├── secrets-manager.sh │ ├── set_build_status.groovy │ ├── setup-exporter-email-optin.sh │ ├── setup-exporter.sh │ ├── setup-platform-venv-legacy.sh │ ├── setup-platform-venv-py3.sh │ ├── setup-platform-venv.sh │ ├── snowflake-collect-metrics.sh │ ├── snowflake-expire-individual-password.sh │ ├── snowflake-expire-passwords.sh │ ├── snowflake-public-grants-cleaner.sh │ ├── snowflake-refresh-snowpipe.sh │ ├── snowflake-replica-import.sh │ ├── snowflake-schema-builder.sh │ ├── snowflake-user-retirement-status-cleanup.sh │ ├── update-users.sh │ ├── user-retirement-bulk-status.sh │ ├── user-retirement-collector.sh │ ├── user-retirement-driver.sh │ ├── warehouse-transforms-ci-dbt.sh │ ├── warehouse-transforms-ci-manual.sh │ ├── warehouse-transforms-ci-master-merges.sh │ ├── warehouse-transforms-ci.sh │ └── warehouse-transforms.sh ├── devops ├── jobs │ ├── AddXqueueToDashboard.groovy │ ├── AppPermissionsFailure.groovy │ ├── AppPermissionsRunner.groovy │ ├── AppPermissionsWatcher.groovy │ ├── AppWatcher.groovy │ ├── BastionAccess.groovy │ ├── CheckASGLifeCycleHooks.groovy │ ├── CheckPrimaryKeys.groovy │ ├── CheckRDSConfigs.groovy │ ├── CheckRetireUsers.groovy │ ├── CheckSesLimits.groovy │ ├── CheckTableSize.groovy │ ├── CloudFlareHitRate.groovy │ ├── CloudFlarePurgeCacheOrigin.groovy │ ├── ClusterInstanceMonitoring.groovy │ ├── ConfigChecker.groovy │ ├── ConfigurationWatcher.groovy │ ├── CreateASGNotifications.groovy │ ├── CreateDataCzar.groovy │ ├── CreatePingdomAlerts.groovy │ ├── CreateSandbox.groovy │ ├── CreateSandboxCI.groovy │ ├── CreateSandboxCNAME.groovy │ ├── CreateSandboxSSHAccess.groovy │ ├── DeleteMergedGitBranches.groovy │ ├── DockerCleanup.groovy │ ├── ExportRDSDeadLocks.groovy │ ├── ExportRDSSlowQueryLogs.groovy │ ├── ImageBuilder.groovy │ ├── Janitor.groovy │ ├── JenkinsHeartbeat.groovy │ ├── ListMysqlProcess.groovy │ ├── MissingRDSAlarms.groovy │ ├── MongoPruner.groovy │ ├── ProspectusJanitor.groovy │ ├── RecoverTracking.groovy │ ├── RemoveDataCzar.groovy │ ├── ReplaceUsernames.groovy │ ├── RunAnsible.groovy │ ├── RunLocalAnsiblePlaybook.groovy │ ├── RunRemoteAnsiblePlaybook.groovy │ ├── SAMLSSLExpirationCheck.groovy │ ├── SSLExpirationCheck.groovy │ ├── SandboxCertRenewal.groovy │ ├── SandboxTermination.groovy │ ├── StaffSuperAccountAudit.groovy │ ├── UpdateAdhocReporting.groovy │ ├── UpdateDataCzar.groovy │ ├── UpdateMastersSandbox.groovy │ └── UserRetirementArchiver.groovy └── resources │ ├── add_xqueue_to_dashboard.sh │ ├── app-permission-runner-failure.sh │ ├── app-permission-runner-success.sh │ ├── bastion-access.sh │ ├── build-push-app.sh │ ├── check-lifecycle-hooks.sh │ ├── check-rds-configs.sh │ ├── check-ses-limits.sh │ ├── check_primary_keys.sh │ ├── check_retire_users.sh │ ├── cloudflare-hit-rate.sh │ ├── cloudflare_purge_cache.sh │ ├── cluster-instance-monitoring.sh │ ├── create-asg-notifications.sh │ ├── create-data-czar.sh │ ├── create-pingdom-alerts.sh │ ├── create-sandbox-cname.sh │ ├── create-sandbox-ssh-access.sh │ ├── create-sandbox.sh │ ├── delete-merged-git-branches.sh │ ├── export-dead-locks.sh │ ├── export-slow-query-logs.sh │ ├── janitor.sh │ ├── list-mysql-process.sh │ ├── missing-rds-alarms.sh │ ├── mongo-pruner.sh │ ├── recover_tracking_logs.sh │ ├── remove-data-czar.sh │ ├── replace-usernames.sh │ ├── run-ansible.sh │ ├── run-app-permissions.sh │ ├── run-local-ansible-playbook.sh │ ├── run-remote-ansible-playbook.sh │ ├── saml-ssl-expiration-check.sh │ ├── sandbox-cert-renew.sh │ ├── sandbox-termination.sh │ ├── ssl-expiration-check.sh │ ├── staff-super-account-audit.sh │ ├── syntax-check-config.sh │ ├── table-size-monitoring.sh │ ├── trigger-builds.sh │ ├── update-adhoc-reporting.sh │ ├── update-data-czar.sh │ ├── update-masters-sandbox.sh │ └── user-retirement-archiver.sh ├── docker-compose.yml ├── experiments ├── jobs │ └── BackUpOptimizely.groovy └── resources │ └── back_up_optimizely.sh ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── openedx.yaml ├── requirements ├── base.in ├── base.txt ├── common_constraints.txt ├── constraints.txt ├── pip-tools.in ├── pip-tools.txt ├── pip.in └── pip.txt ├── sample └── jobs │ ├── sampleJob.groovy │ └── sampleProgrammaticBuild.groovy └── src ├── main └── groovy │ └── org │ └── edx │ └── jenkins │ └── dsl │ ├── AnalyticsConstants.groovy │ ├── DevopsConstants.groovy │ ├── JenkinsPublicConstants.groovy │ └── UserRetirementConstants.groovy └── test ├── groovy ├── devops │ └── UserRetirementArchiverJobSpec.groovy └── sample │ └── SampleJobSpec.groovy └── resources └── devops └── seeders └── UserRetirementArchiverTestSeeder.groovy /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ 'master' ] 6 | pull_request: 7 | 8 | jobs: 9 | run_gradle_check: 10 | runs-on: ubuntu-latest 11 | name: Gradle Check 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-java@v4 15 | with: 16 | distribution: 'temurin' 17 | java-version: '11' 18 | cache: 'gradle' 19 | - run: ./gradlew check --stacktrace --info 20 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | # Run commitlint on the commit messages in a pull request. 2 | 3 | name: Lint Commit Messages 4 | 5 | on: 6 | - pull_request 7 | 8 | jobs: 9 | commitlint: 10 | uses: openedx/.github/.github/workflows/commitlint.yml@master 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.pyc 3 | 4 | # Mobile Tools for Java (J2ME) 5 | .mtj.tmp/ 6 | 7 | # Package Files # 8 | *.war 9 | *.ear 10 | 11 | # Ignore libs 12 | lib/* 13 | build/* 14 | .gradle/* 15 | 16 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 17 | hs_err_pid* 18 | 19 | *.DS_Store 20 | 21 | # PyCharm files 22 | *.idea/ 23 | -------------------------------------------------------------------------------- /config/codenarc/ignored-field-names.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edx/jenkins-job-dsl/af8587323226bf3b2a6f08dc3a272ffa4eed659f/config/codenarc/ignored-field-names.txt -------------------------------------------------------------------------------- /dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.to_date_interval_parameter 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_parameters 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 9 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 10 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_multiscm 11 | 12 | class AggregateDailyTrackingLogs { 13 | public static def job = { dslFactory, allVars -> 14 | allVars.get('ENVIRONMENTS').each { environment, env_config -> 15 | dslFactory.job("aggregate-daily-tracking-logs-$environment") { 16 | disabled(env_config.get('DISABLED', false)) 17 | logRotator common_log_rotator(allVars, env_config) 18 | parameters to_date_interval_parameter(env_config) 19 | parameters common_parameters(allVars, env_config) 20 | parameters { 21 | stringParam('SOURCE_BUCKET_PATH', env_config.get('SOURCE_BUCKET_PATH')) 22 | stringParam('DEST_BUCKET_PATH', env_config.get('DEST_BUCKET_PATH')) 23 | stringParam('TARGET_SIZE', env_config.get('TARGET_SIZE')) 24 | } 25 | multiscm common_multiscm(allVars) 26 | triggers common_triggers(allVars, env_config) 27 | wrappers common_wrappers(allVars) 28 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 29 | steps { 30 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 31 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/aggregate-daily-tracking-logs.sh')) 32 | } 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/DBTDocs.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 8 | 9 | class DBTDocs{ 10 | public static def job = { dslFactory, allVars -> 11 | dslFactory.job("dbt-docs"){ 12 | logRotator common_log_rotator(allVars) 13 | parameters { 14 | stringParam('WAREHOUSE_TRANSFORMS_URL', allVars.get('WAREHOUSE_TRANSFORMS_URL'), 'URL for the warehouse-transforms repository.') 15 | stringParam('WAREHOUSE_TRANSFORMS_BRANCH', allVars.get('WAREHOUSE_TRANSFORMS_BRANCH'), 'Branch of warehouse-transforms repository to use.') 16 | stringParam('DBT_TARGET', allVars.get('DBT_TARGET'), 'DBT target from profiles.yml in analytics-secure.') 17 | stringParam('DBT_PROFILE', allVars.get('DBT_PROFILE'), 'DBT profile from profiles.yml in analytics-secure.') 18 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 19 | stringParam('BUILD_STATUS') 20 | } 21 | multiscm { 22 | git { 23 | remote { 24 | url('$WAREHOUSE_TRANSFORMS_URL') 25 | branch('$WAREHOUSE_TRANSFORMS_BRANCH') 26 | credentials('1') 27 | } 28 | extensions { 29 | relativeTargetDirectory('warehouse-transforms') 30 | pruneBranches() 31 | cleanAfterCheckout() 32 | } 33 | } 34 | } 35 | triggers common_triggers(allVars) 36 | wrappers { 37 | colorizeOutput('xterm') 38 | } 39 | wrappers common_wrappers(allVars) 40 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 41 | steps { 42 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 43 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh')) 44 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/dbt-docs.sh')) 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_multiscm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_parameters 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.opsgenie_heartbeat_publisher 9 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 10 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 11 | 12 | class DatabaseExportCoursewareStudentmodule { 13 | public static def job = { dslFactory, allVars -> 14 | allVars.get('ENVIRONMENTS').each { environment, env_config -> 15 | dslFactory.job("database-export-courseware-studentmodule-$environment") { 16 | disabled(env_config.get('DISABLED', false)) 17 | logRotator common_log_rotator(allVars, env_config) 18 | multiscm common_multiscm(allVars) 19 | triggers common_triggers(allVars, env_config) 20 | parameters { 21 | stringParam('BASE_OUTPUT_URL', env_config.get('BASE_OUTPUT_URL', allVars.get('BASE_OUTPUT_URL')), '') 22 | stringParam('OUTPUT_DIR', env_config.get('OUTPUT_DIR', allVars.get('OUTPUT_DIR')), '') 23 | stringParam('OUTPUT_SUFFIX', env_config.get('OUTPUT_SUFFIX', allVars.get('OUTPUT_SUFFIX')), '') 24 | stringParam('CREDENTIALS', env_config.get('CREDENTIALS', allVars.get('CREDENTIALS')), '') 25 | } 26 | parameters common_parameters(allVars, env_config) 27 | environmentVariables { 28 | env('OPSGENIE_HEARTBEAT_NAME', env_config.get('OPSGENIE_HEARTBEAT_NAME')) 29 | env('OPSGENIE_HEARTBEAT_DURATION_NUM', env_config.get('OPSGENIE_HEARTBEAT_DURATION_NUM')) 30 | env('OPSGENIE_HEARTBEAT_DURATION_UNIT', env_config.get('OPSGENIE_HEARTBEAT_DURATION_UNIT')) 31 | } 32 | wrappers common_wrappers(allVars) 33 | wrappers { 34 | credentialsBinding { 35 | string('OPSGENIE_HEARTBEAT_CONFIG_KEY', 'opsgenie_heartbeat_config_key') 36 | } 37 | } 38 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 39 | publishers opsgenie_heartbeat_publisher(allVars) 40 | steps { 41 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 42 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/opsgenie-enable-heartbeat.sh')) 43 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/database-export-courseware-studentmodule.sh')) 44 | } 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/DeployCluster.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.analytics_configuration_scm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.emr_cluster_parameters 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 6 | 7 | class DeployCluster { 8 | public static def job = { dslFactory, allVars -> 9 | dslFactory.job("deploy-cluster") { 10 | logRotator common_log_rotator(allVars) 11 | parameters { 12 | stringParam('CONFIG_REPO', 'git@github.com:edx/edx-analytics-configuration.git', '') 13 | stringParam('CONFIG_BRANCH', '$ANALYTICS_CONFIGURATION_RELEASE', 'e.g. tagname or origin/branchname, or $ANALYTICS_CONFIGURATION_RELEASE') 14 | } 15 | parameters emr_cluster_parameters(allVars) 16 | multiscm analytics_configuration_scm(allVars) 17 | wrappers common_wrappers(allVars) 18 | steps { 19 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/deploy-cluster.sh')) 20 | } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/EmrCostReporter.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | 6 | class EmrCostReporter { 7 | public static def job = { dslFactory, allVars -> 8 | dslFactory.job('emr-cost-reporter') { 9 | parameters { 10 | stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '') 11 | stringParam('TOOLS_BRANCH', 'origin/master', '') 12 | stringParam('THRESHOLD', allVars.get('THRESHOLD'), 'Number of dollars "budgeted". Going over this threshold will cause the job to fail.') 13 | stringParam('WEEKLY_JOB_THRESHOLD_ADJUSTMENT', allVars.get('WEEKLY_JOB_THRESHOLD_ADJUSTMENT'), '') 14 | stringParam('GRAPHITE_HOST', allVars.get('GRAPHITE_HOST'), '') 15 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 16 | } 17 | logRotator common_log_rotator(allVars) 18 | multiscm { 19 | git { 20 | remote { 21 | url('$TOOLS_REPO') 22 | branch('$TOOLS_BRANCH') 23 | credentials('1') 24 | } 25 | } 26 | } 27 | triggers common_triggers(allVars) 28 | wrappers { 29 | timestamps() 30 | } 31 | publishers common_publishers(allVars) 32 | steps { 33 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/emr-cost-reporter.sh')) 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/EventExportIncremental.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_multiscm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.data_czar_keys_scm 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_parameters 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.from_date_interval_parameter 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.to_date_interval_parameter 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 9 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 10 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 11 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 12 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 13 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 14 | 15 | class EventExportIncremental { 16 | public static def job = { dslFactory, allVars -> 17 | allVars.get('ENVIRONMENTS').each { environment, env_config -> 18 | dslFactory.job("event-export-incremental-$environment") { 19 | disabled(env_config.get('DISABLED', false)) 20 | logRotator common_log_rotator(allVars, env_config) 21 | parameters common_parameters(allVars, env_config) 22 | parameters from_date_interval_parameter(allVars) 23 | parameters to_date_interval_parameter(allVars) 24 | parameters { 25 | stringParam('SOURCE', env_config.get('EVENT_LOGS_SOURCE'), '') 26 | stringParam('OUTPUT_ROOT', allVars.get('OUTPUT_ROOT')) 27 | stringParam('EXPORTER_CONFIG', 'config.yaml', 'Exporter configuration relative to analytics-secure/analytics-exporter') 28 | stringParam('ONLY_ORGS', '', "i.e. --org-id [\\\"FooX\\\",\\\"BarX\\\"]") 29 | stringParam('DATA_CZAR_KEYS_BRANCH', 'master', '') 30 | stringParam('ENVIRONMENT', env_config.get('ENVIRONMENT_SUBDIRECTORY'), '') 31 | } 32 | multiscm common_multiscm(allVars) >> secure_scm(allVars) >> data_czar_keys_scm(allVars) 33 | 34 | triggers common_triggers(allVars, env_config) 35 | wrappers common_wrappers(allVars) 36 | publishers { 37 | postBuildTask { 38 | task('org with Errors=', 'exit 1', true) 39 | } 40 | // Mark the build as failed if a GPG key is near expiry. 41 | textFinder( 42 | "Keys expiring", // regularExpression 43 | '', // fileSet 44 | true, // alsoCheckConsoleOutput 45 | false, // succeedIfFound 46 | false, // unstableIfFound 47 | ) 48 | } 49 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 50 | steps { 51 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 52 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/event-export-incremental.sh')) 53 | } 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/EventExportIncrementalLarge.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_multiscm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.data_czar_keys_scm 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_parameters 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.from_date_interval_parameter 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.to_date_interval_parameter 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 9 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 10 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 11 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 12 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization 13 | 14 | class EventExportIncrementalLarge { 15 | public static def job = { dslFactory, allVars -> 16 | allVars.get('ENVIRONMENTS').each { environment, env_config -> 17 | dslFactory.job("event-export-incremental-large-$environment") { 18 | disabled(env_config.get('DISABLED', false)) 19 | authorization common_authorization(allVars) 20 | logRotator common_log_rotator(allVars, env_config) 21 | parameters common_parameters(allVars, env_config) 22 | parameters from_date_interval_parameter(allVars) 23 | parameters to_date_interval_parameter(allVars) 24 | parameters { 25 | stringParam('SOURCE', env_config.get('EVENT_LOGS_SOURCE'), '') 26 | stringParam('OUTPUT_ROOT', allVars.get('OUTPUT_ROOT')) 27 | stringParam('EXPORTER_CONFIG', allVars.get('EXPORTER_CONFIG'), 'Exporter configuration relative to analytics-secure/analytics-exporter') 28 | stringParam('ONLY_ORGS', allVars.get('ONLY_ORGS'), "i.e. --org-id [\\\"FooX\\\",\\\"BarX\\\"]") 29 | stringParam('DATA_CZAR_KEYS_BRANCH', allVars.get('DATA_CZAR_KEYS_BRANCH'), '') 30 | stringParam('ENVIRONMENT', env_config.get('ENVIRONMENT'), '') 31 | } 32 | multiscm common_multiscm(allVars) >> secure_scm(allVars) >> data_czar_keys_scm(allVars) 33 | 34 | triggers common_triggers(allVars, env_config) 35 | wrappers common_wrappers(allVars) 36 | publishers common_publishers(allVars) 37 | steps { 38 | shell(dslFactory.readFileFromWorkspace("dataeng/resources/event-export-incremental.sh")) 39 | } 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/ExpireVerticaPassword.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 6 | 7 | 8 | class ExpireVerticaPassword { 9 | 10 | public static def job = { dslFactory, allVars -> 11 | 12 | dslFactory.job('expire-vertica-password') { 13 | logRotator common_log_rotator(allVars) 14 | parameters { 15 | stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '') 16 | stringParam('TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH', 'origin/master'), 'e.g. tagname or origin/branchname') 17 | stringParam('CREDENTIALS', allVars.get('CREDENTIALS')) 18 | stringParam('EXCLUDE', allVars.get('EXCLUDE')) 19 | stringParam('MAPPING', allVars.get('MAPPING')) 20 | stringParam('NOTIFY', '$PAGER_NOTIFY', 'Space separated list of emails to send notifications to.') 21 | stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') 22 | } 23 | multiscm { 24 | git { 25 | remote { 26 | url('$TOOLS_REPO') 27 | branch('$TOOLS_BRANCH') 28 | credentials('1') 29 | } 30 | extensions { 31 | relativeTargetDirectory('analytics-tools') 32 | pruneBranches() 33 | cleanAfterCheckout() 34 | } 35 | } 36 | } 37 | triggers common_triggers(allVars) 38 | wrappers { 39 | timestamps() 40 | } 41 | publishers common_publishers(allVars) 42 | steps { 43 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/expire-vertica-password.sh')) 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/JenkinsBackup.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 7 | 8 | class JenkinsBackup { 9 | public static def job = { dslFactory, allVars -> 10 | dslFactory.job('jenkins-backup') { 11 | parameters { 12 | stringParam('S3_BACKUP_BUCKET', allVars.get('S3_BACKUP_BUCKET')) 13 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 14 | stringParam('PYTHON_VENV_VERSION', 'python3.8', 'Python virtual environment version to used.') 15 | stringParam('BUILD_STATUS') 16 | } 17 | wrappers common_wrappers(allVars) 18 | triggers common_triggers(allVars) 19 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 20 | steps { 21 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 22 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/jenkins-backup.sh')) 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/PipelineAcceptanceTestManual.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters 8 | 9 | 10 | class PipelineAcceptanceTestManual { 11 | 12 | public static def job = { dslFactory, allVars -> 13 | dslFactory.job("edx-analytics-pipeline-acceptance-test-manual"){ 14 | 15 | description( 16 | 'This job is used for manually running acceptance tests for the edx-analytics pipeline, ' + 17 | 'in the absence of automated CI on that repository.' 18 | ) 19 | authorization common_authorization(allVars) 20 | parameters { 21 | stringParam('TASKS_REPO', allVars.get('TASKS_REPO'), '') 22 | stringParam('TASKS_BRANCH', allVars.get('TASKS_BRANCH'), '') 23 | stringParam('EXPORTER_BRANCH', allVars.get('EXPORTER_BRANCH'), '') 24 | stringParam('EXPORTER_BUCKET_PATH', allVars.get('EXPORTER_BUCKET_PATH'), '') 25 | textParam('ACCEPTANCE_TEST_CONFIG', allVars.get('ACCEPTANCE_TEST_CONFIG'), '') 26 | stringParam( 27 | 'ONLY_TESTS', allVars.get('ONLY_TESTS'), 28 | 'Only run the tests specified in this parameter (using nosetest path formatting, i.e. ' + 29 | 'path.to.some.file). If left blank, the entire suite will be run.' 30 | ) 31 | stringParam('DISABLE_RESET_STATE', allVars.get('DISABLE_RESET_STATE'), '') 32 | stringParam('MAX_DIFF', allVars.get('MAX_DIFF'), '') 33 | } 34 | 35 | parameters secure_scm_parameters(allVars) 36 | multiscm secure_scm(allVars) << { 37 | git { 38 | remote { 39 | url('git@github.com:edx/edx-analytics-exporter.git') 40 | branch('$EXPORTER_BRANCH') 41 | } 42 | extensions { 43 | pruneBranches() 44 | relativeTargetDirectory('analytics-exporter') 45 | } 46 | } 47 | git { 48 | remote { 49 | url('$TASKS_REPO') 50 | branch('$TASKS_BRANCH') 51 | } 52 | extensions { 53 | relativeTargetDirectory('analytics-tasks') 54 | pruneBranches() 55 | cleanAfterCheckout() 56 | } 57 | } 58 | } 59 | 60 | concurrentBuild(true) 61 | wrappers common_wrappers(allVars) 62 | 63 | steps { 64 | shell(dslFactory.readFileFromWorkspace("dataeng/resources/run-pipeline-acceptance-test.sh")) 65 | } 66 | 67 | publishers { 68 | archiveArtifacts { 69 | pattern('build') 70 | allowEmpty() 71 | } 72 | archiveJunit('analytics-tasks/nosetests.xml') { 73 | allowEmptyResults(true) 74 | } 75 | } 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/PipelineAcceptanceTestMaster.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters 9 | 10 | 11 | class PipelineAcceptanceTestMaster { 12 | 13 | public static def job = { dslFactory, allVars -> 14 | dslFactory.job("edx-analytics-pipeline-acceptance-test-master"){ 15 | disabled() 16 | description( 17 | 'This job is used to run acceptance tests on the master branch of ' + 18 | 'the edx-analytics-pipeline repo. It regularly polls that branch ' + 19 | 'and will trigger a new build when a new commit is detected' 20 | ) 21 | parameters { 22 | textParam('ACCEPTANCE_TEST_CONFIG', allVars.get('ACCEPTANCE_TEST_CONFIG'), '') 23 | stringParam('NOTIFY', '$PAGER_NOTIFY', 'Space separated list of emails to send notifications to.') 24 | } 25 | 26 | parameters secure_scm_parameters(allVars) 27 | multiscm secure_scm(allVars) << { 28 | git { 29 | remote { 30 | url('git@github.com:edx/edx-analytics-pipeline.git') 31 | branch('origin/master') 32 | } 33 | extensions { 34 | relativeTargetDirectory('analytics-tasks') 35 | perBuildTag() 36 | } 37 | } 38 | git { 39 | remote { 40 | url('git@github.com:edx/edx-analytics-exporter.git') 41 | branch('origin/master') 42 | } 43 | extensions { 44 | relativeTargetDirectory('analytics-exporter') 45 | cleanAfterCheckout() 46 | pruneBranches() 47 | perBuildTag() 48 | } 49 | } 50 | } 51 | 52 | environmentVariables { 53 | env('EXPORTER_BUCKET_PATH', allVars.get('EXPORTER_BUCKET_PATH')) 54 | } 55 | concurrentBuild(true) 56 | wrappers common_wrappers(allVars) 57 | 58 | triggers { 59 | pollSCM { 60 | scmpoll_spec(allVars.get('JOB_FREQUENCY')) 61 | } 62 | } 63 | 64 | steps { 65 | shell(dslFactory.readFileFromWorkspace("dataeng/resources/run-pipeline-acceptance-test.sh")) 66 | } 67 | 68 | publishers common_publishers(allVars) << { 69 | archiveArtifacts { 70 | pattern('build') 71 | allowEmpty() 72 | } 73 | archiveJunit('analytics-tasks/nosetests.xml') { 74 | allowEmptyResults(true) 75 | } 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/RetirementJobEdxTriggers.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization 9 | 10 | class RetirementJobEdxTriggers{ 11 | public static def job = { dslFactory, allVars -> 12 | allVars.get('ENVIRONMENTS').each { environment, env_config -> 13 | dslFactory.job("$environment"){ 14 | // Creates user_retirement_trigger_ and retirement_partner_report_trigger_ jobs 15 | // This defines the job which triggers the collector and reporter job for a given environment. 16 | description("Scheduled trigger of the " + env_config.get('DOWNSTREAM_JOB_NAME') + " job for the " + env_config.get('DOWNSTREAM_JOB_NAME') + " environment") 17 | 18 | disabled(env_config.get('DISABLED')) 19 | 20 | authorization common_authorization(env_config) 21 | // Disallow this job to have simultaneous instances building at the same 22 | // time. This might help prevent race conditions related to triggering 23 | // multiple retirement driver jobs against the same user. 24 | concurrentBuild(false) 25 | triggers common_triggers(allVars, env_config) 26 | 27 | // keep jobs around for 30 days 28 | // allVars contains the value for DAYS_TO_KEEP_BUILD which will be used inside AnalyticsConstants.common_log_rotator 29 | logRotator common_log_rotator(allVars) 30 | 31 | wrappers { 32 | buildName('#${BUILD_NUMBER}') 33 | timestamps() 34 | colorizeOutput('xterm') 35 | } 36 | wrappers common_wrappers(allVars) 37 | parameters secure_scm_parameters(allVars) 38 | steps { 39 | downstreamParameterized { 40 | trigger(env_config.get('DOWNSTREAM_JOB_NAME')) { 41 | // This section causes the build to block on completion of downstream builds. 42 | block { 43 | // Mark this build step as FAILURE if at least one of the downstream builds were marked FAILED. 44 | buildStepFailure('FAILURE') 45 | // Mark this entire build as FAILURE if at least one of the downstream builds were marked FAILED. 46 | failure('FAILURE') 47 | // Mark this entire build as UNSTABLE if at least one of the downstream builds were marked UNSTABLE. 48 | unstable('UNSTABLE') 49 | } 50 | parameters { 51 | predefinedProp('ENVIRONMENT', env_config.get('ENVIRONMENTS_DEPLOYMENT')) 52 | } 53 | } 54 | } 55 | } 56 | } 57 | } 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | 6 | 7 | class SnowflakeCollectMetrics { 8 | 9 | public static def job = { dslFactory, allVars -> 10 | 11 | Map SnowflakeWarehouseCreditConfig = [ 12 | NAME: 'snowflake-collect-credit-metrics', 13 | CRON: '0 * * * *' 14 | ] 15 | List jobConfigs = [ 16 | SnowflakeWarehouseCreditConfig 17 | ] 18 | 19 | jobConfigs.each { jobConfig -> 20 | 21 | dslFactory.job(jobConfig['NAME']){ 22 | 23 | logRotator common_log_rotator(allVars) 24 | parameters { 25 | stringParam('ANALYTICS_TOOLS_URL', allVars.get('ANALYTICS_TOOLS_URL'), 'URL for the analytics tools repo.') 26 | stringParam('ANALYTICS_TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH'), , 'Branch of analytics tools repo to use.') 27 | stringParam('NOTIFY', '$PAGER_NOTIFY', 'Space separated list of emails to send notifications to.') 28 | stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') 29 | } 30 | environmentVariables { 31 | env('SNOWFLAKE_USER', 'SNOWFLAKE_TASK_AUTOMATION_USER') 32 | env('SNOWFLAKE_ACCOUNT', 'edx.us-east-1') 33 | env('SNOWFLAKE_WAREHOUSE', 'LOADING') 34 | env('METRIC_NAME', jobConfig['NAME']) 35 | } 36 | multiscm { 37 | git { 38 | remote { 39 | url('$ANALYTICS_TOOLS_URL') 40 | branch('$ANALYTICS_TOOLS_BRANCH') 41 | credentials('1') 42 | } 43 | extensions { 44 | relativeTargetDirectory('analytics-tools') 45 | pruneBranches() 46 | cleanAfterCheckout() 47 | } 48 | } 49 | } 50 | triggers { 51 | cron(jobConfig['CRON']) 52 | } 53 | wrappers { 54 | timestamps() 55 | } 56 | publishers common_publishers(allVars) 57 | steps { 58 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-collect-metrics.sh')) 59 | } 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 6 | 7 | 8 | class SnowflakeExpirePasswords { 9 | 10 | public static def job = { dslFactory, allVars -> 11 | 12 | Map SnowflakeExpirePasswordsQuarterlyConfig = [ 13 | NAME: 'expire-snowflake-passwords-quarterly', 14 | MANUAL: false, 15 | SCRIPT_TO_RUN: 'dataeng/resources/snowflake-expire-passwords.sh' 16 | ] 17 | Map SnowflakeExpirePasswordsManuallyConfig = [ 18 | NAME: 'expire-snowflake-passwords-manually', 19 | MANUAL: true, 20 | SCRIPT_TO_RUN: 'dataeng/resources/snowflake-expire-individual-password.sh' 21 | ] 22 | List jobConfigs = [ 23 | SnowflakeExpirePasswordsQuarterlyConfig, 24 | SnowflakeExpirePasswordsManuallyConfig 25 | ] 26 | 27 | jobConfigs.each { jobConfig -> 28 | 29 | dslFactory.job(jobConfig['NAME']) { 30 | 31 | logRotator common_log_rotator(allVars) 32 | parameters { 33 | stringParam('ANALYTICS_TOOLS_URL', allVars.get('ANALYTICS_TOOLS_URL'), 'URL for the analytics tools repo.') 34 | stringParam('ANALYTICS_TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH'), 'Branch of analytics tools repo to use.') 35 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 36 | if (jobConfig['MANUAL']) { 37 | stringParam('USER_TO_EXPIRE', '', 'Snowflake user to mark for password expiration.') 38 | } 39 | } 40 | environmentVariables { 41 | env('USER', allVars.get('USER')) 42 | env('ACCOUNT', allVars.get('ACCOUNT')) 43 | } 44 | logRotator common_log_rotator(allVars) 45 | multiscm { 46 | git { 47 | remote { 48 | url('$ANALYTICS_TOOLS_URL') 49 | branch('$ANALYTICS_TOOLS_BRANCH') 50 | credentials('1') 51 | } 52 | extensions { 53 | relativeTargetDirectory('analytics-tools') 54 | pruneBranches() 55 | cleanAfterCheckout() 56 | } 57 | } 58 | } 59 | if (!jobConfig['MANUAL']) { 60 | triggers common_triggers(allVars) 61 | } 62 | wrappers { 63 | timestamps() 64 | } 65 | publishers common_publishers(allVars) 66 | steps { 67 | shell(dslFactory.readFileFromWorkspace(jobConfig['SCRIPT_TO_RUN'])) 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_multiscm 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_parameters 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 7 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 8 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 9 | 10 | 11 | 12 | class SnowflakePublicGrantsCleaner { 13 | public static def job = { dslFactory, allVars -> 14 | dslFactory.job("snowflake-public-grants-cleaner") { 15 | logRotator common_log_rotator(allVars) 16 | parameters { 17 | stringParam('ANALYTICS_TOOLS_URL', allVars.get('ANALYTICS_TOOLS_URL'), 'URL for the analytics tools repo.') 18 | stringParam('ANALYTICS_TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH'), 'Branch of analtyics tools repo to use.') 19 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 20 | stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') 21 | } 22 | environmentVariables { 23 | env('KEY_PATH', allVars.get('KEY_PATH')) 24 | env('PASSPHRASE_PATH', allVars.get('PASSPHRASE_PATH')) 25 | env('USER', allVars.get('USER')) 26 | env('ACCOUNT', allVars.get('ACCOUNT')) 27 | } 28 | multiscm { 29 | git { 30 | remote { 31 | url('$ANALYTICS_TOOLS_URL') 32 | branch('$ANALYTICS_TOOLS_BRANCH') 33 | credentials('1') 34 | } 35 | extensions { 36 | relativeTargetDirectory('analytics-tools') 37 | pruneBranches() 38 | cleanAfterCheckout() 39 | } 40 | } 41 | } 42 | triggers common_triggers(allVars) 43 | wrappers { 44 | timestamps() 45 | } 46 | publishers common_publishers(allVars) 47 | steps { 48 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-public-grants-cleaner.sh')) 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 7 | 8 | class SnowflakeSchemaBuilder { 9 | public static def job = { dslFactory, allVars -> 10 | dslFactory.job('snowflake-schema-builder') { 11 | logRotator common_log_rotator(allVars) 12 | parameters { 13 | stringParam('WAREHOUSE_TRANSFORMS_URL', allVars.get('WAREHOUSE_TRANSFORMS_URL'), 'URL for the Warehouse Transforms Repo.') 14 | stringParam('WAREHOUSE_TRANSFORMS_BRANCH', allVars.get('WAREHOUSE_TRANSFORMS_BRANCH'), 'Branch of Warehouse Transforms to use.') 15 | stringParam('SOURCE_PROJECT', allVars.get('SOURCE_PROJECT'), 'The dbt project where the models will be generated and run, relative to the "projects" directory.') 16 | stringParam('DESTINATION_PROJECT', allVars.get('DESTINATION_PROJECT'), 'The dbt project that will use the generated sources, relative to the SOURCE_PROJECT.') 17 | stringParam('DBT_PROFILE', allVars.get('DBT_PROFILE'), 'dbt profile from analytics-secure to work on.') 18 | stringParam('DBT_TARGET', allVars.get('DBT_TARGET'), 'dbt target from analytics-secure to work on.') 19 | stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') 20 | stringParam('BUILD_STATUS') 21 | } 22 | logRotator common_log_rotator(allVars) 23 | multiscm { 24 | git { 25 | remote { 26 | url('$WAREHOUSE_TRANSFORMS_URL') 27 | branch('$WAREHOUSE_TRANSFORMS_BRANCH') 28 | credentials('1') 29 | } 30 | extensions { 31 | relativeTargetDirectory('warehouse-transforms') 32 | pruneBranches() 33 | cleanAfterCheckout() 34 | } 35 | } 36 | } 37 | triggers common_triggers(allVars) 38 | wrappers { 39 | timestamps() 40 | credentialsBinding { 41 | string('GITHUB_TOKEN', 'GHPRB_BOT_TOKEN'); 42 | } 43 | } 44 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << common_publishers(allVars) 45 | steps { 46 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/datadog_job_start.sh')) 47 | // This will create python 3.8 venv inside shell script instead of using shiningpanda 48 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh')) 49 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-schema-builder.sh')) 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/TerminateCluster.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.analytics_configuration_scm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 5 | 6 | class TerminateCluster { 7 | public static def job = { dslFactory, allVars -> 8 | dslFactory.job("terminate-cluster") { 9 | logRotator common_log_rotator(allVars) 10 | parameters { 11 | stringParam('CLUSTER_NAME', '', 'Name of the EMR cluster to terminate.') 12 | stringParam('CONFIG_REPO', 'git@github.com:edx/edx-analytics-configuration.git', '') 13 | stringParam('CONFIG_BRANCH', '$ANALYTICS_CONFIGURATION_RELEASE', 'e.g. tagname or origin/branchname, or $ANALYTICS_CONFIGURATION_RELEASE') 14 | } 15 | multiscm analytics_configuration_scm(allVars) 16 | steps { 17 | virtualenv { 18 | nature("shell") 19 | command('cd $WORKSPACE/analytics-configuration && EXTRA_VARS="name=$CLUSTER_NAME" make terminate.emr') 20 | } 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dataeng/jobs/analytics/UpdateUsers.groovy: -------------------------------------------------------------------------------- 1 | package analytics 2 | import static org.edx.jenkins.dsl.AnalyticsConstants.analytics_configuration_scm 3 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator 4 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers 5 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_groovy_postbuild 6 | import static org.edx.jenkins.dsl.AnalyticsConstants.common_datadog_build_end 7 | 8 | class UpdateUsers { 9 | public static def job = { dslFactory, allVars -> 10 | dslFactory.job("update-users") { 11 | logRotator common_log_rotator(allVars) 12 | parameters { 13 | textParam('EXTRA_VARS', allVars.get('EXTRA_VARS'), '') 14 | stringParam('REMOTE_USER', allVars.get('REMOTE_USER'), 'User which runs the analytics task on the EMR cluster.') 15 | stringParam('CONFIG_REPO', 'git@github.com:edx/edx-analytics-configuration.git', '') 16 | stringParam('CONFIG_BRANCH', allVars.get('BRANCH'), 'e.g. tagname or origin/branchname, or $ANALYTICS_CONFIGURATION_RELEASE') 17 | stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') 18 | stringParam('BUILD_STATUS') 19 | } 20 | multiscm analytics_configuration_scm(allVars) 21 | wrappers common_wrappers(allVars) 22 | steps { 23 | shell(dslFactory.readFileFromWorkspace('dataeng/resources/update-users.sh')) 24 | } 25 | publishers common_datadog_build_end(dslFactory, allVars) << common_groovy_postbuild(dslFactory, allVars) << { 26 | postBuildTask { 27 | task('skipping: no hosts matched', 'exit 1', true, true) 28 | } 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dataeng/resources/aggregate-daily-tracking-logs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | group_by_date=$(date +%Y%m%d -d "$TO_DATE") 4 | group_by_pattern=".*(tracking\.log-$group_by_date).*(.gz)" 5 | manifest_file_name="manifest-$group_by_date.gz" 6 | previous_manifest_path="$DEST_BUCKET_PATH/$manifest_file_name" 7 | 8 | aws s3 ls $previous_manifest_path 9 | 10 | if [ $? == 0 ]; then 11 | SHELL_COMMAND="s3-dist-cp --src=$SOURCE_BUCKET_PATH --dest=$DEST_BUCKET_PATH --groupBy='$group_by_pattern' --targetSize=$TARGET_SIZE --outputManifest=$manifest_file_name --previousManifest=$previous_manifest_path" 12 | else 13 | SHELL_COMMAND="s3-dist-cp --src=$SOURCE_BUCKET_PATH --dest=$DEST_BUCKET_PATH --groupBy='$group_by_pattern' --targetSize=$TARGET_SIZE --outputManifest=$manifest_file_name" 14 | fi 15 | 16 | ${WORKSPACE}/analytics-configuration/automation/run-shell.sh "$SHELL_COMMAND" 17 | -------------------------------------------------------------------------------- /dataeng/resources/database-export-courseware-studentmodule.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | env 4 | 5 | DAY_OF_MONTH=$(date +%d) 6 | OUTPUT_URL=$BASE_OUTPUT_URL/$DAY_OF_MONTH/$OUTPUT_DIR 7 | NUM_CAPACITY=$(( $NUM_TASK_CAPACITY + $ON_DEMAND_CAPACITY )) 8 | NUM_MAPPERS=$(( ($NUM_CAPACITY + 1) * 2 )) 9 | NUM_REDUCE_TASKS=$(( $NUM_CAPACITY + 1 )) 10 | 11 | ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \ 12 | StudentModulePerCourseAfterImportWorkflow --local-scheduler \ 13 | --credentials $CREDENTIALS \ 14 | --dump-root $OUTPUT_URL/raw/ \ 15 | --output-root $OUTPUT_URL/ \ 16 | --delete-output-root \ 17 | --output-suffix $OUTPUT_SUFFIX \ 18 | --num-mappers $NUM_MAPPERS \ 19 | --n-reduce-tasks $NUM_REDUCE_TASKS \ 20 | --verbose \ 21 | $EXTRA_ARGS 22 | 23 | -------------------------------------------------------------------------------- /dataeng/resources/datadog_job_end.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +x 4 | API_KEY=$(aws secretsmanager get-secret-value --secret-id "analytics-secure/datadog" --region "us-east-1" --output text --query 'SecretString' | jq -r '.api_key') 5 | 6 | END_TIME="$(date +"%Y-%m-%d %H:%M:%S")" 7 | 8 | RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "https://api.datadoghq.com/api/v1/events" \ 9 | -H "Content-Type: application/json" \ 10 | -H "DD-API-KEY: $API_KEY" \ 11 | -d @- <${THRESHOLD}" | bc)" = "1" ] 26 | then 27 | echo "Actual cost \$${COST} exceeded the adjusted threshold of \$${THRESHOLD}, failing." 28 | exit 1 29 | else 30 | exit 0 31 | fi 32 | -------------------------------------------------------------------------------- /dataeng/resources/event-export-incremental.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #################################################################### 4 | # Exporter configuration 5 | 6 | ROOT=${WORKSPACE}/analytics-secure/analytics-exporter 7 | SECURE_HASH=`GIT_DIR=./analytics-secure/.git git rev-parse HEAD` 8 | EXPORTER_CONFIG_BUCKET=s3://edx-analytics-scratch/exporter/config/$SECURE_HASH 9 | EXPORTER_CONFIG_PATH=${EXPORTER_CONFIG_BUCKET}/${EXPORTER_CONFIG} 10 | GPG_KEYS_PATH=${EXPORTER_CONFIG_BUCKET}/gpg-keys 11 | 12 | #################################################################### 13 | # Upload configuration files to s3 14 | 15 | pip install awscli 16 | 17 | echo ${WORKSPACE}/data-czar-keys/${EXPORTER_CONFIG} 18 | 19 | aws s3 cp ${WORKSPACE}/data-czar-keys/${EXPORTER_CONFIG} ${EXPORTER_CONFIG_PATH} 20 | aws s3 cp --recursive ${WORKSPACE}/data-czar-keys/ ${GPG_KEYS_PATH} 21 | 22 | #################################################################### 23 | # Run exporter task 24 | 25 | if [ -z "$NUM_REDUCE_TASKS" ]; then 26 | NUM_REDUCE_TASKS=$(( $NUM_TASK_CAPACITY + 1 )) 27 | fi 28 | 29 | env | sort 30 | 31 | ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \ 32 | EventExportTask --local-scheduler \ 33 | --interval $(date +%Y-%m-%d -d "$FROM_DATE")-$(date +%Y-%m-%d -d "$TO_DATE") \ 34 | --output-root ${OUTPUT_ROOT} \ 35 | --config ${EXPORTER_CONFIG_PATH} \ 36 | --gpg-key-dir ${GPG_KEYS_PATH} \ 37 | --environment ${ENVIRONMENT} \ 38 | ${ONLY_ORGS} \ 39 | --n-reduce-tasks ${NUM_REDUCE_TASKS} \ 40 | --source "$SOURCE" \ 41 | ${EXTRA_ARGS} 42 | -------------------------------------------------------------------------------- /dataeng/resources/expire-vertica-password.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/vertica 11 | pip install -r requirements.txt 12 | 13 | python expire_user_passwords.py \ 14 | --credentials $CREDENTIALS \ 15 | --exclude $EXCLUDE \ 16 | --mapping $MAPPING 17 | -------------------------------------------------------------------------------- /dataeng/resources/jenkins-backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | env 4 | 5 | # Creating Python virtual env 6 | PYTHON_VENV="python_venv" 7 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 8 | source "${PYTHON_VENV}/bin/activate" 9 | 10 | pip install awscli 11 | 12 | # Delete all files in the workspace 13 | rm -rf * 14 | # Create a directory for the job definitions 15 | mkdir -p jenkins/jobs 16 | # Copy global configuration files into the workspace 17 | cp $JENKINS_HOME/*.xml jenkins/ 18 | # Copy user configuration files into the workspace 19 | #cp -r $JENKINS_HOME/users jenkins/ 20 | # Copy job definitions into the workspace 21 | rsync -am --include='config.xml' --include='*/' --prune-empty-dirs --exclude='*' $JENKINS_HOME/jobs/ jenkins/jobs/ 22 | # Get current UTC date and time with minutes 23 | CURRENT_UTC_TIME=$(TZ=UTC date +%s) 24 | # jenkins backup name 25 | BACKUP_NAME=${NODE_NAME}-build${BUILD_ID}-${CURRENT_UTC_TIME} 26 | # Create an archive from all copied files (since the S3 plugin cannot copy folders recursively) 27 | tar czf $BACKUP_NAME.tar.gz jenkins/ 28 | # Remove the directory so only the archive gets copied to S3 29 | rm -rf jenkins 30 | 31 | aws s3 cp ${BACKUP_NAME}.tar.gz $S3_BACKUP_BUCKET/jenkins-analytics/${BACKUP_NAME}.tar.gz 32 | # Remove tar jenkins backup file after uploading to S3 33 | rm -rf ${BACKUP_NAME}.tar.gz 34 | -------------------------------------------------------------------------------- /dataeng/resources/model-transfers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating python 3.11 virtual environment to run dbt warehouse-transform job 5 | PYTHON_VENV="py311_venv" 6 | virtualenv --python=python3.11 --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/warehouse-transforms 11 | # To install right version of dbt 12 | pip install -r requirements.txt 13 | 14 | 15 | cd $WORKSPACE/warehouse-transforms/projects/$DBT_PROJECT 16 | 17 | # Choose the marts from which to transfer DBT models based on Jenkins job parameter. 18 | if [ "$MODELS_TO_TRANSFER" = 'daily' ] 19 | then 20 | MART_NAME=programs_reporting 21 | elif [ "$MODELS_TO_TRANSFER" = 'enterprise' ] 22 | then 23 | MART_NAME=enterprise 24 | else 25 | echo "Unknown MODELS_TO_TRANSFER: '${MODELS_TO_TRANSFER}'" 26 | exit 1 27 | fi 28 | 29 | ARGS="{mart: ${MART_NAME} }" 30 | 31 | source $WORKSPACE/secrets-manager.sh 32 | # Fetch the secrets from AWS 33 | set +x 34 | get_secret_value analytics-secure/warehouse-transforms/profiles PRIVATE_KEY 35 | set -x 36 | export PRIVATE_KEY 37 | 38 | dbt deps --profiles-dir $WORKSPACE/warehouse-transforms/profiles --profile $DBT_PROFILE --target $DBT_TARGET 39 | 40 | # Call DBT to perform all transfers for this mart. 41 | dbt run-operation perform_s3_transfers --args "${ARGS}" --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ 42 | -------------------------------------------------------------------------------- /dataeng/resources/opsgenie-enable-heartbeat.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Enables an Opsgenie Heartbeat, while also setting the expected duration. 4 | # 5 | # Assumes that the heartbeat is already created in the Opsgenie settings here: 6 | # https://2u-internal.app.opsgenie.com/settings/heartbeat 7 | # 8 | # Required environment variables: 9 | # OPSGENIE_HEARTBEAT_NAME: Name of the already-created Opsgenie heartbeat. 10 | # OPSGENIE_HEARTBEAT_CONFIG_KEY: API key provided by Opsgenie which is authorized to modify heartbeat configuration. 11 | # OPSGENIE_HEARTBEAT_DURATION_NUM: Specifies how often a heartbeat message should be expected. 12 | # OPSGENIE_HEARTBEAT_DURATION_UNIT: Interval specified as minutes, hours or days. 13 | 14 | 15 | OPSGENIE_HEARTBEAT_API_URL="https://api.opsgenie.com/v2/heartbeats" 16 | 17 | if [ -n "$OPSGENIE_HEARTBEAT_NAME" ] && \ 18 | [ -n "$OPSGENIE_HEARTBEAT_CONFIG_KEY" ] && \ 19 | [ -n "$OPSGENIE_HEARTBEAT_DURATION_NUM" ] && \ 20 | [ -n "$OPSGENIE_HEARTBEAT_DURATION_UNIT" ]; then 21 | 22 | AUTH_HEADER="Authorization: GenieKey $OPSGENIE_HEARTBEAT_CONFIG_KEY" 23 | JSON_HEADER="Content-Type: application/json" 24 | HEARTBEAT_API_URL="$OPSGENIE_HEARTBEAT_API_URL/$OPSGENIE_HEARTBEAT_NAME" 25 | HEARTBEAT_API_PING_URL="$HEARTBEAT_API_URL/ping" 26 | ALERT_MESSAGE="Heartbeat [$OPSGENIE_HEARTBEAT_NAME] expired - job is likely stuck." 27 | 28 | # Make API request to get existing heartbeat 29 | # If the heartbeat already exists, then GET_EXISTING_HEARTBEAT will set to heartbeat name "GET_EXISTING_HEARTBEAT=heatbeat_name" 30 | # If heartbeat does not exist GET_EXISTING_HEARTBEAT is set to empty 31 | GET_EXISTING_HEARTBEAT=$(curl -X GET $HEARTBEAT_API_URL --header "$AUTH_HEADER" | grep -o $OPSGENIE_HEARTBEAT_NAME | sort -u) 32 | # Add the heartbeat if heartbeat doesn't exist 33 | if [ -z "$GET_EXISTING_HEARTBEAT" ]; then 34 | JSON_REQ_DATA_CREATE_HEARTBEAT="{ 35 | \"name\":\"$OPSGENIE_HEARTBEAT_NAME\", 36 | \"description\": \"$JOB_URL\", 37 | \"intervalUnit\": \"$OPSGENIE_HEARTBEAT_DURATION_UNIT\", 38 | \"interval\": \"$OPSGENIE_HEARTBEAT_DURATION_NUM\", 39 | \"ownerTeam\": {\"name\": \"Data Engineering\"}, 40 | \"alertMessage\": \"$ALERT_MESSAGE\", 41 | \"alertPriority\": \"P3\", 42 | \"enabled\" : true 43 | }" 44 | # Add heartbeat 45 | curl -X POST $OPSGENIE_HEARTBEAT_API_URL --header "$AUTH_HEADER" --header "$JSON_HEADER" --data "$JSON_REQ_DATA_CREATE_HEARTBEAT" 46 | fi 47 | # Json request body to update existing heartbeat 48 | JSON_REQ_DATA="{ 49 | \"description\": \"$JOB_URL\", 50 | \"interval\": \"$OPSGENIE_HEARTBEAT_DURATION_NUM\", 51 | \"intervalUnit\": \"$OPSGENIE_HEARTBEAT_DURATION_UNIT\", 52 | \"alertMessage\": \"$ALERT_MESSAGE\", 53 | \"enabled\": true 54 | }" 55 | # Enable the heartbeat and set the duration num/units. 56 | curl -X PATCH "$HEARTBEAT_API_URL" --header "$AUTH_HEADER" --header "$JSON_HEADER" --data "$JSON_REQ_DATA" 57 | # A re-enabled heartbeat will likely be expired, so ping it once to make it active and begin a new duration countdown. 58 | curl -X GET "$HEARTBEAT_API_PING_URL" --header "$AUTH_HEADER" 59 | fi 60 | -------------------------------------------------------------------------------- /dataeng/resources/org-exporter-worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | TODAY=$(date +%d) 5 | 6 | env | sort 7 | 8 | ${EXPORTER_VENV}/bin/exporter \ 9 | --org=${ORG} \ 10 | --output-bucket=${OUTPUT_BUCKET} \ 11 | --external-prefix=databases/${DATE:-$TODAY} \ 12 | --django-admin=${PLATFORM_VENV}/bin/django-admin \ 13 | --django-pythonpath=${PLATFORM_VENV}/edx-platform \ 14 | --gpg-keys=${GPG_KEYS_PATH} \ 15 | ${EXTRA_OPTIONS} ${CONFIG_PATH} ${ORG_CONFIG_PATH} 16 | -------------------------------------------------------------------------------- /dataeng/resources/prefect-flows-deployment-identify.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | cd $WORKSPACE/prefect-flows 5 | 6 | #Only run deployment on merge commits, otherwise exit 7 | HEAD_COMMIT=$(git rev-parse HEAD) 8 | LAST_MERGE_COMMIT=$(git log --merges origin/master --format='%H' --max-count=1) 9 | if [ $HEAD_COMMIT == $LAST_MERGE_COMMIT ] 10 | then 11 | echo "This is one of merge commit, Run CD" 12 | else 13 | echo "Exiting because not a merge commit" 14 | exit 0 15 | fi 16 | 17 | 18 | #Get second last merge commit id and compares it with the HEAD to find (git diff) files changed. 19 | PREV_MERGE_COMMIT_ID=$(git log --merges origin/master --format='%H' --max-count=2 | sed -n 2p) 20 | 21 | git diff $PREV_MERGE_COMMIT_ID --name-only # Printing for debug purpose 22 | 23 | 24 | # Remove downstream.properties file to write new input 25 | rm -f ${WORKSPACE}/downstream.properties 26 | # Output 'FLOWS_TO_DEPLOY=' in file. This will be used as name of parameter in downstream jobs 27 | echo -n "FLOWS_TO_DEPLOY=" > "${WORKSPACE}/downstream.properties" 28 | 29 | # Extract the immediate parent directory name of all the .toml or .py files those are changed 30 | # also removing the duplicates that may occur if both .toml and .py files of a flow has changed. 31 | FLOW_NAMES=$(git diff $PREV_MERGE_COMMIT_ID --name-only | grep -E ".*\.(py|toml)" |cut -d'/' -f2 |sort -u) 32 | 33 | # Loop will parse list and append it with 'FLOWS_TO_DEPLOY=' to pass this parameter in downstream job 34 | list=($FLOW_NAMES) 35 | for item in "${list[@]}" 36 | do 37 | # Writing all the changed flows in downstream.properties file which is passed as a parameter to downstream job 38 | echo -n "prefect-flows-deployment-${item}, " >> "${WORKSPACE}/downstream.properties" 39 | done 40 | 41 | sed -i 's/[ \t]*$//' ${WORKSPACE}/downstream.properties # Removing trailing space character 42 | sed -i 's/,$//' ${WORKSPACE}/downstream.properties # Removing trailing comma 43 | -------------------------------------------------------------------------------- /dataeng/resources/prefect-flows-deployment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating python3.9 virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=python3.9 --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Removing prefix 'prefect-flows-deployment-' 10 | # $FLOW_NAME will contain the name of flow going to be deployed 11 | FLOW_NAME=$(echo $JOB_NAME | cut -c 26-) 12 | 13 | # Install prefect-flow requirements, as it contains edx-prefectutils in itself so no need to install it separately 14 | cd $WORKSPACE/prefect-flows 15 | pip install -r requirements.txt 16 | 17 | # prune unused images, if there are any to prune. 18 | unused_images=$(docker images -f dangling=true -q) 19 | if [ -n "$unused_images" ]; then 20 | docker rmi -f $unused_images 21 | fi 22 | 23 | # Get ECR authetication 24 | aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR_LOGIN 25 | 26 | # The following statement will help us to determine if repository already exists otherwise it will create a new repository with the name of flow 27 | aws ecr describe-repositories --repository-names $FLOW_NAME --region us-east-1 || aws ecr create-repository --repository-name $FLOW_NAME --region us-east-1 28 | 29 | # Do not print commands in this function since they may contain secrets. 30 | set +x 31 | 32 | # Fetch the secrets from AWS 33 | PREFECT_CLOUD_AGENT_TOKEN=$(aws secretsmanager get-secret-value --secret-id analytics-secure/prefect-cd --region us-east-1 --query SecretString --output text | jq -r ".PREFECT_CLOUD_AGENT_TOKEN") 34 | 35 | # Get Authenticated with Prefect Cloud 36 | prefect auth login --key $PREFECT_CLOUD_AGENT_TOKEN 37 | 38 | set -x 39 | # Deploy the flow. $FLOW_NAME will contain the name of flow to be deployed 40 | make -C flows $FLOW_NAME 41 | -------------------------------------------------------------------------------- /dataeng/resources/read-replica-export.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | env 6 | 7 | # Cleanup downstream properties file first: 8 | DOWNSTREAM_PROPERTIES_FILE="${WORKSPACE}/downstream.properties" 9 | rm "${DOWNSTREAM_PROPERTIES_FILE}" || true 10 | 11 | # Keep track of the start time to feed into downstream validation scripts. 12 | echo "SQOOP_START_TIME=$(date --utc --iso=minutes)" >> "${DOWNSTREAM_PROPERTIES_FILE}" 13 | 14 | # Interpolate the RUN_DATE now so that the downstream job is guaranteed to use 15 | # the same exact date as this job. Otherwise, if this job runs over a date 16 | # boundary, the downstream job would re-interpolate the value of 'yesterday' on 17 | # a different date. 18 | INTERPOLATED_RUN_DATE="$(date +%Y-%m-%d -d "$TO_DATE")" 19 | echo "RUN_DATE=${INTERPOLATED_RUN_DATE}" >> "${DOWNSTREAM_PROPERTIES_FILE}" 20 | 21 | ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \ 22 | ExportMysqlDatabaseToS3Task --local-scheduler \ 23 | --date ${INTERPOLATED_RUN_DATE} \ 24 | --db-credentials $DB_CREDENTIALS \ 25 | --database $DATABASE \ 26 | --overwrite \ 27 | ${EXCLUDE_FIELD} \ 28 | ${INCLUDE} \ 29 | ${EXCLUDE} \ 30 | ${EXTRA_ARGS} 31 | -------------------------------------------------------------------------------- /dataeng/resources/retirement-partner-report-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Create and activate a virtualenv. In case we ever change the concurrency 6 | # setting on the jenkins worker, it would be safest to keep the builds from 7 | # clobbering each other's virtualenvs. 8 | VENV="venv-${BUILD_NUMBER}" 9 | virtualenv --python=python3.9 --clear "${VENV}" 10 | source "${VENV}/bin/activate" 11 | 12 | #Fetch secrets from AWS 13 | cd $WORKSPACE/configuration 14 | pip install -r util/jenkins/requirements.txt 15 | # hide the sensitive information in the logs 16 | set +x 17 | 18 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 19 | 20 | # Create a temporary file to store the YAML 21 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 22 | 23 | # Write the YAML data to the temporary file 24 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 25 | 26 | # Fetch google-service-account secrets 27 | GOOGLE_SERVICE_ACCOUNT_JSON=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json" --region "us-east-1" --output json | jq -r '.SecretString') 28 | # Create a temporary file to store the YAML 29 | TEMP_GOOGLE_SECRETS=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.json) 30 | 31 | # Write the YAML data to the temporary file 32 | echo "$GOOGLE_SERVICE_ACCOUNT_JSON" > "$TEMP_GOOGLE_SECRETS" 33 | 34 | set -x 35 | 36 | # prepare tubular 37 | cd $WORKSPACE/tubular 38 | # snapshot the current latest versions of pip and setuptools. 39 | pip install 'pip==21.0.1' 'setuptools==53.0.0' 40 | pip install -r requirements.txt 41 | 42 | # Call the script to cleanup the reports 43 | python scripts/delete_expired_partner_gdpr_reports.py \ 44 | --config_file=$TEMP_CONFIG_YAML \ 45 | --google_secrets_file=$TEMP_GOOGLE_SECRETS \ 46 | --age_in_days=$AGE_IN_DAYS 47 | 48 | # Remove the temporary files after processing 49 | rm -f "$TEMP_CONFIG_YAML" 50 | rm -f "$TEMP_GOOGLE_SECRETS" 51 | -------------------------------------------------------------------------------- /dataeng/resources/retirement-partner-reporter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Display the environment variables again, this time within the context of a 6 | # new subshell inside of the job. N.B. this should not print plain credentials 7 | # because the only credentialsBindings we currently use is of type "file" which 8 | # just stores a filename in the environment (rather than the content). 9 | env 10 | 11 | # Create and activate a virtualenv. In case we ever change the concurrency 12 | # setting on the jenkins worker, it would be safest to keep the builds from 13 | # clobbering each other's virtualenvs. 14 | VENV="venv-${BUILD_NUMBER}" 15 | virtualenv --python=python3.9 --clear "${VENV}" 16 | source "${VENV}/bin/activate" 17 | 18 | # Make sure that when we try to write unicode to the console, it 19 | # correctly encodes to UTF-8 rather than exiting with a UnicodeEncode 20 | # error. 21 | export PYTHONIOENCODING=UTF-8 22 | export LC_CTYPE=en_US.UTF-8 23 | 24 | #Fetch secrets from AWS 25 | cd $WORKSPACE/configuration 26 | pip install -r util/jenkins/requirements.txt 27 | # hide the sensitive information in the logs 28 | set +x 29 | 30 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 31 | 32 | # Create a temporary file to store the YAML 33 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 34 | 35 | # Write the YAML data to the temporary file 36 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 37 | 38 | # Fetch google-service-account secrets 39 | GOOGLE_SERVICE_ACCOUNT_JSON=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json" --region "us-east-1" --output json | jq -r '.SecretString') 40 | # Create a temporary file to store the YAML 41 | TEMP_GOOGLE_SECRETS=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.json) 42 | 43 | # Write the YAML data to the temporary file 44 | echo "$GOOGLE_SERVICE_ACCOUNT_JSON" > "$TEMP_GOOGLE_SECRETS" 45 | 46 | set -x 47 | 48 | # prepare tubular 49 | cd $WORKSPACE/tubular 50 | # snapshot the current latest versions of pip and setuptools. 51 | pip install 'pip==21.0.1' 'setuptools==53.0.0' 52 | pip install -r requirements.txt 53 | 54 | # Create the directory where we will store reports, one per partner 55 | rm -rf $PARTNER_REPORTS_DIR 56 | mkdir $PARTNER_REPORTS_DIR 57 | 58 | # Call the script to generate the reports and upload them to Google Drive 59 | python scripts/retirement_partner_report.py \ 60 | --config_file=$TEMP_CONFIG_YAML \ 61 | --google_secrets_file=$TEMP_GOOGLE_SECRETS \ 62 | --output_dir=$PARTNER_REPORTS_DIR 63 | 64 | # Remove the temporary files after processing 65 | rm -f "$TEMP_CONFIG_YAML" 66 | rm -f "$TEMP_GOOGLE_SECRETS" 67 | -------------------------------------------------------------------------------- /dataeng/resources/run-course-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Creating python2.7 virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=python2.7 --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Create destination directory 10 | WORKING_DIRECTORY=/var/lib/jenkins/tmp/analytics-course-exporter 11 | mkdir -p ${WORKING_DIRECTORY}/course-data 12 | 13 | # Install requirements into this (exporter) virtual environment 14 | pushd analytics-exporter/ 15 | pip install 'setuptools<45' 16 | pip install -r github_requirements.txt 17 | pip install mysql-connector-python -e . 18 | popd 19 | 20 | # Get name of other (platform) virtual environment 21 | source platform_venv_path 22 | 23 | # Configuration paths in analytics-secure 24 | SECURE_ROOT=${WORKSPACE}/analytics-secure/analytics-exporter 25 | CONFIG_PATH=${SECURE_ROOT}/${EXPORTER_CONFIG_FILENAME} 26 | 27 | DATE=$(date +%d ${DATE_MODIFIER}) 28 | TODAY=$(date +%d) 29 | 30 | env | sort 31 | 32 | # Export job configuration files 33 | course-exporter \ 34 | ${COURSES} \ 35 | ${TASKS} \ 36 | --work-dir=${WORKING_DIRECTORY} \ 37 | --output-bucket=${OUTPUT_BUCKET} \ 38 | --external-prefix=databases/${DATE:-$TODAY} \ 39 | --django-admin=${PLATFORM_VENV}/bin/django-admin.py \ 40 | --django-pythonpath=${PLATFORM_VENV}/edx-platform \ 41 | ${CONFIG_PATH} 42 | -------------------------------------------------------------------------------- /dataeng/resources/run-pipeline-acceptance-test.sh: -------------------------------------------------------------------------------- 1 | #################################################################### 2 | # Common configuration 3 | # Assumes that ACCEPTANCE_TEST_CONFIG has been defined. 4 | 5 | export VENV_ROOT=$WORKSPACE/build/venvs 6 | PIP_INSTALL="pip install" 7 | 8 | #################################################################### 9 | # Tasks configuration 10 | 11 | TASKS_BIN=$VENV_ROOT/analytics-tasks/bin 12 | 13 | # Set environment variable used in task acceptance service: 14 | export REMOTE_TASK=$TASKS_BIN/remote-task 15 | 16 | #################################################################### 17 | # Exporter configuration 18 | 19 | EXPORTER_BIN=$VENV_ROOT/analytics-exporter/bin 20 | 21 | EXPORTER_CONFIG=default.yaml 22 | 23 | # Set environment variable used in test_database_export 24 | export EXPORTER=$EXPORTER_BIN/exporter 25 | export COURSE_EXPORTER=$EXPORTER_BIN/course-exporter 26 | 27 | # Exporter configuration destination 28 | 29 | ROOT=${WORKSPACE}/analytics-secure/analytics-exporter 30 | SECURE_HASH=`GIT_DIR=./analytics-secure/.git git rev-parse HEAD` 31 | EXPORTER_CONFIG_BUCKET=$EXPORTER_BUCKET_PATH/$SECURE_HASH 32 | EXPORTER_CONFIG_PATH=${EXPORTER_CONFIG_BUCKET}/${EXPORTER_CONFIG} 33 | GPG_KEYS_PATH=${EXPORTER_CONFIG_BUCKET}/gpg-keys 34 | 35 | #################################################################### 36 | # Install pre-requisites 37 | 38 | env | sort 39 | 40 | mkdir -p $VENV_ROOT 41 | 42 | [ -f $TASKS_BIN/activate ] || virtualenv -p /usr/bin/python2.7 $VENV_ROOT/analytics-tasks 43 | virtualenv -p /usr/bin/python2.7 $VENV_ROOT/analytics-exporter 44 | # The virtualenv on this version of Jenkins shiningpanda is old, so manually update both pip and setuptools before loading exporter. 45 | $EXPORTER_BIN/$PIP_INSTALL -U 'pip==20.3.4' 46 | $EXPORTER_BIN/$PIP_INSTALL -U 'setuptools<45' 47 | $EXPORTER_BIN/$PIP_INSTALL -U 'wheel' 48 | $EXPORTER_BIN/$PIP_INSTALL -U 'six' 49 | $EXPORTER_BIN/$PIP_INSTALL -U -r $WORKSPACE/analytics-exporter/requirements.txt -r $WORKSPACE/analytics-exporter/github_requirements.txt 50 | $EXPORTER_BIN/$PIP_INSTALL -e $WORKSPACE/analytics-exporter/ 51 | 52 | #################################################################### 53 | # Upload exporter configuration 54 | 55 | $EXPORTER_BIN/$PIP_INSTALL awscli 56 | 57 | $EXPORTER_BIN/aws s3 cp ${ROOT}/${EXPORTER_CONFIG} ${EXPORTER_CONFIG_PATH} 58 | $EXPORTER_BIN/aws s3 cp --recursive ${ROOT}/gpg-keys ${GPG_KEYS_PATH} 59 | 60 | #################################################################### 61 | # Run tests 62 | 63 | . $TASKS_BIN/activate && make -C analytics-tasks install test-acceptance 64 | -------------------------------------------------------------------------------- /dataeng/resources/secrets-manager-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define the location of the script in the Jenkins workspace 4 | SCRIPT_PATH="$WORKSPACE/secrets-manager.sh" 5 | 6 | # Write the script content to the specified location 7 | cat < "$SCRIPT_PATH" 8 | #!/usr/bin/env bash 9 | 10 | get_secret_value() { 11 | local secret_to_call="\$1" 12 | local secret_name="\$2" 13 | local SECRET_JSON 14 | 15 | SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_to_call" --region "us-east-1" --output json) 16 | 17 | # Check the exit status of the AWS CLI command 18 | if [ \$? -eq 0 ]; then 19 | extract_and_store_secret_value "\$SECRET_JSON" "\$secret_name" 20 | else 21 | echo "AWS CLI command failed" 22 | fi 23 | } 24 | 25 | extract_and_store_secret_value() { 26 | local json_data="\$1" 27 | local secret_name="\$2" 28 | local value 29 | 30 | value=\$(echo "\$json_data" | jq -r ".SecretString | fromjson.\"\$secret_name\"" 2>/dev/null) 31 | if [ \$? -eq 0 ]; then 32 | eval "\$secret_name"='\$value' 33 | else 34 | echo "Failed to extract secret value for \$secret_name" 35 | fi 36 | } 37 | EOF 38 | -------------------------------------------------------------------------------- /dataeng/resources/secrets-manager.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | extract_value_from_json() { 4 | local json="$1" 5 | local key="$2" 6 | local value=$(echo "$json" | jq -r ".$key") 7 | } 8 | 9 | fetch_whole_secret() { 10 | local secret_name="$1" 11 | local variable_name="$2" 12 | local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text) 13 | #set whole file as env var 14 | declare "${secret_name%=*}=${secret_value}" 15 | } 16 | 17 | fetch_specific_key() { 18 | local secret_name="$1" 19 | local key="$2" 20 | local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text) 21 | local extracted_value=$(extract_value_from_json "$secret_value" "$key") 22 | declare "${key%=*}=${extracted_value}" 23 | } 24 | 25 | # Main script 26 | if [[ "$1" == "-w" ]]; then 27 | if [ $# -ne 3 ]; then 28 | echo "Usage: $0 -w " 29 | exit 1 30 | fi 31 | fetch_whole_secret "$2" "$3" 32 | else 33 | if [ $# -ne 2 ]; then 34 | echo "Usage: $0 " 35 | exit 1 36 | fi 37 | fetch_specific_key "$1" "$2" 38 | fi -------------------------------------------------------------------------------- /dataeng/resources/set_build_status.groovy: -------------------------------------------------------------------------------- 1 | import hudson.model.* 2 | def res = manager.build.getResult().toString() 3 | def buildStatusParam = new StringParameterValue('BUILD_STATUS', res) 4 | manager.build.replaceAction(new ParametersAction(buildStatusParam)) 5 | -------------------------------------------------------------------------------- /dataeng/resources/setup-exporter-email-optin.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Create destination directory 5 | mkdir -p /var/lib/jenkins/tmp/analytics-exporter/course-data 6 | 7 | # Create and activate a virtualenv in shell script 8 | EXPORTER_VENV="exporter_venv" 9 | virtualenv --python=python3.11 --clear "${EXPORTER_VENV}" 10 | source "${EXPORTER_VENV}/bin/activate" 11 | 12 | cd $WORKSPACE/analytics-tools/snowflake 13 | pip install boto3 14 | 15 | python3 secrets-manager.py -w -n analytics-secure/analytics-exporter/task-auth.json -v ${WORKSPACE}/analytics-secure/analytics-exporter/task-auth.json 16 | cd $WORKSPACE 17 | 18 | # Install requirements into this (exporter) virtual environment 19 | pushd analytics-exporter/ 20 | pip install 'setuptools<65' 21 | pip install -r github_requirements.txt 22 | pip install mysql-connector-python -e . 23 | popd 24 | 25 | # Configuration paths in analytics-secure 26 | SECURE_ROOT=${WORKSPACE}/analytics-secure/analytics-exporter 27 | CONFIG_PATH=${SECURE_ROOT}/${EXPORTER_CONFIG_FILENAME} 28 | GPG_KEYS_PATH=${WORKSPACE}/data-czar-keys 29 | 30 | # Save virtualenv location and configuration paths 31 | echo " 32 | EXPORTER_VENV=${WORKSPACE}/${EXPORTER_VENV} 33 | CONFIG_PATH=${CONFIG_PATH} 34 | GPG_KEYS_PATH=${GPG_KEYS_PATH} 35 | DATE=$(date +%d ${DATE_MODIFIER}) 36 | ORG_CONFIG_PATH=${WORKSPACE}/${ORG_CONFIG} 37 | " > exporter_vars 38 | 39 | env | sort 40 | 41 | # Export job configuration files. 42 | # 'orgranizations' is the directory where to save Jenkins property files and is used by fileBuildParameterFactory in 43 | # master job DSL to generate parameters for each of the worker jobs. 44 | exporter-properties \ 45 | --output-bucket=${OUTPUT_BUCKET} \ 46 | --output-prefix=${OUTPUT_PREFIX} \ 47 | --orgs="${ORGS}" \ 48 | --include=platform_venv_path \ 49 | --include=exporter_vars \ 50 | ${CONFIG_PATH} \ 51 | ${WORKSPACE}/${ORG_CONFIG} \ 52 | organizations 53 | -------------------------------------------------------------------------------- /dataeng/resources/setup-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Create destination directory 5 | mkdir -p /var/lib/jenkins/tmp/analytics-exporter/course-data 6 | 7 | # Create and activate a virtualenv in shell script 8 | EXPORTER_VENV="exporter_venv" 9 | virtualenv --python=python3.11 --clear "${EXPORTER_VENV}" 10 | source "${EXPORTER_VENV}/bin/activate" 11 | 12 | cd $WORKSPACE/analytics-tools/snowflake 13 | pip install boto3 14 | 15 | python3 secrets-manager.py -w -n analytics-secure/analytics-exporter/task-auth.json -v ${WORKSPACE}/analytics-secure/analytics-exporter/task-auth.json 16 | cd $WORKSPACE 17 | 18 | # Install requirements into this (exporter) virtual environment 19 | pushd analytics-exporter/ 20 | pip install 'setuptools<65' 21 | pip install -r github_requirements.txt 22 | pip install mysql-connector-python -e . 23 | popd 24 | 25 | # Configuration paths in analytics-secure 26 | SECURE_ROOT=${WORKSPACE}/analytics-secure/analytics-exporter 27 | CONFIG_PATH=${SECURE_ROOT}/${EXPORTER_CONFIG_FILENAME} 28 | GPG_KEYS_PATH=${WORKSPACE}/data-czar-keys 29 | 30 | # Save virtualenv location and configuration paths 31 | echo " 32 | EXPORTER_VENV=${WORKSPACE}/${EXPORTER_VENV} 33 | CONFIG_PATH=${CONFIG_PATH} 34 | GPG_KEYS_PATH=${GPG_KEYS_PATH} 35 | DATE=$(date +%d ${DATE_MODIFIER}) 36 | EXTRA_OPTIONS=${EXTRA_OPTIONS} 37 | ORG_CONFIG_PATH=${WORKSPACE}/${ORG_CONFIG} 38 | SECURE_BRANCH=${SECURE_BRANCH} 39 | " > exporter_vars 40 | 41 | env | sort 42 | 43 | 44 | # Export job configuration files 45 | exporter-properties \ 46 | --output-bucket=${OUTPUT_BUCKET} \ 47 | --orgs="${ORGS}" \ 48 | --include=platform_venv_path \ 49 | --include=exporter_vars \ 50 | ${CONFIG_PATH} \ 51 | ${WORKSPACE}/${ORG_CONFIG} \ 52 | organizations 53 | 54 | # Dirty hack: 55 | # Some orgs can take an exceptionally long time to run. Depending on the concurrency 56 | # settings for the analytics-exporter-master job and the location of the organization 57 | # alphabetically in the organizations directory, it's possible that these long-running 58 | # jobs will be started towards the end of the master run, which can extend the total 59 | # run-time quite a bit. Use PRIORITY_ORGS to pass a space separated list of orgs that 60 | # should run first. This is accomplished by prepending a number to the name of the orgs 61 | # in question. 62 | for ORG in ${PRIORITY_ORGS}; do 63 | if [ -f organizations/$ORG ]; then 64 | mv organizations/$ORG organizations/1_$ORG 65 | fi 66 | done 67 | -------------------------------------------------------------------------------- /dataeng/resources/setup-platform-venv-legacy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Some recent changes in edx-platform breaks the exporter. 5 | # We are currently using edx-platform's aed/analytics-exporter-settings-hotfix(Nov 2017) which follows an old 6 | # requirements installation strategy. This file would go away in favor of 'setup-platform-env' once we figure out the 7 | # underlying issue. 8 | #!/usr/bin/env bash 9 | 10 | # Install requirements 11 | pushd edx-platform 12 | pip install --exists-action w -r requirements/edx/pre.txt 13 | pip install --exists-action w -r requirements/edx/django.txt 14 | pip install --exists-action w -r requirements/edx/base.txt 15 | 16 | # Remove if https://github.com/openedx/edx-platform/pull/7465 has been merged 17 | if [[ -f requirements/edx/post.txt ]]; then 18 | pip install --exists-action w -r requirements/edx/post.txt 19 | fi 20 | 21 | pip install --exists-action w -r requirements/edx/github.txt 22 | pip install --exists-action w -r requirements/edx/local.txt 23 | popd 24 | 25 | # Save virtualenv location 26 | echo "PLATFORM_VENV=${VIRTUAL_ENV}" > platform_venv 27 | -------------------------------------------------------------------------------- /dataeng/resources/setup-platform-venv-py3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # This version of setup-platform-venv is specifically designed for newer 4 | # python3 versions of the platform. 5 | 6 | 7 | # Create and activate a virtualenv in shell script 8 | PLATFORM_VENV="platform_venv" 9 | virtualenv --python=python3.11 --clear "${PLATFORM_VENV}" 10 | source "${PLATFORM_VENV}/bin/activate" 11 | 12 | # Install requirements 13 | pushd edx-platform 14 | # This is the same pip version we currently pin in our devstack edxapp container: 15 | pip install pip==20.2.3 16 | make requirements 17 | # This is a new requirement for the skill-tagging plugin, specifically for 2U 18 | pip install skill-tagging==0.2.0 19 | 20 | popd 21 | 22 | # Save virtualenv location 23 | echo "PLATFORM_VENV=${WORKSPACE}/${PLATFORM_VENV}" > platform_venv_path 24 | -------------------------------------------------------------------------------- /dataeng/resources/setup-platform-venv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Install requirements 4 | pushd edx-platform 5 | pip install "setuptools<45" 6 | make requirements 7 | popd 8 | 9 | # Save virtualenv location 10 | echo "PLATFORM_VENV=${VIRTUAL_ENV}" > platform_venv 11 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-collect-metrics.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 -v rsa_key_snowflake_task_automation_user 14 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowflake_task_automation_user -v rsa_key_passphrase_snowflake_task_automation_user 15 | 16 | 17 | 18 | python collect-metrics.py \ 19 | --metric_name $METRIC_NAME \ 20 | --automation_user $SNOWFLAKE_USER \ 21 | --account $SNOWFLAKE_ACCOUNT \ 22 | --warehouse $SNOWFLAKE_WAREHOUSE \ 23 | --key_file rsa_key_snowflake_task_automation_user \ 24 | --passphrase_file rsa_key_passphrase_snowflake_task_automation_user 25 | 26 | 27 | rm rsa_key_snowflake_task_automation_user 28 | rm rsa_key_passphrase_snowflake_task_automation_user 29 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-expire-individual-password.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | 14 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 -v rsa_key_snowflake_task_automation_user 15 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowflake_task_automation_user -v rsa_key_passphrase_snowflake_task_automation_user 16 | 17 | python expire_user_passwords.py \ 18 | --automation_user 'SNOWFLAKE_TASK_AUTOMATION_USER' \ 19 | --account 'edx.us-east-1' \ 20 | --user_to_expire $USER_TO_EXPIRE \ 21 | --key_file rsa_key_snowflake_task_automation_user \ 22 | --pass_file rsa_key_passphrase_snowflake_task_automation_user 23 | 24 | 25 | rm rsa_key_snowflake_task_automation_user 26 | rm rsa_key_passphrase_snowflake_task_automation_user 27 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-expire-passwords.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | 14 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 -v rsa_key_snowflake_task_automation_user 15 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowflake_task_automation_user -v rsa_key_passphrase_snowflake_task_automation_user 16 | 17 | python expire_user_passwords.py \ 18 | --automation_user 'SNOWFLAKE_TASK_AUTOMATION_USER' \ 19 | --account 'edx.us-east-1' \ 20 | --key_file rsa_key_snowflake_task_automation_user \ 21 | --pass_file rsa_key_passphrase_snowflake_task_automation_user 22 | 23 | rm rsa_key_snowflake_task_automation_user 24 | rm rsa_key_passphrase_snowflake_task_automation_user 25 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-public-grants-cleaner.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_stitch_loader.p8 -v rsa_key_stitch_loader 14 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_stitch_loader -v rsa_key_passphrase_stitch_loader 15 | 16 | unset KEY_PATH 17 | unset PASSPHRASE_PATH 18 | 19 | python snowflake_public_grants_cleaner.py \ 20 | --user "STITCH_LOADER" \ 21 | --account "edx.us-east-1" \ 22 | --key_file rsa_key_stitch_loader \ 23 | --passphrase_file rsa_key_passphrase_stitch_loader 24 | 25 | 26 | rm rsa_key_stitch_loader 27 | rm rsa_key_passphrase_stitch_loader 28 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-refresh-snowpipe.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | 14 | 15 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user 16 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user 17 | #set -x 18 | 19 | unset KEY_PATH 20 | unset PASSPHRASE_PATH 21 | 22 | python refresh_snowpipe.py \ 23 | --user 'SNOWPIPE' \ 24 | --schema $SCHEMA \ 25 | --account 'edx.us-east-1' \ 26 | --pipe_name $PIPE_NAME \ 27 | --table_name $TABLE_NAME \ 28 | --delay $DELAY \ 29 | --limit $LIMIT \ 30 | --key_file rsa_key_snowpipe_user \ 31 | --passphrase_file rsa_key_passphrase_snowpipe_user 32 | 33 | rm rsa_key_snowpipe_user 34 | rm rsa_key_passphrase_snowpipe_user 35 | 36 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-replica-import.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | env 6 | 7 | # First delete the downstream properties file so that we do not trigger 8 | # downstream jobs if not necessary. 9 | DOWNSTREAM_PROPERTIES_FILE="${WORKSPACE}/downstream.properties" 10 | rm "${DOWNSTREAM_PROPERTIES_FILE}" || true 11 | 12 | # Only invoke this script on Mondays, or when forced to run via the FORCE 13 | # parameter, otherwise skip it. Since this job is triggered by an upstream for 14 | # which we can't change the cron schedule (due to vertica not yet being 15 | # deprecated), we must skip it manually in this shell script. 16 | DAY_OF_WEEK=$(date +%u) 17 | MONDAY=1 18 | if [[ ${DAY_OF_WEEK} -eq ${MONDAY} || ${FORCE} = "true" ]]; then 19 | 20 | # Actually run the Luigi tasks: 21 | ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \ 22 | ImportMysqlDatabaseFromS3ToSnowflakeSchemaTask --local-scheduler \ 23 | --date $(date +%Y-%m-%d -d "$RUN_DATE") \ 24 | --credentials $SNOWFLAKE_CREDENTIALS \ 25 | --warehouse $WAREHOUSE \ 26 | --role $ROLE \ 27 | --sf-database $SNOWFLAKE_DATABASE \ 28 | --schema $SCHEMA \ 29 | --scratch-schema $SCRATCH_SCHEMA \ 30 | --run-id $BUILD_ID \ 31 | --database $DATABASE \ 32 | --overwrite \ 33 | ${INCLUDE} \ 34 | ${EXCLUDE} \ 35 | ${EXTRA_ARGS} 36 | 37 | # All went well (we know that because of set -e), so we should generate a 38 | # downstream properties file which signals this job to invoke the downstream 39 | # job. 40 | echo "SQOOP_START_TIME=${SQOOP_START_TIME}" >> "${DOWNSTREAM_PROPERTIES_FILE}" 41 | 42 | else 43 | echo "SKIPPING build because today is not Monday." 44 | fi 45 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-schema-builder.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating python 3.11 virtual environment to run schema builder 5 | PYTHON311_VENV="py311_venv" 6 | virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" 7 | source "${PYTHON311_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/warehouse-transforms 11 | pip install --upgrade dbt-schema-builder 12 | 13 | source $WORKSPACE/secrets-manager.sh 14 | # Fetch the secrets from AWS 15 | set +x 16 | get_secret_value analytics-secure/warehouse-transforms/profiles PRIVATE_KEY 17 | set -x 18 | export PRIVATE_KEY 19 | 20 | cd $WORKSPACE/warehouse-transforms/projects/$SOURCE_PROJECT 21 | dbt clean --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 22 | 23 | # DESTINATION_PROJECT is always relative to SOURCE_PROJECT 24 | cd $DESTINATION_PROJECT 25 | dbt clean --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 26 | 27 | cd $WORKSPACE/warehouse-transforms 28 | 29 | # Create a new git branch 30 | now=$(date +%Y_%m_%d_%H_%M_%S) 31 | branchname="builder_$now" 32 | git checkout -b "$branchname" 33 | 34 | # Run the dbt script to update schemas and sql, from the source project directory (necessary for dbt to run) 35 | cd $WORKSPACE/warehouse-transforms/projects/$SOURCE_PROJECT 36 | dbt_schema_builder build --destination-project $DESTINATION_PROJECT --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ 37 | 38 | # Check if any files are added, deleted, or changed. If so, commit them and create a PR. 39 | if [[ -z $(git status -s) ]] 40 | then 41 | echo "No changes to commit." 42 | else 43 | # ssh -vT git@github.com 44 | git config --global user.email "edx-analytics-automation@edx.org" 45 | git config --global user.name "edX Analytics Automation" 46 | 47 | # Commit all changes to the new branch, making sure new files are added 48 | git add --all 49 | git commit --message "chore: Schema Builder automated dbt update at $now" 50 | 51 | # Create a PR on Github from the new branch 52 | HUB_PROTOCOL=ssh /snap/bin/hub pull-request --push --no-edit -r edx/edx-data-engineering 53 | fi 54 | -------------------------------------------------------------------------------- /dataeng/resources/snowflake-user-retirement-status-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating Python virtual env 5 | PYTHON_VENV="python_venv" 6 | virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" 7 | source "${PYTHON_VENV}/bin/activate" 8 | 9 | # Setup 10 | cd $WORKSPACE/analytics-tools/snowflake 11 | make requirements 12 | 13 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_stitch_loader.p8 -v rsa_key_stitch_loader 14 | python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_stitch_loader -v rsa_key_passphrase_stitch_loader 15 | 16 | python retirement_cleanup.py \ 17 | --user $USER \ 18 | --account $ACCOUNT \ 19 | --key_file rsa_key_stitch_loader \ 20 | --passphrase_file rsa_key_passphrase_stitch_loader 21 | 22 | rm rsa_key_stitch_loader 23 | rm rsa_key_passphrase_stitch_loader 24 | -------------------------------------------------------------------------------- /dataeng/resources/update-users.sh: -------------------------------------------------------------------------------- 1 | VENV_ROOT=$WORKSPACE/venvs 2 | mkdir -p $VENV_ROOT 3 | 4 | virtualenv --python=python3.8 $VENV_ROOT/analytics-configuration 5 | . $VENV_ROOT/analytics-configuration/bin/activate 6 | make -C analytics-configuration users.update 7 | -------------------------------------------------------------------------------- /dataeng/resources/user-retirement-bulk-status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Display the environment variables again, this time within the context of a 6 | # new subshell inside of the job. N.B. this should not print plain credentials 7 | # because the only credentialsBindings we currently use is of type "file" which 8 | # just stores a filename in the environment (rather than the content). 9 | env 10 | 11 | # Create and activate a virtualenv. In case we ever change the concurrency 12 | # setting on the jenkins worker, it would be safest to keep the builds from 13 | # clobbering each other's virtualenvs. 14 | VENV="venv-${BUILD_NUMBER}" 15 | virtualenv --python=python3.9 --clear "${VENV}" 16 | source "${VENV}/bin/activate" 17 | 18 | # Make sure that when we try to write unicode to the console, it 19 | # correctly encodes to UTF-8 rather than exiting with a UnicodeEncode 20 | # error. 21 | export PYTHONIOENCODING=UTF-8 22 | export LC_CTYPE=en_US.UTF-8 23 | 24 | #Fetch secrets from AWS 25 | cd $WORKSPACE/configuration 26 | pip install -r util/jenkins/requirements.txt 27 | # hide the sensitive information in the logs 28 | set +x 29 | 30 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 31 | 32 | # Create a temporary file to store the YAML 33 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 34 | 35 | # Write the YAML data to the temporary file 36 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 37 | 38 | set -x 39 | 40 | # prepare tubular 41 | cd $WORKSPACE/tubular 42 | # snapshot the current latest versions of pip and setuptools. 43 | pip install 'pip==21.0.1' 'setuptools==53.0.0' 44 | pip install -r requirements.txt 45 | 46 | # Call the script to collect the list of learners that are to be retired. 47 | python scripts/retirement_bulk_status_update.py \ 48 | --config_file=$TEMP_CONFIG_YAML \ 49 | --start_date=$START_DATE \ 50 | --end_date=$END_DATE \ 51 | --initial_state=$INITIAL_STATE_NAME \ 52 | ${NEW_STATE_NAME:+ "--new_state=$NEW_STATE_NAME"} \ 53 | $(if [[ $REWIND_STATE == "true" ]]; then echo --rewind-state; fi) 54 | 55 | # Remove the temporary file after processing 56 | rm -f "$TEMP_CONFIG_YAML" -------------------------------------------------------------------------------- /dataeng/resources/user-retirement-collector.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Display the environment variables again, this time within the context of a 6 | # new subshell inside of the job. N.B. this should not print plain credentials 7 | # because the only credentialsBindings we currently use is of type "file" which 8 | # just stores a filename in the environment (rather than the content). 9 | env 10 | 11 | # Create and activate a virtualenv. In case we ever change the concurrency 12 | # setting on the jenkins worker, it would be safest to keep the builds from 13 | # clobbering each other's virtualenvs. 14 | VENV="venv-${BUILD_NUMBER}" 15 | virtualenv --python=python3.8 --clear "${VENV}" 16 | source "${VENV}/bin/activate" 17 | 18 | # Make sure that when we try to write unicode to the console, it 19 | # correctly encodes to UTF-8 rather than exiting with a UnicodeEncode 20 | # error. 21 | export PYTHONIOENCODING=UTF-8 22 | export LC_CTYPE=en_US.UTF-8 23 | 24 | #Fetch secrets from AWS 25 | cd $WORKSPACE/configuration 26 | pip install -r util/jenkins/requirements.txt 27 | # hide the sensitive information in the logs 28 | set +x 29 | 30 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 31 | 32 | # Create a temporary file to store the YAML 33 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 34 | 35 | # Write the YAML data to the temporary file 36 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 37 | 38 | set -x 39 | 40 | # prepare tubular 41 | cd $WORKSPACE/tubular 42 | # snapshot the current latest versions of pip and setuptools. 43 | pip install 'pip==21.0.1' 'setuptools==53.0.0' 44 | pip install -r requirements.txt 45 | 46 | # Create the directory where we will populate properties files, one per 47 | # downstream build. 48 | rm -rf $LEARNERS_TO_RETIRE_PROPERTIES_DIR 49 | mkdir $LEARNERS_TO_RETIRE_PROPERTIES_DIR 50 | 51 | # Call the script to collect the list of learners that are to be retired. 52 | python scripts/get_learners_to_retire.py \ 53 | --config_file=$TEMP_CONFIG_YAML \ 54 | --output_dir=$LEARNERS_TO_RETIRE_PROPERTIES_DIR \ 55 | --cool_off_days=$COOL_OFF_DAYS \ 56 | --user_count_error_threshold=$USER_COUNT_ERROR_THRESHOLD \ 57 | --max_user_batch_size=$MAX_USER_BATCH_SIZE 58 | 59 | # Remove the temporary file after processing 60 | rm -f "$TEMP_CONFIG_YAML" -------------------------------------------------------------------------------- /dataeng/resources/user-retirement-driver.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Display the environment variables again, this time within the context of a 6 | # new subshell inside of the job. N.B. this should not print plain credentials 7 | # because the only credentialsBindings we currently use is of type "file" which 8 | # just stores a filename in the environment (rather than the content). 9 | env 10 | 11 | # Create and activate a virtualenv. In case we ever change the concurrency 12 | # setting on the jenkins worker, it would be safest to keep the builds from 13 | # clobbering each other's virtualenvs. 14 | VENV="venv-${BUILD_NUMBER}" 15 | virtualenv --python=python3.8 --clear "${VENV}" 16 | source "${VENV}/bin/activate" 17 | 18 | # Make sure that when we try to write unicode to the console, it 19 | # correctly encodes to UTF-8 rather than exiting with a UnicodeEncode 20 | # error. 21 | export PYTHONIOENCODING=UTF-8 22 | export LC_CTYPE=en_US.UTF-8 23 | 24 | #Fetch secrets from AWS 25 | cd $WORKSPACE/configuration 26 | pip install -r util/jenkins/requirements.txt 27 | # hide the sensitive information in the logs 28 | set +x 29 | 30 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 31 | 32 | # Create a temporary file to store the YAML 33 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 34 | 35 | # Write the YAML data to the temporary file 36 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 37 | 38 | set -x 39 | 40 | # prepare tubular 41 | cd $WORKSPACE/tubular 42 | # snapshot the current latest versions of pip and setuptools. 43 | pip install 'pip==21.0.1' 'setuptools==53.0.0' 44 | pip install -r requirements.txt 45 | 46 | # Call the script to retire one learner. This assumes the following build 47 | # parameters / environment variable is set: RETIREMENT_USERNAME. 48 | python scripts/retire_one_learner.py \ 49 | --config_file=$TEMP_CONFIG_YAML 50 | 51 | # Remove the temporary file after processing 52 | rm -f "$TEMP_CONFIG_YAML" 53 | -------------------------------------------------------------------------------- /dataeng/resources/warehouse-transforms-ci-dbt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # This script runs the dbt deps/seed/run/test commands and serve warehouse-transforms-ci.sh script. 5 | # DBT_RDBT_RUN_OPTIONS, DBT_TEST_OPTIONS etc comes from warehouse-transforms-ci.sh 6 | 7 | cd $WORKSPACE/warehouse-transforms/projects/$DBT_PROJECT_PATH 8 | 9 | source $WORKSPACE/secrets-manager.sh 10 | # Fetch the secrets from AWS 11 | set +x 12 | get_secret_value analytics-secure/warehouse-transforms/profiles DBT_TRANSFORMER_CI_PRIVATE_KEY 13 | export PRIVATE_KEY="$DBT_TRANSFORMER_CI_PRIVATE_KEY" 14 | set -x 15 | 16 | 17 | dbt clean --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 18 | dbt deps --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 19 | dbt seed --full-refresh --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 20 | 21 | if [ "$WITH_SNAPSHOT" == "true" ] 22 | then 23 | dbt snapshot --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 24 | fi 25 | 26 | dbt run $DBT_RUN_OPTIONS $DBT_RUN_EXCLUDE --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 27 | # Jenkins jobs are marked as failed when any of command fails so writing the following test command with && true so it will give a chance to 28 | # evaluate its success or failure base on success or failure we can do further re-tries on failed tests 29 | dbt test $DBT_TEST_OPTIONS $DBT_TEST_EXCLUDE --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET && true 30 | if [ $? -eq 1 ] 31 | then 32 | if [ "$WITH_RETRY" == "true" ] 33 | then 34 | pip install -r ../../tools/ci_scripts/requirements.txt 35 | if [ "$DBT_TEST_EXCLUDE" == "" ] 36 | then 37 | python ../../tools/ci_scripts/rerun_flaky_tests.py --project-path . --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE \ 38 | --target $DBT_TARGET --count $NO_OF_TRIES 39 | else 40 | PREFIX="--exclude " 41 | TEST_EXCLUSIONS=$(echo "$DBT_TEST_EXCLUDE" | sed -e "s/^$PREFIX//") 42 | python ../../tools/ci_scripts/rerun_flaky_tests.py --project-path . --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE \ 43 | --target $DBT_TARGET --exclusions $TEST_EXCLUSIONS --count $NO_OF_TRIES 44 | fi 45 | else 46 | echo "Tests failed but retry is not enabled" 47 | exit 1 48 | fi 49 | fi 50 | -------------------------------------------------------------------------------- /dataeng/resources/warehouse-transforms-ci-manual.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # Creating python 3.11 virtual environment to run dbt warehouse-transform job 5 | PYTHON311_VENV="py311_venv" 6 | virtualenv --python=python3.11 --clear "${PYTHON311_VENV}" 7 | source "${PYTHON311_VENV}/bin/activate" 8 | 9 | # Specifying GITHUB_PR_ID and WAREHOUSE_TRANSFORMS_BRANCH is a must 10 | if [[ "$GITHUB_PR_ID" == "" || "$WAREHOUSE_TRANSFORMS_BRANCH" == "" ]] 11 | then 12 | echo "Please provide GITHUB_PR_ID and WAREHOUSE_TRANSFORMS_BRANCH" 13 | exit 1 14 | fi 15 | 16 | # Setup to run python script to create snowflake schema 17 | cd $WORKSPACE/analytics-tools/snowflake 18 | make requirements 19 | 20 | # Download Prod build manifest.json file from S3 and creating directory to place manifest file. 21 | cd $WORKSPACE/ && mkdir -p manifest 22 | 23 | pip install awscli 24 | 25 | aws s3 cp s3://edx-dbt-docs/manifest.json ${WORKSPACE}/manifest 26 | 27 | # Find the project name to append with CI_SCHEMA_NAME 28 | if echo $DBT_PROJECT_PATH | egrep "reporting" -q; then PROJECT_NAME="reporting"; fi 29 | if echo $DBT_PROJECT_PATH | egrep "automated/applications" -q; then PROJECT_NAME="automated/applications"; fi 30 | if echo $DBT_PROJECT_PATH | egrep "automated/raw_to_source" -q; then PROJECT_NAME="automated/raw_to_source"; fi 31 | if echo $DBT_PROJECT_PATH | egrep "automated/telemetry" -q; then PROJECT_NAME="automated/telemetry"; fi 32 | 33 | 34 | # Setup to run dbt commands 35 | cd $WORKSPACE/warehouse-transforms 36 | 37 | # To install right version of dbt 38 | pip install -r requirements.txt 39 | 40 | cd $WORKSPACE/analytics-tools/snowflake 41 | export CI_SCHEMA_NAME=PR_${GITHUB_PR_ID}_${PROJECT_NAME} 42 | # Schema is dynamically created against each run. 43 | # profiles.yml contains the name of Schema which is used to create output models when dbt runs. 44 | python create_ci_schema.py --key_path $KEY_PATH --passphrase_path $PASSPHRASE_PATH --automation_user $USER --account $ACCOUNT --db_name $DB_NAME --schema_name $CI_SCHEMA_NAME --no_drop_old 45 | 46 | cd $WORKSPACE/warehouse-transforms/projects/$DBT_PROJECT_PATH 47 | 48 | source $WORKSPACE/secrets-manager.sh 49 | # Fetch the secrets from AWS 50 | set +x 51 | get_secret_value analytics-secure/warehouse-transforms/profiles DBT_TRANSFORMER_CI_PRIVATE_KEY 52 | export PRIVATE_KEY="$DBT_TRANSFORMER_CI_PRIVATE_KEY" 53 | set -x 54 | 55 | 56 | dbt clean --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 57 | dbt deps --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 58 | dbt seed --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 59 | 60 | if [[ "$RUN_TESTS_ONLY" != "true" ]] 61 | then 62 | dbt run $DBT_RUN_OPTIONS $DBT_RUN_EXCLUDE --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 63 | fi 64 | 65 | dbt test $DBT_TEST_OPTIONS $DBT_TEST_EXCLUDE --profiles-dir $WORKSPACE/warehouse-transforms/profiles/ --profile $DBT_PROFILE --target $DBT_TARGET 66 | -------------------------------------------------------------------------------- /devops/jobs/AppPermissionsFailure.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | import static org.edx.jenkins.dsl.Constants.common_wrappers 3 | import static org.edx.jenkins.dsl.Constants.common_logrotator 4 | import static org.edx.jenkins.dsl.DevopsConstants.common_read_permissions 5 | 6 | class AppPermissionsFailure { 7 | public static def job = { dslFactory, extraVars -> 8 | dslFactory.job(extraVars.get("FOLDER_NAME","App-Permissions") + "/app-permissions-failure") { 9 | 10 | wrappers common_wrappers 11 | logRotator common_logrotator 12 | 13 | wrappers { 14 | credentialsBinding { 15 | string('GIT_TOKEN',"edx_git_bot_token") 16 | } 17 | } 18 | 19 | parameters { 20 | stringParam('ENVIRONMENT') 21 | stringParam('DEPLOYMENT') 22 | stringParam('JOB_TYPE') 23 | stringParam('GIT_PREVIOUS_COMMIT_1') 24 | stringParam('GIT_COMMIT_1') 25 | stringParam('UPSTREAM_BUILD_URL') 26 | stringParam('TUBULAR_BRANCH', 'master', 'Repo branch for the tubular scripts.') 27 | } 28 | 29 | 30 | scm{ 31 | git { 32 | remote { 33 | url('https://github.com/edx/tubular.git') 34 | branch('$TUBULAR_BRANCH') 35 | } 36 | extensions { 37 | cleanAfterCheckout() 38 | pruneBranches() 39 | relativeTargetDirectory('tubular') 40 | } 41 | } 42 | } 43 | 44 | steps { 45 | shell(dslFactory.readFileFromWorkspace('devops/resources/app-permission-runner-failure.sh')) 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /devops/jobs/CheckRDSConfigs.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | import static org.edx.jenkins.dsl.Constants.common_wrappers 3 | 4 | class CheckRDSConfigs { 5 | public static def job = { dslFactory, extraVars -> 6 | assert extraVars.containsKey("DEPLOYMENTS") : "Please define DEPLOYMENTS. It should be list of strings." 7 | assert !(extraVars.get("DEPLOYMENTS") instanceof String) : "Make sure DEPLOYMENTS is a list and not a string" 8 | 9 | extraVars.get('DEPLOYMENTS').each { deployment, configuration -> 10 | 11 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/check-rds-configs-${deployment}") { 12 | parameters { 13 | stringParam('CONFIGURATION_REPO', 'https://github.com/edx/configuration.git') 14 | stringParam('CONFIGURATION_BRANCH', 'master') 15 | } 16 | 17 | wrappers common_wrappers 18 | 19 | wrappers { 20 | credentialsBinding { 21 | def variable = "check-rds-slow-query-logs-${deployment}" 22 | string("ROLE_ARN", variable) 23 | } 24 | } 25 | 26 | triggers { 27 | cron('H 13 * * 1-5 ') 28 | } 29 | 30 | def ignore_options = "" 31 | configuration.IGNORE_LIST.each { db -> 32 | ignore_options = "${ignore_options}--ignore ${db} " 33 | } 34 | 35 | environmentVariables { 36 | env('AWS_DEFAULT_REGION', extraVars.get('REGION')) 37 | env('IGNORE_OPTIONS', ignore_options) 38 | } 39 | 40 | multiscm { 41 | git { 42 | remote { 43 | url('$CONFIGURATION_REPO') 44 | branch('$CONFIGURATION_BRANCH') 45 | } 46 | extensions { 47 | cleanAfterCheckout() 48 | pruneBranches() 49 | relativeTargetDirectory('configuration') 50 | } 51 | } 52 | } 53 | steps { 54 | shell(dslFactory.readFileFromWorkspace('devops/resources/check-rds-configs.sh')) 55 | 56 | } 57 | 58 | publishers { 59 | extendedEmail { 60 | recipientList(extraVars.get('NOTIFY_ON_FAILURE')) 61 | triggers { 62 | failure { 63 | attachBuildLog(false) // build log contains PII! 64 | compressBuildLog(false) // build log contains PII! 65 | subject('Failed build: ${JOB_NAME} #${BUILD_NUMBER}') 66 | content('Jenkins job: ${JOB_NAME} failed. \nFor' + " ${deployment} " + 'Environment. \n\nSee ${BUILD_URL} for details.') 67 | contentType('text/plain') 68 | sendTo { 69 | recipientList() 70 | } 71 | } 72 | } 73 | } 74 | } 75 | } 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /devops/jobs/CheckRetireUsers.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | import static org.edx.jenkins.dsl.Constants.common_wrappers 3 | 4 | class CheckRetireUsers { 5 | public static def job = { dslFactory, extraVars -> 6 | assert extraVars.containsKey("DEPLOYMENTS") : "Please define DEPLOYMENTS. It should be list of strings." 7 | assert !(extraVars.get("DEPLOYMENTS") instanceof String) : "Make sure DEPLOYMENTS is a list of string" 8 | 9 | extraVars.get('DEPLOYMENTS').each { deployment , configuration -> 10 | configuration.environments.each { environment -> 11 | 12 | 13 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/check-retire-users-for-${deployment}-${environment}") { 14 | parameters { 15 | stringParam('CONFIGURATION_REPO', 'https://github.com/edx/configuration.git') 16 | stringParam('CONFIGURATION_BRANCH', 'master') 17 | } 18 | 19 | wrappers common_wrappers 20 | 21 | wrappers { 22 | credentialsBinding { 23 | usernamePassword("DB_USER", "DB_PASSWORD", "${deployment}-${environment}-users-retire-credentials") 24 | def variable = "${deployment}-retired-users-certs" 25 | string("ROLE_ARN", variable) 26 | } 27 | } 28 | 29 | environmentVariables { 30 | env('ENVIRONMENT', environment) 31 | env('DEPLOYMENT', deployment) 32 | env('AWS_DEFAULT_REGION', extraVars.get('REGION')) 33 | } 34 | 35 | multiscm { 36 | git { 37 | remote { 38 | url('$CONFIGURATION_REPO') 39 | branch('$CONFIGURATION_BRANCH') 40 | } 41 | extensions { 42 | cleanAfterCheckout() 43 | pruneBranches() 44 | relativeTargetDirectory('configuration') 45 | } 46 | } 47 | } 48 | steps { 49 | shell(dslFactory.readFileFromWorkspace('devops/resources/check_retire_users.sh')) 50 | } 51 | 52 | publishers { 53 | mailer(extraVars.get('NOTIFY_ON_FAILURE'), false, false) 54 | } 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /devops/jobs/CheckTableSize.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | import static org.edx.jenkins.dsl.Constants.common_wrappers 3 | 4 | class CheckTableSize { 5 | public static def job = { dslFactory, extraVars -> 6 | assert extraVars.containsKey("DEPLOYMENTS") : "Please define DEPLOYMENTS. It should be list of strings." 7 | assert extraVars.containsKey("IGNORE_LIST") : "Please define IGNORE_LIST. It should be list of strings." 8 | assert !(extraVars.get("DEPLOYMENTS") instanceof String) : "Make sure DEPLOYMENTS is a list of string" 9 | 10 | extraVars.get('DEPLOYMENTS').each { deployment , configuration -> 11 | configuration.environments.each { environment, rds_config -> 12 | 13 | 14 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/table-size-monitoring-${deployment}") { 15 | parameters { 16 | stringParam('CONFIGURATION_REPO', 'https://github.com/edx/configuration.git') 17 | stringParam('CONFIGURATION_BRANCH', 'master') 18 | } 19 | 20 | wrappers common_wrappers 21 | 22 | wrappers { 23 | credentialsBinding { 24 | usernamePassword("USERNAME", "PASSWORD", "${deployment}-table-size-credentials") 25 | def variable = "${deployment}-table-size-monitoring" 26 | string("ROLE_ARN", variable) 27 | string("GENIE_KEY", "opsgenie_heartbeat_key") 28 | string("DD_KEY", "datadog_heartbeat_key") 29 | } 30 | } 31 | 32 | triggers { 33 | cron("H * * * *") 34 | } 35 | 36 | def rdsthreshold = "" 37 | rds_config.rds.each { rds, threshold -> 38 | rdsthreshold = "${rdsthreshold}--rdsthreshold ${rds} ${threshold} " 39 | } 40 | 41 | def rdsignore = "" 42 | extraVars.get('IGNORE_LIST').each { ignore -> 43 | rdsignore = "${rdsignore}-i ${ignore} " 44 | } 45 | 46 | environmentVariables { 47 | env('AWS_DEFAULT_REGION', extraVars.get('REGION')) 48 | env('THRESHOLD', extraVars.get('THRESHOLD')) 49 | env('RDSTHRESHOLD', rdsthreshold) 50 | env('RDSIGNORE', rdsignore) 51 | env('DEPLOYMENT', deployment) 52 | } 53 | 54 | multiscm { 55 | git { 56 | remote { 57 | url('$CONFIGURATION_REPO') 58 | branch('$CONFIGURATION_BRANCH') 59 | } 60 | extensions { 61 | cleanAfterCheckout() 62 | pruneBranches() 63 | relativeTargetDirectory('configuration') 64 | } 65 | } 66 | } 67 | steps { 68 | shell(dslFactory.readFileFromWorkspace('devops/resources/table-size-monitoring.sh')) 69 | 70 | } 71 | } 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /devops/jobs/CloudFlareHitRate.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Variables without defaults are marked (required) 4 | 5 | Variables consumed for this job: 6 | * NOTIFY_ON_FAILURE: (Required - email address) 7 | * SECURE_GIT_CREDENTIALS: (Required - jenkins name of git credentials) 8 | * ZONE_ID: (Required - The CloudFlare Zone ID) 9 | * AUTH_KEY: (Required - Authentication key for the account that would make API calls) 10 | * EMAIL: (Required - Email of the account for making API calls) 11 | 12 | */ 13 | package devops.jobs 14 | 15 | 16 | import static org.edx.jenkins.dsl.Constants.common_wrappers 17 | import static org.edx.jenkins.dsl.Constants.common_logrotator 18 | 19 | 20 | class CloudFlareHitRate { 21 | public static def job = { dslFactory, extraVars -> 22 | assert extraVars.containsKey('ZONE_ID') : "Required ZONE_ID setting missing from configuration" 23 | assert extraVars.containsKey('EMAIL') : "Required email(EMAIL) setting missing from configuration" 24 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/cloudflare-hit-rate-edx") { 25 | wrappers common_wrappers 26 | logRotator common_logrotator 27 | 28 | wrappers { 29 | credentialsBinding { 30 | string('AUTH_KEY', 'AUTH_KEY') 31 | } 32 | } 33 | 34 | environmentVariables { 35 | env('ZONE_ID', extraVars.get('ZONE_ID')) 36 | env('EMAIL', extraVars.get('EMAIL')) 37 | env('THRESHOLD', extraVars.get('THRESHOLD')) 38 | } 39 | 40 | triggers { 41 | cron("H */1 * * *") 42 | } 43 | 44 | steps { 45 | shell(dslFactory.readFileFromWorkspace('devops/resources/cloudflare-hit-rate.sh')) 46 | 47 | 48 | } 49 | 50 | multiscm{ 51 | git { 52 | remote { 53 | url('https://github.com/edx/configuration.git') 54 | branch('master') 55 | } 56 | extensions { 57 | cleanAfterCheckout() 58 | pruneBranches() 59 | relativeTargetDirectory('configuration') 60 | } 61 | } 62 | } 63 | 64 | publishers { 65 | mailer(extraVars.get('NOTIFY_ON_FAILURE'), false, false) 66 | } 67 | 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /devops/jobs/ConfigChecker.groovy: -------------------------------------------------------------------------------- 1 | class ConfigChecker extends RunLocalAnsiblePlaybook { 2 | public def post_ansible_steps() { 3 | shell(dslFactory.readFileFromWorkspace('devops/resources/syntax-check-config.sh')) 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /devops/jobs/CreateSandboxCNAME.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This job creates a CreateSandboxCNAME job which allows you to set an alias 4 | for your existing sandbox. 5 | 6 | Variables consumed from the EXTRA_VARS input to your seed job in addition 7 | to those listed in the seed job. 8 | 9 | * FOLDER_NAME: "Sandboxes" 10 | * ACCESS_CONTROL: List of org or org*team from GitHub who get access to the jobs 11 | 12 | This job expects the following credentials to be defined on the folder 13 | 14 | sandbox-jenkins-aws-credentials: file with key/secret in boto config format 15 | sandbox-role-arn: the role to aws sts assume-role 16 | */ 17 | package devops.jobs 18 | 19 | import static org.edx.jenkins.dsl.DevopsConstants.common_wrappers 20 | import static org.edx.jenkins.dsl.DevopsConstants.common_logrotator 21 | import static org.edx.jenkins.dsl.DevopsConstants.common_read_permissions 22 | 23 | class CreateSandboxCNAME { 24 | public static def job = { dslFactory, extraVars -> 25 | return dslFactory.job(extraVars.get("FOLDER_NAME","Sandboxes") + "/CreateSandboxCNAME") { 26 | 27 | description("Sets a DNS Alias for your sandbox") 28 | 29 | wrappers common_wrappers 30 | 31 | def access_control = extraVars.get('ACCESS_CONTROL',[]) 32 | access_control.each { acl -> 33 | common_read_permissions.each { perm -> 34 | authorization { 35 | permission(perm,acl) 36 | } 37 | } 38 | } 39 | 40 | wrappers { 41 | credentialsBinding { 42 | string('ROLE_ARN','sandbox-role-arn') 43 | } 44 | } 45 | 46 | logRotator common_logrotator 47 | 48 | multiscm { 49 | git { 50 | remote { 51 | url('https://github.com/edx/configuration.git') 52 | branch('master') 53 | } 54 | extensions { 55 | cleanAfterCheckout() 56 | pruneBranches() 57 | } 58 | } 59 | } 60 | 61 | parameters { 62 | stringParam("dns_name","","Required - cname to create in the dns_zone.
\nExample: test will create test.sandbox.edx.org") 63 | stringParam("sandbox",'${BUILD_USER_ID}.sandbox.edx.org',"Optional - Only change this if you want to alias a sandbox other than your default sandbox") 64 | } 65 | 66 | // We don't allow any other access from this box, so making this configurable is unnecesary 67 | environmentVariables { 68 | env('dns_zone','sandbox.edx.org') 69 | } 70 | 71 | properties { 72 | rebuild { 73 | autoRebuild(false) 74 | rebuildDisabled(false) 75 | } 76 | } 77 | 78 | concurrentBuild() 79 | 80 | steps { 81 | shell(dslFactory.readFileFromWorkspace('devops/resources/create-sandbox-cname.sh')) 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /devops/jobs/CreateSandboxSSHAccess.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This job creates a GrantSSHAccess job which adds an additional 4 | use to your sandbox. 5 | 6 | Variables consumed from the EXTRA_VARS input to your seed job in addition 7 | to those listed in the seed job. 8 | 9 | * FOLDER_NAME: "Sandboxes" 10 | * ACCESS_CONTROL: List of org or org*team from GitHub who get access to the jobs 11 | * SSH_USER: ssh username that we can use to access the sandbox and sudo to add a user 12 | 13 | This job expects the sandbox-ssh-keys credential to contain an ssh key it can user 14 | to access any sandbox. 15 | */ 16 | package devops.jobs 17 | 18 | import static org.edx.jenkins.dsl.DevopsConstants.common_wrappers 19 | import static org.edx.jenkins.dsl.DevopsConstants.common_logrotator 20 | import static org.edx.jenkins.dsl.DevopsConstants.common_read_permissions 21 | 22 | class CreateSandboxSSHAccess { 23 | public static def job = { dslFactory, extraVars -> 24 | return dslFactory.job(extraVars.get("FOLDER_NAME","Sandboxes") + "/GrantSSHAccess") { 25 | 26 | description('Give ssh access to a github user.\nThen to log in:\n$ ssh YOUR_GITHUB_USERNAME@THE_SANDBOX_HOST_NAME') 27 | 28 | 29 | wrappers common_wrappers 30 | 31 | def access_control = extraVars.get('ACCESS_CONTROL',[]) 32 | access_control.each { acl -> 33 | common_read_permissions.each { perm -> 34 | authorization { 35 | permission(perm,acl) 36 | } 37 | } 38 | } 39 | 40 | logRotator common_logrotator 41 | 42 | multiscm { 43 | git { 44 | remote { 45 | url('https://github.com/edx/configuration.git') 46 | branch('master') 47 | } 48 | extensions { 49 | cleanAfterCheckout() 50 | pruneBranches() 51 | } 52 | } 53 | } 54 | 55 | parameters { 56 | stringParam("user","","Required - - github user to grant ssh access") 57 | stringParam("sandbox",'${BUILD_USER_ID}.sandbox.edx.org',"Optional - Only change this if you want to give access to a sandbox other than your default sandbox") 58 | } 59 | 60 | properties { 61 | rebuild { 62 | autoRebuild(false) 63 | rebuildDisabled(false) 64 | } 65 | } 66 | 67 | concurrentBuild() 68 | 69 | wrappers { 70 | sshAgent('sandbox-ssh-keys') 71 | } 72 | 73 | authenticationToken(extraVars.get('JENKINS_SANDBOX_JOB_KEY')) 74 | 75 | environmentVariables { 76 | env('SSH_USER',extraVars.get('SSH_USER','ubuntu')) 77 | } 78 | 79 | steps { 80 | shell(dslFactory.readFileFromWorkspace('devops/resources/create-sandbox-ssh-access.sh')) 81 | } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /devops/jobs/DeleteMergedGitBranches.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | Variables without defaults are marked (required) 3 | 4 | Variables consumed for this job: 5 | * SECURE_GIT_CREDENTIALS: secure-bot-user (required) 6 | * SSH_AGENT_KEY : ssh-credential-name 7 | * NOTIFY_ON_FAILURE: alert@example.com 8 | * FOLDER_NAME: folder, default is Monitoring 9 | */ 10 | 11 | package devops.jobs 12 | import static org.edx.jenkins.dsl.Constants.common_wrappers 13 | import static org.edx.jenkins.dsl.Constants.common_logrotator 14 | 15 | class DeleteMergedGitBranches{ 16 | public static def job = { dslFactory, extraVars -> 17 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/delete-merged-git-branches") { 18 | wrappers common_wrappers 19 | logRotator common_logrotator 20 | 21 | def gitCredentialId = extraVars.get('SECURE_GIT_CREDENTIALS','') 22 | 23 | multiscm{ 24 | git { 25 | remote { 26 | url('git@github.com:openedx/edx-platform.git') 27 | branch('master') 28 | if (gitCredentialId) { 29 | credentials(gitCredentialId) 30 | } 31 | } 32 | extensions { 33 | cleanAfterCheckout() 34 | pruneBranches() 35 | relativeTargetDirectory('edx-platform') 36 | } 37 | } 38 | } 39 | 40 | triggers { 41 | cron("H 15 * * 7") 42 | } 43 | 44 | steps { 45 | shell(dslFactory.readFileFromWorkspace('devops/resources/delete-merged-git-branches.sh')) 46 | } 47 | 48 | if (extraVars.get('NOTIFY_ON_FAILURE')){ 49 | publishers { 50 | extendedEmail { 51 | recipientList(extraVars.get('NOTIFY_ON_FAILURE')) 52 | triggers { 53 | failure { 54 | attachBuildLog(false) // build log contains PII! 55 | compressBuildLog(false) // build log contains PII! 56 | subject('Failed build: ${JOB_NAME} #${BUILD_NUMBER}') 57 | content('Jenkins job: ${JOB_NAME} failed.\n\nSee ${BUILD_URL} for details.') 58 | contentType('text/plain') 59 | sendTo { 60 | recipientList() 61 | } 62 | } 63 | } 64 | } 65 | } 66 | } 67 | wrappers { 68 | sshAgent(extraVars.get("SSH_AGENT_KEY")) 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /devops/jobs/DockerCleanup.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | Removes all unused resources (images, volumes, containers, networks) 3 | Vars consumed for this job: 4 | * NOTIFY_ON_FAILURE: alert@example.com 5 | * FOLDER_NAME: folder 6 | */ 7 | 8 | package devops.jobs 9 | import static org.edx.jenkins.dsl.Constants.common_logrotator 10 | import static org.edx.jenkins.dsl.Constants.common_wrappers 11 | 12 | class DockerCleanup{ 13 | public static def job = { dslFactory, extraVars -> 14 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/docker-cleanup") { 15 | 16 | logRotator common_logrotator 17 | wrappers common_wrappers 18 | 19 | triggers{ 20 | cron("H 0 * * *") 21 | } 22 | 23 | steps { 24 | shell('docker system prune -f') 25 | } 26 | 27 | if (extraVars.get('NOTIFY_ON_FAILURE')){ 28 | publishers { 29 | extendedEmail { 30 | recipientList(extraVars.get('NOTIFY_ON_FAILURE')) 31 | triggers { 32 | failure { 33 | attachBuildLog(false) // build log contains PII! 34 | compressBuildLog(false) // build log contains PII! 35 | subject('Failed build: ${JOB_NAME} #${BUILD_NUMBER}') 36 | content('Jenkins job: ${JOB_NAME} failed.\n\nSee ${BUILD_URL} for details.') 37 | contentType('text/plain') 38 | sendTo { 39 | recipientList() 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /devops/jobs/ExportRDSSlowQueryLogs.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | import static org.edx.jenkins.dsl.Constants.common_wrappers 3 | 4 | class ExportRDSSlowQueryLogs { 5 | public static def job = { dslFactory, extraVars -> 6 | assert extraVars.containsKey("DEPLOYMENTS") : "Please define DEPLOYMENTS. It should be list of strings." 7 | assert extraVars.containsKey("IGNORE_LIST") : "Please define IGNORE_LIST. It should be list of strings." 8 | assert !(extraVars.get("DEPLOYMENTS") instanceof String) : "Make sure DEPLOYMENTS is a list of string" 9 | 10 | extraVars.get('DEPLOYMENTS').each { deployment, configuration -> 11 | configuration.environments.each { environment -> 12 | 13 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/export-slow-query-logs-${deployment}-${environment}") { 14 | parameters { 15 | stringParam('CONFIGURATION_REPO', 'https://github.com/edx/configuration.git') 16 | stringParam('CONFIGURATION_BRANCH', 'master') 17 | } 18 | 19 | wrappers common_wrappers 20 | 21 | wrappers { 22 | credentialsBinding { 23 | usernamePassword("USERNAME", "PASSWORD", "${deployment}-${environment}-export-slow-logs-credentials") 24 | def variable = "${deployment}-export-slow-query-logs" 25 | string("ROLE_ARN", variable) 26 | } 27 | } 28 | 29 | triggers { 30 | cron("0 * * * *") 31 | } 32 | 33 | def rdsignore = "" 34 | extraVars.get('IGNORE_LIST').each { ignore -> 35 | rdsignore = "${rdsignore}-i ${ignore} " 36 | } 37 | 38 | def whitelistregions = "" 39 | configuration.REGION_LIST.each { include -> 40 | whitelistregions = "${whitelistregions}-r ${include} " 41 | } 42 | 43 | environmentVariables { 44 | env('AWS_DEFAULT_REGION', extraVars.get('REGION')) 45 | env('ENVIRONMENT', environment) 46 | env('RDSIGNORE', rdsignore) 47 | env('WHITELISTREGIONS', whitelistregions) 48 | } 49 | 50 | multiscm { 51 | git { 52 | remote { 53 | url('$CONFIGURATION_REPO') 54 | branch('$CONFIGURATION_BRANCH') 55 | } 56 | extensions { 57 | cleanAfterCheckout() 58 | pruneBranches() 59 | relativeTargetDirectory('configuration') 60 | } 61 | } 62 | } 63 | steps { 64 | shell(dslFactory.readFileFromWorkspace('devops/resources/export-slow-query-logs.sh')) 65 | } 66 | 67 | publishers { 68 | mailer(extraVars.get('NOTIFY_ON_FAILURE'), false, false) 69 | 70 | } 71 | } 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /devops/jobs/JenkinsHeartbeat.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | Job to check if Jenkins is successfully running 3 | */ 4 | package devops.jobs 5 | import javaposse.jobdsl.dsl.DslFactory 6 | import static org.edx.jenkins.dsl.Constants.common_logrotator 7 | import static org.edx.jenkins.dsl.Constants.common_wrappers 8 | 9 | class JenkinsHeartbeat{ 10 | public static job( DslFactory dslFactory, Map extraVars){ 11 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/jenkins-heartbeat") { 12 | description("Job to check in with Opsgenie heartbeat to make sure that Jenkins is still running.") 13 | 14 | logRotator { 15 | daysToKeep(1) 16 | } 17 | wrappers common_wrappers 18 | 19 | wrappers { 20 | credentialsBinding { 21 | string("GENIE_KEY", "opsgenie_heartbeat_key") 22 | string("DD_KEY", "datadog_heartbeat_key") 23 | } 24 | } 25 | 26 | triggers { 27 | cron("H/5 * * * *") 28 | } 29 | steps { 30 | String opsgenie_heartbeat_name = extraVars.get('OPSGENIE_HEARTBEAT_NAME','') 31 | if (opsgenie_heartbeat_name) { 32 | shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"') 33 | } 34 | String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '') 35 | if (datadog_heartbeat_name) { 36 | String DD_JSON = """ 37 | { 38 | "series": [{ 39 | "metric": "${datadog_heartbeat_name}", 40 | "points": [['\$(date +%s)', 1]], 41 | "type": "gauge" 42 | }] 43 | } 44 | """ 45 | 46 | shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'') 47 | } 48 | } 49 | 50 | } 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /devops/jobs/SAMLSSLExpirationCheck.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | Variables without defaults are marked (required) 3 | 4 | Variables consumed for this job: 5 | * DEPLOYMENTS (required) 6 | deployment: 7 | environments: 8 | environment (required) 9 | saml_secret (required) 10 | secret_key (required) 11 | * SECURE_GIT_CREDENTIALS: secure-bot-user (required) 12 | * MONITORING_SCRIPT_REPO: name of config repo, default is https://github.com/edx/configuration.git 13 | * MONITORING_SCRIPT_REPO_BRANCH: default is master 14 | * REGION: default is us-east-1 15 | * NOTIFY_ON_FAILURE: alert@example.com 16 | * FOLDER_NAME: folder, default is Monitoring 17 | * DAYS: alert if SSL certificate will expire within these days 18 | 19 | */ 20 | 21 | package devops.jobs 22 | import static org.edx.jenkins.dsl.Constants.common_logrotator 23 | 24 | class SAMLSSLExpirationCheck { 25 | public static def job = { 26 | dslFactory, 27 | extraVars -> 28 | assert extraVars.containsKey('DEPLOYMENTS'): "Please define DEPLOYMENTS. It should be a list of strings." 29 | assert!(extraVars.get('DEPLOYMENTS') instanceof String): "Make sure DEPLOYMENTS is a list and not a string" 30 | extraVars.get('DEPLOYMENTS').each { 31 | deployment, 32 | configuration -> 33 | configuration.environments.each { 34 | environment, 35 | inner_config -> 36 | dslFactory.job(extraVars.get("FOLDER_NAME", "Monitoring") + "/saml-ssl-expiration-check-${environment}-${deployment}") { 37 | logRotator common_logrotator 38 | 39 | def gitCredentialId = extraVars.get('SECURE_GIT_CREDENTIALS', '') 40 | 41 | parameters { 42 | stringParam('MONITORING_SCRIPTS_REPO', extraVars.get('MONITORING_SCRIPTS_REPO', 'git@github.com:edx/monitoring-scripts.git'), 43 | 'Git repo containing edX monitoring scripts, which contains the ssl expiration check script.') 44 | stringParam('MONITORING_SCRIPTS_BRANCH', extraVars.get('MONITORING_SCRIPTS_BRANCH', 'master'), 45 | 'e.g. tagname or origin/branchname') 46 | } 47 | 48 | multiscm { 49 | git { 50 | remote { 51 | url('$MONITORING_SCRIPTS_REPO') 52 | branch('$MONITORING_SCRIPTS_BRANCH') 53 | if (gitCredentialId) { 54 | credentials(gitCredentialId) 55 | } 56 | } 57 | extensions { 58 | cleanAfterCheckout() 59 | pruneBranches() 60 | relativeTargetDirectory('monitoring-scripts') 61 | } 62 | } 63 | } 64 | 65 | triggers { 66 | cron("H 15 * * * ") 67 | } 68 | 69 | environmentVariables { 70 | env('REGION', extraVars.get('REGION', 'us-east-1')) 71 | env('DAYS', extraVars.get('DAYS', 90)) 72 | env('SAML_SECRET', inner_config.get('saml_secret')) 73 | env('SECRET_KEY', inner_config.get('secret_key')) 74 | } 75 | 76 | steps { 77 | shell(dslFactory.readFileFromWorkspace('devops/resources/saml-ssl-expiration-check.sh')) 78 | } 79 | 80 | if (extraVars.get('NOTIFY_ON_FAILURE')) { 81 | publishers { 82 | mailer(extraVars.get('NOTIFY_ON_FAILURE'), false, false) 83 | } 84 | } 85 | 86 | } 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /devops/jobs/SandboxCertRenewal.groovy: -------------------------------------------------------------------------------- 1 | package devops.jobs 2 | 3 | import static org.edx.jenkins.dsl.Constants.common_wrappers 4 | import static org.edx.jenkins.dsl.Constants.common_logrotator 5 | 6 | class SandboxCertRenewal { 7 | public static def job = { dslFactory, extraVars -> 8 | assert extraVars.containsKey("DOMAIN") : "Please define DOMAIN (e.g., sandbox.edx.org)" 9 | 10 | def domain = extraVars.get("DOMAIN") 11 | 12 | dslFactory.job(extraVars.get("FOLDER_NAME","Monitoring") + "/sandbox-cert-renew") { 13 | parameters { 14 | stringParam('EMAIL', 'devops@edx.org', 'Email for certbot registration') 15 | stringParam('AWS_REGION', 'us-east-1', 'AWS region') 16 | } 17 | 18 | wrappers common_wrappers 19 | logRotator common_logrotator 20 | 21 | wrappers { 22 | credentialsBinding { 23 | string("ROLE_ARN", "certbot-role-arn") 24 | } 25 | } 26 | 27 | environmentVariables { 28 | env('DOMAIN', domain) 29 | } 30 | 31 | multiscm{ 32 | git { 33 | remote { 34 | url('https://github.com/edx/configuration.git') 35 | branch('master') 36 | } 37 | extensions { 38 | cleanAfterCheckout() 39 | pruneBranches() 40 | relativeTargetDirectory('configuration') 41 | } 42 | } 43 | } 44 | 45 | triggers { 46 | cron("H H 1 */3 *") // 1st day of every 3rd month 47 | } 48 | 49 | steps { 50 | shell(dslFactory.readFileFromWorkspace('devops/resources/sandbox-cert-renew.sh')) 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /devops/resources/add_xqueue_to_dashboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | # Required by click http://click.pocoo.org/5/python3/ 12 | export LC_ALL=C.UTF-8 13 | export LANG=C.UTF-8 14 | 15 | cd $WORKSPACE/configuration/util/jenkins/add_new_xqueues_to_dashboard 16 | pip install -r requirements.txt 17 | env 18 | 19 | . ../assume-role.sh 20 | assume-role ${ROLE_ARN} 21 | python ./add_xqueue_to_dashboard.py --environment ${ENVIRONMENT} --deploy ${DEPLOYMENT} 22 | -------------------------------------------------------------------------------- /devops/resources/app-permission-runner-failure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/tubular 11 | pip install -r requirements.txt 12 | 13 | python scripts/message_prs_in_range.py --org "edx" --repo "app-permissions" --base_sha ${GIT_PREVIOUS_COMMIT_1} --head_sha ${GIT_COMMIT_1} --release "jenkins_failed" --extra_text " on ${ENVIRONMENT}-${DEPLOYMENT}-${JOB_TYPE}. ${UPSTREAM_BUILD_URL}" 14 | -------------------------------------------------------------------------------- /devops/resources/app-permission-runner-success.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/tubular 11 | pip install -r requirements.txt 12 | 13 | python scripts/message_prs_in_range.py --org "edx" --repo "app-permissions" --base_sha ${GIT_PREVIOUS_COMMIT_1} --head_sha ${GIT_COMMIT_1} --release "jenkins" --extra_text " on ${ENVIRONMENT}-${DEPLOYMENT}-${JOB_TYPE}. ${BUILD_URL}" 14 | -------------------------------------------------------------------------------- /devops/resources/bastion-access.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | cd $WORKSPACE/configuration 5 | 6 | pip install -r requirements.txt 7 | 8 | cd playbooks 9 | if [[ "${ENVIRONMENT}" = "qa" ]]; then 10 | ansible-playbook -i "$BASTION_HOST," tools-gp.yml -u ubuntu -e@../../configuration-internal/ansible/vars/${DEPLOYMENT}.yml -e@../../configuration-internal/ansible/vars/${ENVIRONMENT}-${DEPLOYMENT}.yml --tags users 11 | 12 | else 13 | ansible-playbook -i "$BASTION_HOST," tools-gp.yml -u ubuntu -e${USERS_YAML} -e@../../configuration-internal/ansible/vars/${DEPLOYMENT}.yml -e@../../configuration-internal/ansible/vars/${ENVIRONMENT}-${DEPLOYMENT}.yml --tags users 14 | fi 15 | -------------------------------------------------------------------------------- /devops/resources/build-push-app.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # unofficial bash strict mode 4 | set -euxo pipefail 5 | 6 | set +u 7 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 8 | # creates a venv with its location stored in variable "venvpath" 9 | create_virtualenv --python=python${CI_PYTHON_VERSION} --clear 10 | . "$venvpath/bin/activate" 11 | set -u 12 | 13 | openedx_release=${OPENEDX_RELEASE:-master} 14 | tag_name=${TAG_NAME:-latest} 15 | image_tag=edxops/${APP_NAME}:${tag_name} 16 | 17 | # build the Dockerfile of the IDA defined by APP_NAME and tag it with its name; don't use cache 18 | cd configuration 19 | docker build -f docker/build/${APP_NAME}/Dockerfile --build-arg BASE_IMAGE_TAG=${tag_name} --build-arg OPENEDX_RELEASE=${openedx_release} -t ${image_tag} --no-cache . 20 | 21 | # push built image to DockerHub 22 | docker --config $(dirname ${CONFIG_JSON_FILE}) push ${image_tag} 23 | -------------------------------------------------------------------------------- /devops/resources/check-lifecycle-hooks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euox pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/configuration 12 | . util/jenkins/assume-role.sh 13 | 14 | cd $WORKSPACE/sysadmin 15 | pip install -r requirements/base.txt 16 | export AWS_DEFAULT_REGION=us-east-1 17 | 18 | : ${ASG?"Need to set ASG"} 19 | 20 | assume-role ${ROLE_ARN} 21 | 22 | python aws-management/check-lifecycle-hooks.py --asg ${ASG} --hook GetTrackingLogs 23 | -------------------------------------------------------------------------------- /devops/resources/check-rds-configs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -x 11 | 12 | cd $WORKSPACE/configuration 13 | 14 | pip install awscli 15 | 16 | . util/jenkins/assume-role.sh 17 | assume-role ${ROLE_ARN} 18 | 19 | cd $WORKSPACE/configuration/util/check_rds_configs 20 | 21 | pip install -r requirements.txt 22 | 23 | if [[ ! -v IGNORE_OPTIONS ]]; then 24 | IGNORE_OPTIONS="" 25 | fi 26 | 27 | python check_rds_configs.py --db_engine mysql ${IGNORE_OPTIONS} 28 | -------------------------------------------------------------------------------- /devops/resources/check-ses-limits.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/configuration 12 | pip install -r requirements3.txt 13 | env 14 | export EC2_CACHE_PATH="ec2-cache" 15 | . util/jenkins/assume-role.sh 16 | 17 | # Assume the role that will allow call getSesLimits 18 | assume-role ${ROLE_ARN} 19 | 20 | python util/jenkins/check-ses-limits.py\ 21 | --critical ${CRIT_THRESHOLD}\ 22 | --warning ${WARN_THRESHOLD}\ 23 | --region ${REGIONS} 24 | -------------------------------------------------------------------------------- /devops/resources/check_primary_keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/primary_keys 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume role for different envs 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | # Set RDSIGNORE if not set in job, need because we're setting -u 21 | # Otherwise we get an error "RDSIGNORE: unbound variable" 22 | if [[ ! -v RDSIGNORE ]]; then 23 | RDSIGNORE="" 24 | fi 25 | if [[ ! -v WHITELISTREGIONS ]]; then 26 | WHITELISTREGIONS="" 27 | fi 28 | 29 | python ./check_primary_keys.py --environment ${ENVIRONMENT} --deploy ${DEPLOYMENT} --region $AWS_DEFAULT_REGION --recipient $TO_ADDRESS --sender $FROM_ADDRESS ${RDSIGNORE} ${WHITELISTREGIONS} 30 | -------------------------------------------------------------------------------- /devops/resources/check_retire_users.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/retired_user_cert_remover 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume role for different envs 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | python ./retired_user_cert_remover.py -h "${ENVIRONMENT}-${DEPLOYMENT}-edxapp.rds.edx.org" -db wwc --dry-run 21 | -------------------------------------------------------------------------------- /devops/resources/cloudflare-hit-rate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/configuration 12 | pip install -r util/jenkins/requirements-cloudflare.txt 13 | env 14 | 15 | python util/jenkins/cloudflare-hit-rate.py\ 16 | --zone ${ZONE_ID}\ 17 | --auth_key ${AUTH_KEY}\ 18 | --email ${EMAIL}\ 19 | --threshold ${THRESHOLD} 20 | -------------------------------------------------------------------------------- /devops/resources/cloudflare_purge_cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | env 11 | set -x 12 | 13 | cd $WORKSPACE/configuration 14 | pip install -r util/cloudflare/by_origin_purger/requirements.txt 15 | pip install -r requirements.txt 16 | . util/jenkins/assume-role.sh 17 | 18 | assume-role ${ROLE_ARN} 19 | 20 | aws s3 ls s3://${BUCKET} --recursive | awk '{print $4}' > targets 21 | 22 | if [[ "${SITE}" = "https://edx.org" ]]; then 23 | ZONE_ID=${EDX_ZONE_ID} 24 | elif [[ "${SITE}" = "https://edx-cdn.org" ]]; then 25 | ZONE_ID=${EDX_CDN_ZONE_ID} 26 | elif [[ "${SITE}" = "https://edx-video.net" ]]; then 27 | ZONE_ID=${EDX_VIDEO_ZONE_ID} 28 | fi 29 | 30 | if ${CONFIRM_PURGE}; then 31 | python util/cloudflare/by_origin_purger/purger.py\ 32 | --cloudflare_zone_id ${ZONE_ID}\ 33 | --cloudflare_api_key ${AUTH_KEY}\ 34 | --cloudflare_site_url ${SITE}\ 35 | --cloudflare_email ${EMAIL}\ 36 | --origin ${ORIGIN}\ 37 | --target_path targets --confirm 38 | else 39 | python util/cloudflare/by_origin_purger/purger.py\ 40 | --cloudflare_zone_id ${ZONE_ID}\ 41 | --cloudflare_api_key ${AUTH_KEY}\ 42 | --cloudflare_site_url ${SITE}\ 43 | --cloudflare_email ${EMAIL}\ 44 | --origin ${ORIGIN}\ 45 | --target_path targets 46 | fi 47 | -------------------------------------------------------------------------------- /devops/resources/cluster-instance-monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euox pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/configuration 12 | pip install -r requirements.txt 13 | 14 | env 15 | . util/jenkins/assume-role.sh 16 | assume-role ${ROLE_ARN} 17 | 18 | CLUSTER_FILE=$"${WORKSPACE}/configuration-internal/tools-edx-jenkins/cluster-monitoring-triples.yml" 19 | 20 | python util/cluster_instance_monitoring.py --file ${CLUSTER_FILE} --region ${REGION} 21 | -------------------------------------------------------------------------------- /devops/resources/create-asg-notifications.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -x 11 | 12 | export AWS_DEFAULT_REGION='us-east-1' 13 | 14 | cd $WORKSPACE/configuration 15 | 16 | pip install -r requirements3.txt 17 | pip install click 18 | 19 | . util/jenkins/assume-role.sh 20 | 21 | assume-role ${ROLE_ARN} 22 | 23 | 24 | python util/asg_event_notifications_util.py create-asg-event-notifications --topic_arn ${SNS_TOPIC_ARN} --confirm 25 | -------------------------------------------------------------------------------- /devops/resources/create-data-czar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd "$WORKSPACE/configuration" 12 | pip install -r util/jenkins/requirements.txt 13 | . util/jenkins/assume-role.sh 14 | 15 | assume-role ${ROLE_ARN} 16 | 17 | cd util/create_data_czar 18 | 19 | # Create Policy 20 | if [ ${CREATE_ORG} == "true" ]; then 21 | python ./create_org_data_czar_policy.py --org ${ORGANIZATION} 22 | python ./create_data_czar.py --user ${USER_EMAIL} --file "$WORKSPACE/user_gpg_key.gpg" --org ${ORGANIZATION} --creator ${BUILD_USER_ID} 23 | else 24 | # Create User and add to group 25 | python ./create_data_czar.py --user ${USER_EMAIL} --file "$WORKSPACE/user_gpg_key.gpg" --org ${ORGANIZATION} --creator ${BUILD_USER_ID} 26 | fi 27 | -------------------------------------------------------------------------------- /devops/resources/create-pingdom-alerts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/configuration 12 | pip install -r util/pingdom/requirements.txt 13 | env 14 | 15 | python util/pingdom/create_pingdom_alerts.py\ 16 | --pingdom-email ${PINGDOM_EMAIL}\ 17 | --pingdom-password ${PINGDOM_PASSWORD}\ 18 | --pingdom-api-key ${PINGDOM_API_KEY}\ 19 | --alert-config-file ../${PINGDOM_ALERT_CONFIG_FILE} 20 | -------------------------------------------------------------------------------- /devops/resources/create-sandbox-cname.sh: -------------------------------------------------------------------------------- 1 | set +u 2 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 3 | # creates a venv with its location stored in variable "venvpath" 4 | create_virtualenv --python=python3.8 --clear 5 | . "$venvpath/bin/activate" 6 | set -u 7 | 8 | set -xe 9 | 10 | pip install --upgrade pip 11 | pip install -r requirements.txt 12 | 13 | . util/jenkins/assume-role.sh 14 | set +x 15 | assume-role ${ROLE_ARN} 16 | set -x 17 | 18 | cd $WORKSPACE/playbooks 19 | ansible-playbook -c local -i "localhost," -vv create_cname.yml -e "dns_zone=${dns_zone} dns_name=${dns_name} sandbox=${sandbox}" 20 | -------------------------------------------------------------------------------- /devops/resources/create-sandbox-ssh-access.sh: -------------------------------------------------------------------------------- 1 | set +u 2 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 3 | # creates a venv with its location stored in variable "venvpath" 4 | create_virtualenv --python=python3.8 --clear 5 | . "$venvpath/bin/activate" 6 | set -u 7 | 8 | set -xe 9 | 10 | pip install --upgrade pip 11 | pip install -r requirements.txt 12 | 13 | cd playbooks 14 | 15 | if [ -z "${SSH_USER}" ] || [ -z "${USER}" ] || [ "${USER}" == "jenkins" ] ; then 16 | exit 1 17 | else 18 | ansible-playbook --user ${SSH_USER} -i "$sandbox," -e "user=${USER} give_sudo=true USER_FAIL_MISSING_KEYS=true" create_user.yml 19 | fi 20 | -------------------------------------------------------------------------------- /devops/resources/create-sandbox.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -ex 11 | cd configuration 12 | pip install -r requirements.txt 13 | pip install awscli 14 | 15 | # we don't want to print out the temporary session keys 16 | set +x 17 | 18 | 19 | SESSIONID=$(date +"%s") 20 | 21 | RESULT=(`aws sts assume-role --role-arn $ROLE_ARN \ 22 | --role-session-name $SESSIONID --duration-seconds 10800 \ 23 | --query '[Credentials.AccessKeyId,Credentials.SecretAccessKey,Credentials.SessionToken]' \ 24 | --output text`) 25 | 26 | export AWS_ACCESS_KEY_ID=${RESULT[0]} 27 | export AWS_SECRET_ACCESS_KEY=${RESULT[1]} 28 | export AWS_SECURITY_TOKEN=${RESULT[2]} 29 | export AWS_SESSION_TOKEN=${AWS_SECURITY_TOKEN} 30 | 31 | set -x 32 | 33 | bash util/jenkins/ansible-provision.sh 34 | -------------------------------------------------------------------------------- /devops/resources/delete-merged-git-branches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | # 10 | # Script to delete merged branches in edx-platform repository 11 | # Options 12 | # -r dry run 13 | # Example 14 | # ./delete-merged-git-branches.sh 15 | # ./delete-merged-git-branches.sh -r 16 | 17 | dry_run=0 18 | 19 | while getopts 'r' opt; do 20 | case "$opt" in 21 | r) dry_run=1 ;; 22 | *) echo 'error in command line parsing' >&2 23 | exit 1 24 | esac 25 | done 26 | 27 | IGNORE_BRANCHES="open-release|origin/release$|olive" 28 | 29 | cd edx-platform/ 30 | 31 | # this will list branches which are merged into origin/master but not deleted 32 | # loop into all branches except HEAD, origin/master and branches in IGNORE_BRANCHES 33 | for branch in $(git branch -r --merged origin/master | grep -v HEAD | grep -v origin/master | grep -vE "${IGNORE_BRANCHES}"); do 34 | # check if merged branch is older than 1 week 35 | if [ -z "$(git log -1 --since='1 week ago' -s $branch)" ]; then 36 | echo -e "$(git show -s --format="%ai %ar by %an" $branch) $branch" 37 | fi 38 | done | sort -r > branches.txt 39 | 40 | for branch in $(awk '{print $NF}' branches.txt | sed 's|origin/||'); do 41 | if [ "$dry_run" -eq 1 ]; then 42 | echo Would have deleted branch "${branch}" 43 | else 44 | echo Deleting "${branch}" 45 | git push origin --delete "${branch}" 46 | fi 47 | done 48 | -------------------------------------------------------------------------------- /devops/resources/export-dead-locks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/export_dead_locks 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume the role 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | # Set RDSIGNORE if not set in job, need because we're setting -u 21 | # Otherwise we get an error "RDSIGNORE: unbound variable" 22 | if [[ ! -v RDSIGNORE ]]; then 23 | RDSIGNORE="" 24 | fi 25 | if [[ ! -v WHITELISTREGIONS ]]; then 26 | WHITELISTREGIONS="" 27 | fi 28 | 29 | python export_dead_locks.py --environment ${ENVIRONMENT} --hostname ${HOSTNAME} --port ${PORT} --indexname ${INDEXNAME} ${RDSIGNORE} ${WHITELISTREGIONS} 30 | -------------------------------------------------------------------------------- /devops/resources/export-slow-query-logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/export_slow_logs 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume the role 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | # Set RDSIGNORE if not set in job, need because we're setting -u 21 | # Otherwise we get an error "RDSIGNORE: unbound variable" 22 | if [[ ! -v RDSIGNORE ]]; then 23 | RDSIGNORE="" 24 | fi 25 | if [[ ! -v WHITELISTREGIONS ]]; then 26 | WHITELISTREGIONS="" 27 | fi 28 | 29 | python export_slow_query_logs.py --environment ${ENVIRONMENT} ${RDSIGNORE} ${WHITELISTREGIONS} 30 | -------------------------------------------------------------------------------- /devops/resources/janitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | HOME=/edx/var/jenkins 12 | 13 | env 14 | set -x 15 | 16 | cd $WORKSPACE/configuration 17 | pip install -r requirements.txt 18 | . util/jenkins/assume-role.sh 19 | 20 | assume-role ${ROLE_ARN} 21 | 22 | cd $WORKSPACE/jenkins-job-dsl-internal 23 | cd util/janitor 24 | pip install -r requirements.txt 25 | 26 | deny_prospectus=0 27 | 28 | if [ -v DENY_LIST ]; then 29 | for play in "${DENY_LIST}"; do 30 | if [[ "$play" == "prospectus" ]]; then 31 | deny_prospectus=1 32 | break 33 | fi 34 | done 35 | fi 36 | 37 | if [ "$NOOP" = true ]; then 38 | python janitor.py --noop --region $AWS_REGION --cleaner $AWS_CLEANER --log-bucket $S3_LOG_BUCKET --deny-list $DENY_LIST 39 | elif [ "$deny_prospectus" == 1 ]; then 40 | python janitor.py --region $AWS_REGION --cleaner $AWS_CLEANER --log-bucket $S3_LOG_BUCKET --deny-list $DENY_LIST 41 | else 42 | python prospectus-janitor.py --region $AWS_REGION --cleaner $AWS_CLEANER --log-bucket $S3_LOG_BUCKET 43 | fi 44 | 45 | curl -X GET 'https://api.opsgenie.com/v2/heartbeats/'${JOB_NAME##*/}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY} 46 | curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \ 47 | -H "Content-Type: application/json" \ 48 | -d '{ 49 | "series" : [{ 50 | "metric": "'${JOB_NAME##*/}'.heartbeat", 51 | "points": [['"$(date +%s)"', 1]], 52 | "type": "gauge", 53 | "tags": ["deployment:'${DEPLOYMENT}'"] 54 | }] 55 | }' 56 | -------------------------------------------------------------------------------- /devops/resources/list-mysql-process.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/list_mysql_process 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume the role 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | # Set RDSIGNORE if not set in job, need because we're setting -u 21 | # Otherwise we get an error "RDSIGNORE: unbound variable" 22 | if [[ ! -v RDSIGNORE ]]; then 23 | RDSIGNORE="" 24 | fi 25 | if [[ ! -v WHITELISTREGIONS ]]; then 26 | WHITELISTREGIONS="" 27 | fi 28 | 29 | python list_mysql_process.py --environment ${ENVIRONMENT} ${RDSIGNORE} ${WHITELISTREGIONS} 30 | -------------------------------------------------------------------------------- /devops/resources/missing-rds-alarms.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/rds_alarms_checker 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume the role 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | if [[ ! -v IGNORE_OPTIONS ]]; then 21 | IGNORE_OPTIONS="" 22 | fi 23 | if [[ ! -v WHITELISTREGIONS ]]; then 24 | WHITELISTREGIONS="" 25 | fi 26 | 27 | python missing_rds_alarms.py ${IGNORE_OPTIONS} ${WHITELISTREGIONS} 28 | 29 | -------------------------------------------------------------------------------- /devops/resources/mongo-pruner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd $WORKSPACE/edx-platform 12 | pip install -r scripts/structures_pruning/requirements/base.txt 13 | pip install awscli 14 | 15 | set +x 16 | 17 | SESSIONID=$(date +"%s") 18 | 19 | RESULT=(`aws sts assume-role --role-arn $ROLE_ARN \ 20 | --role-session-name $SESSIONID \ 21 | --query '[Credentials.AccessKeyId,Credentials.SecretAccessKey,Credentials.SessionToken]' \ 22 | --output text`) 23 | 24 | export AWS_ACCESS_KEY_ID=${RESULT[0]} 25 | export AWS_SECRET_ACCESS_KEY=${RESULT[1]} 26 | export AWS_SECURITY_TOKEN=${RESULT[2]} 27 | export AWS_SESSION_TOKEN=${AWS_SECURITY_TOKEN} 28 | 29 | set -x 30 | 31 | NAME_TAG="${ENVIRONMENT}-${DEPLOYMENT}-mongo" 32 | IP_ADDRESSES=`aws ec2 describe-instances\ 33 | --filter Name=tag:Name,Values=$NAME_TAG\ 34 | --output text --query 'Reservations[*].Instances[*].PrivateIpAddress'\ 35 | --region us-east-1` 36 | 37 | MONGO_IPS=`echo $IP_ADDRESSES | sed 's/ /,/g'` 38 | 39 | python scripts/structures_pruning/structures.py\ 40 | --database-name ${DATABASE_NAME}\ 41 | --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" make_plan --retain 10 plan.json 42 | 43 | ## ISRE-1377/ISRE-1443 disable running pruner until MySQL functionality has been added to the pruner 44 | ## Module store is in the process of moving the structure id tracking form Mongo to MySQL. 45 | ## Currently (2022-12-16) the active versions are being written to both Mongo and MySQL, but eventually Mongo will stop receiving 46 | ## updates, which will cause the pruner to delete active structures as it currently only looks at Mongo and not MySQL. 47 | ## 48 | ##python scripts/structures.py\ 49 | ## --database-name ${DATABASE_NAME}\ 50 | ## --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" prune --delay 5000 plan.json 51 | -------------------------------------------------------------------------------- /devops/resources/remove-data-czar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd "$WORKSPACE/configuration" 12 | pip install -r util/jenkins/requirements.txt 13 | . util/jenkins/assume-role.sh 14 | 15 | assume-role ${ROLE_ARN} 16 | 17 | cd util/create_data_czar 18 | 19 | # Remove Data Czar 20 | python ./remove_data_czar.py --user ${USER_EMAIL} 21 | -------------------------------------------------------------------------------- /devops/resources/replace-usernames.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -ex 11 | 12 | # Display the environment variables again, this time within the context of a 13 | # new subshell inside of the job. N.B. this should not print plain credentials 14 | # because the only credentialsBindings we currently use is of type "file" which 15 | # just stores a filename in the environment (rather than the content). 16 | env 17 | 18 | # Make sure that when we try to write unicode to the console, it 19 | # correctly encodes to UTF-8 rather than exiting with a UnicodeEncode 20 | # error. 21 | export PYTHONIOENCODING=UTF-8 22 | export LC_CTYPE=en_US.UTF-8 23 | 24 | # prepare tubular 25 | cd $WORKSPACE/edx-platform 26 | pip install -r scripts/user_retirement/requirements/base.txt 27 | 28 | # Call the script to replace the usernames for all users in the CSV file. 29 | python scripts/user_retirement/replace_usernames.py \ 30 | --config_file=$USERNAME_REPLACEMENT_CONFIG_FILE \ 31 | --username_replacement_csv=$WORKSPACE/username_replacements.csv 32 | -------------------------------------------------------------------------------- /devops/resources/run-ansible.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | cd $WORKSPACE/configuration 6 | 7 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 8 | # creates a venv with its location stored in variable "venvpath" 9 | create_virtualenv --python=python3.8 --clear 10 | . "$venvpath/bin/activate" 11 | 12 | pip install -r requirements.txt 13 | pip install awscli 14 | 15 | env 16 | export EC2_CACHE_PATH="ec2-cache" 17 | . util/jenkins/assume-role.sh 18 | set +x 19 | # Assume the role that will allow running ec2.py for getting a dynamic inventory 20 | assume-role ${ROLE_ARN} 21 | set -x 22 | 23 | cd $WORKSPACE/configuration/playbooks 24 | 25 | # Pattern must be supplied explicitly as we take a conservative 26 | # approach given that ec2.py will provide a dynamic inventory 27 | if [[ -n "${PATTERN}" ]]; then 28 | ANSIBLE_PATTERN="${PATTERN}" 29 | else 30 | ANSIBLE_PATTERN="__NONE__" 31 | fi 32 | 33 | if [[ -n "${INVENTORY}" ]]; then 34 | ANSIBLE_INVENTORY="-i ${INVENTORY} " 35 | else 36 | ANSIBLE_INVENTORY="-i ./ec2.py" 37 | fi 38 | 39 | if [[ -n ${CUSTOM_INVENTORY} ]]; then 40 | HOSTS=$($CUSTOM_INVENTORY) 41 | if [[ -n ${HOSTS} ]]; then 42 | ANSIBLE_INVENTORY="-i ${HOSTS}" 43 | else 44 | echo "No HOSTS found from CUSTOM_INVENTORY - refusing to run ansible" 45 | exit 1 46 | fi 47 | fi 48 | 49 | if [[ -n "${BECOME_USER}" ]]; then 50 | ANSIBLE_BECOME=" --become --become-user=${BECOME_USER} " 51 | fi 52 | 53 | # Test if docker shim flag file is present 54 | shim_enabled=true 55 | ansible ${ANSIBLE_PATTERN} ${ANSIBLE_INVENTORY} -u ${ANSIBLE_SSH_USER} ${ANSIBLE_BECOME} -e 'ansible_python_interpreter=/usr/bin/python3' -m ${ANSIBLE_MODULE_NAME} \ 56 | -a 'test -f /edx/etc/docker_shim_enabled' || shim_enabled=false 57 | 58 | echo "Docker shim enabled? $shim_enabled" 59 | 60 | # Use docker shim command if flag file is present 61 | if [[ "$shim_enabled" == "true" ]]; then 62 | command_args="${ANSIBLE_MODULE_ARGS_SHIM}" 63 | else 64 | command_args="${ANSIBLE_MODULE_ARGS}" 65 | fi 66 | 67 | ansible ${ANSIBLE_PATTERN} ${ANSIBLE_INVENTORY} -u ${ANSIBLE_SSH_USER} ${ANSIBLE_BECOME} -e 'ansible_python_interpreter=/usr/bin/python3' -m ${ANSIBLE_MODULE_NAME} \ 68 | -a "${command_args}" 69 | -------------------------------------------------------------------------------- /devops/resources/run-app-permissions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration 11 | pip install -r requirements/pip.txt 12 | pip install -r requirements.txt 13 | pip install -r util/jenkins/requirements.txt 14 | 15 | env 16 | . util/jenkins/assume-role.sh 17 | 18 | assume-role ${ROLE_ARN} 19 | cd $WORKSPACE/configuration/playbooks 20 | 21 | # Function to set ASG and path variables 22 | function setASGAndPath () { 23 | service_name="$1" 24 | if [[ "${service_name}" == "lms" ]] || [[ "${service_name}" == "cms" ]] || [[ "${service_name}" == "edxapp" ]]; then 25 | INVENTORY=$(./active_instances_in_asg.py --asg ${ENVIRONMENT}-${DEPLOYMENT}-worker) 26 | SERVICE_ENV_PATH="/edx/app/edxapp/edxapp_env" 27 | SERVICE_PYTHON_PATH="/edx/app/edxapp/venvs/edxapp/bin/python" 28 | SERVICE_MANAGE_PATH="/edx/bin/manage.edxapp" 29 | else 30 | INVENTORY=$(./active_instances_in_asg.py --asg ${ENVIRONMENT}-${DEPLOYMENT}-${service_name}) 31 | SERVICE_ENV_PATH="/edx/app/${service_name}/${service_name}_env" 32 | SERVICE_PYTHON_PATH="/edx/app/${service_name}/venvs/${service_name}/bin/python" 33 | SERVICE_MANAGE_PATH="/edx/bin/manage.${service_name}" 34 | fi 35 | } 36 | 37 | if [[ "$JOB_TYPE" =~ ^groups-(.+)$ ]]; then 38 | service="${BASH_REMATCH[1]}" 39 | configfile=${WORKSPACE}/app-permissions/groups/${service}.yml 40 | setASGAndPath ${service} 41 | if [[ "${service}" == "lms" ]] || [[ "${service}" == "cms" ]]; then 42 | ANSIBLE_TAG="manage-$JOB_TYPE" 43 | else 44 | ANSIBLE_TAG="manage-groups-ida" 45 | fi 46 | elif [[ "$JOB_TYPE" =~ ^(.+)-users-(.+)$ ]]; then 47 | job_type_prefix="${BASH_REMATCH[1]}" 48 | service="${BASH_REMATCH[2]}" 49 | configfile_relative_path=users/${service}/${ENVIRONMENT}-${DEPLOYMENT}.yml 50 | configfile=${WORKSPACE}/app-permissions/${configfile_relative_path} 51 | setASGAndPath ${service} 52 | if [[ "${service}" == "edxapp" ]]; then 53 | ANSIBLE_TAG="manage-$JOB_TYPE" 54 | else 55 | ANSIBLE_TAG="manage-${job_type_prefix}-users-ida" 56 | fi 57 | 58 | if [[ "${job_type_prefix}" == "recent" ]]; then 59 | current_configfile="${configfile}" 60 | configfile="${WORKSPACE}/recent-${ENVIRONMENT}-${DEPLOYMENT}.yml" 61 | pushd ${WORKSPACE}/app-permissions 62 | ${WORKSPACE}/app-permissions/generate_recent_users.sh ${configfile_relative_path} > ${configfile} 63 | popd 64 | fi 65 | else 66 | echo "Bad job type: ${JOB_TYPE}." 67 | echo "Expected active-users-edxapp, inactive-users-edxapp, recent-users-, inactive-users- or groups-." 68 | exit 1 69 | fi 70 | 71 | if [[ -n ${INVENTORY} ]]; then 72 | ansible-playbook -i ${INVENTORY} manage_edxapp_users_and_groups.yml \ 73 | -e "env_path=${SERVICE_ENV_PATH}" -e "python_path=${SERVICE_PYTHON_PATH}" \ 74 | -e "manage_path=${SERVICE_MANAGE_PATH}" -e "service=${service}" \ 75 | -e@${configfile} -e "group_environment=${ENVIRONMENT}-${DEPLOYMENT}" \ 76 | --user ${USER} --tags ${ANSIBLE_TAG} 77 | else 78 | echo "Skipping ${ENVIRONMENT} ${DEPLOYMENT}, no worker cluster available, get it next time" 79 | fi 80 | -------------------------------------------------------------------------------- /devops/resources/run-local-ansible-playbook.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -ex 11 | 12 | cd $WORKSPACE/configuration 13 | 14 | pip install -r requirements.txt 15 | 16 | cd $WORKSPACE/configuration/playbooks 17 | 18 | if [[ -n ${PLAYBOOK} ]]; then 19 | ANSIBLE_PLAYBOOK=${PLAYBOOK} 20 | else 21 | echo "You must specify PLAYBOOK" 22 | exit 1 23 | fi 24 | 25 | if [[ -n ${TAGS} ]]; then 26 | ANSIBLE_TAGS="--tags ${TAGS}" 27 | else 28 | ANSIBLE_TAGS="" 29 | fi 30 | 31 | if [[ -n ${SKIP_DEPLOYMENT_FILES} ]]; then 32 | ANSIBLE_INCLUDES=$( cat </dev/null 69 | 70 | aws secretsmanager update-secret \ 71 | --region "$AWS_REGION" \ 72 | --secret-id "sandbox-secure/ansible/certs/wildcard.$DOMAIN.key" \ 73 | --secret-string "$KEY_CONTENT" >/dev/null 74 | 75 | echo "Secrets updated successfully." 76 | else 77 | echo "No new certificate generated. Skipping Secrets Manager upload." 78 | fi 79 | -------------------------------------------------------------------------------- /devops/resources/sandbox-termination.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | HOME=/edx/var/jenkins 12 | 13 | env 14 | set -x 15 | 16 | cd $WORKSPACE/configuration 17 | pip install -r requirements.txt 18 | . util/jenkins/assume-role.sh 19 | 20 | assume-role ${ROLE_ARN} 21 | 22 | cd $WORKSPACE/jenkins-job-dsl-internal 23 | cd util/sandbox_terminate 24 | pip install -r requirements.txt 25 | 26 | extra_args="" 27 | if [ "$NOOP" = true ]; then 28 | extra_args="-n" 29 | fi 30 | 31 | python terminate-sandbox.py $extra_args -z $ROUTE53_ZONE -r $AWS_REGION --edx_git_bot_token $EDX_GIT_BOT_TOKEN 32 | 33 | -------------------------------------------------------------------------------- /devops/resources/ssl-expiration-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #we don't use u in the pipefail because Jenkins doesn't set variables and we want to check for that 3 | set -exo pipefail 4 | 5 | set +u 6 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 7 | # creates a venv with its location stored in variable "venvpath" 8 | create_virtualenv --python=python3.8 --clear 9 | . "$venvpath/bin/activate" 10 | set -u 11 | 12 | HOME=/edx/var/jenkins 13 | 14 | env 15 | set -x 16 | 17 | cd $WORKSPACE/configuration 18 | pip install -r requirements.txt 19 | . util/jenkins/assume-role.sh 20 | 21 | assume-role ${ROLE_ARN} 22 | 23 | cd $WORKSPACE/monitoring-scripts 24 | pip install -r requirements/base.txt 25 | cd ssl_expiration_check 26 | 27 | # Set RDSIGNORE if not set in job, need because we're setting -u 28 | # Otherwise we get an error "RDSIGNORE: unbound variable" 29 | if [[ ! -v RDSIGNORE ]]; then 30 | RDSIGNORE="" 31 | fi 32 | 33 | if [[ -n "${FROM_ADDRESS}" && "${TO_ADDRESS}" ]]; then 34 | python ssl-expiration-check.py --region $REGION -d $DAYS -r $TO_ADDRESS -f $FROM_ADDRESS -i $RDSIGNORE 35 | else 36 | python ssl-expiration-check.py --region $REGION -d $DAYS -i $RDSIGNORE 37 | fi 38 | -------------------------------------------------------------------------------- /devops/resources/staff-super-account-audit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | mysql --defaults-extra-file=${MYSQL_CONFIG_FILE} --batch -B -A wwc -e "select id, email, first_name, last_name, last_login, if(is_staff=1,'yes','no') as staff, if(is_superuser=1,'yes','no') as superuser from auth_user where 1=1 and is_staff = 1 or is_superuser = 1 order by superuser desc;" | tr '\t' ',' > ${ENVIRONMENT}_${DEPLOYMENT}_account_report.csv 11 | -------------------------------------------------------------------------------- /devops/resources/syntax-check-config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | set -ex 11 | 12 | FAIL=0 13 | 14 | e_d=${ENVIRONMENT}_${DEPLOYMENT} 15 | 16 | if ! egrep -q -r --include *.json '{{' "${GREP_DIR}"; then 17 | echo "No un-expanded vars in ${e_d}" 18 | else 19 | echo "Found un-expanded vars in ${e_d}" 20 | echo `egrep -r --include *.json '{{' "${GREP_DIR}"` 21 | FAIL=1 22 | fi 23 | 24 | if ! egrep -qi -r --include *.json \'"False"\' "${GREP_DIR}"; then 25 | echo "No quoted False." 26 | else 27 | echo "Found a quoted boolean in ${e_d}" 28 | echo `egrep -qi -r --include *.json "False" "${GREP_DIR}"` 29 | FAIL=1 30 | fi 31 | 32 | if ! egrep -qi -r --include *.json '\"True\"' "${GREP_DIR}"; then 33 | echo "No quoted True." 34 | else 35 | echo "Found a quoted boolean in ${e_d}" 36 | echo `egrep -qi -r --include *.json '\"True\"' "${GREP_DIR}"` 37 | FAIL=1 38 | fi 39 | 40 | for i in `find $WORKSPACE -name "*.yml" | sed '/ansible/d'` 41 | do 42 | /usr/bin/python2.7 -c "import sys,yaml; yaml.load_all(open('$i'))" $i > /dev/null 43 | if [[ $? -ne 0 ]]; then 44 | echo "ERROR parsing $i" 45 | FAIL=1 46 | else 47 | echo "YAML syntax verified" 48 | fi 49 | done 50 | 51 | if [ "$FAIL" -eq 1 ] ; then 52 | echo "Failing..." 53 | exit 1 54 | fi 55 | -------------------------------------------------------------------------------- /devops/resources/table-size-monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | cd $WORKSPACE/configuration/util/jenkins/check_table_size 11 | 12 | pip install -r requirements.txt 13 | . ../assume-role.sh 14 | 15 | # Assume the role 16 | set +x 17 | assume-role ${ROLE_ARN} 18 | set -x 19 | 20 | # Set RDSTHRESHOLD if not set in job, need because we're setting -u 21 | # Otherwise we get an error "RDSTHRESHOLD: unbound variable" 22 | if [[ ! -v RDSTHRESHOLD ]]; then 23 | RDSTHRESHOLD="" 24 | fi 25 | 26 | # Set RDSIGNORE if not set in job, need because we're setting -u 27 | # Otherwise we get an error "RDSIGNORE: unbound variable" 28 | if [[ ! -v RDSIGNORE ]]; then 29 | RDSIGNORE="" 30 | fi 31 | 32 | python check_table_size.py --threshold ${THRESHOLD} ${RDSTHRESHOLD} ${RDSIGNORE} 33 | 34 | curl -X GET 'https://api.opsgenie.com/v2/heartbeats/table-size-monitoring-'${DEPLOYMENT}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY} 35 | curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \ 36 | -H "Content-Type: application/json" \ 37 | -d '{ 38 | "series" : [{ 39 | "metric": "table_size_monitoring_'${DEPLOYMENT}'.heartbeat", 40 | "points": [['"$(date +%s)"', 1]], 41 | "type": "gauge", 42 | "tags": ["deployment:'${DEPLOYMENT}'"] 43 | }] 44 | }' 45 | -------------------------------------------------------------------------------- /devops/resources/trigger-builds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # unofficial bash strict mode 4 | set -euo pipefail 5 | 6 | set +u 7 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 8 | # creates a venv with its location stored in variable "venvpath" 9 | create_virtualenv --python=python${CI_PYTHON_VERSION} --clear 10 | . "$venvpath/bin/activate" 11 | set -u 12 | 13 | function singleline () 14 | { 15 | COUNT=0; 16 | while read LINE; do 17 | if [[ "$COUNT" -gt "0" ]]; then 18 | echo -n "$1"; 19 | fi; 20 | let COUNT=$COUNT+1; 21 | echo -n "$LINE"; 22 | done 23 | } 24 | 25 | # install configuration requirements 26 | pip install -r configuration/requirements.txt 27 | 28 | # set TRAVIS_BUILD_DIR variable to point to root of configuration repository; needed by parsefiles.py 29 | export TRAVIS_BUILD_DIR=${WORKSPACE}/configuration 30 | 31 | # find the intersection of the applications described in the job configuration file and the applications determined 32 | # as necessary to be built via the parsefiles.py script and redirect it into temp_props file to be read in as an 33 | # environment variable TO_BUILD 34 | cd configuration 35 | echo "TO_BUILD=$(comm -12 <(echo ${APPS} | tr " " "\n" | sort) <(git diff --name-only ${GIT_PREVIOUS_COMMIT}...${GIT_COMMIT} | python util/parsefiles.py | tr " " "\n" | sort) | while read play; do echo -n "image-builders/$play-image-builder, "; done)" > ../temp_props 36 | -------------------------------------------------------------------------------- /devops/resources/update-adhoc-reporting.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exo pipefail 4 | 5 | set +u 6 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 7 | # creates a venv with its location stored in variable "venvpath" 8 | create_virtualenv --python=python3.8 --clear 9 | . "$venvpath/bin/activate" 10 | set -u 11 | 12 | env 13 | 14 | cd $WORKSPACE/configuration 15 | 16 | pip install -r requirements.txt 17 | . util/jenkins/assume-role.sh 18 | 19 | set +x 20 | assume-role ${ROLE_ARN} 21 | set -x 22 | 23 | cd playbooks 24 | 25 | environments=(stage prod) 26 | 27 | #update report replica for stage and prod edx 28 | for environment in ${environments[@]}; do 29 | ansible-playbook -i ./ec2.py --limit tag_Name_tools-edx-gp tools-gp.yml -e@../../edx-internal/ansible/vars/edx.yml -e@../../edx-internal/ansible/vars/${environment}-edx.yml \ 30 | -e@../../edx-internal/ansible/vars/ad_hoc_reporting_replica_db_hosts.yml \ 31 | --tags install:code -u ${ANSIBLE_SSH_USER} -D 32 | done 33 | 34 | # update report replica for prod edge 35 | ansible-playbook -i ./ec2.py --limit tag_Name_tools-edx-gp tools-gp.yml -e@../../edge-internal/ansible/vars/edge.yml -e@../../edge-internal/ansible/vars/prod-edge.yml \ 36 | -e@../../edx-internal/ansible/vars/ad_hoc_reporting_replica_db_hosts.yml \ 37 | --tags install:code -u ${ANSIBLE_SSH_USER} -D 38 | 39 | # update users on reporting server 40 | 41 | ansible-playbook -i ./ec2.py --limit tag_Name_tools-edx-gp tools-gp.yml --tags users -e@../../edx-secure/ansible/vars/edx.yml -e@../../edx-internal/ansible/vars/ad_hoc_reporting_users.yml -u ${ANSIBLE_SSH_USER} 42 | -------------------------------------------------------------------------------- /devops/resources/update-data-czar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.8 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | cd "$WORKSPACE/configuration" 12 | pip install -r util/jenkins/requirements.txt 13 | . util/jenkins/assume-role.sh 14 | 15 | assume-role ${ROLE_ARN} 16 | 17 | cd util/create_data_czar 18 | 19 | # Remove Data Czar 20 | python ./remove_data_czar.py --user ${OLD_USER_EMAIL} 21 | 22 | # Create Data Czar 23 | python ./create_data_czar.py --user ${NEW_USER_EMAIL} --file "$WORKSPACE/new_user_gpg_key.gpg" --org ${ORGANIZATION} --creator ${BUILD_USER_ID} 24 | -------------------------------------------------------------------------------- /devops/resources/update-masters-sandbox.sh: -------------------------------------------------------------------------------- 1 | 2 | set +u 3 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 4 | # creates a venv with its location stored in variable "venvpath" 5 | create_virtualenv --python=python3.8 --clear 6 | . "$venvpath/bin/activate" 7 | set -u 8 | 9 | set -xe 10 | 11 | pip install --upgrade pip 12 | pip install -r requirements.txt 13 | 14 | cd playbooks 15 | 16 | if [ -z "${MASTERS_AUTOMATION_CLIENT_ID}" ] || [ -z "${MASTERS_AUTOMATION_CLIENT_SECRET}" ]; then 17 | echo "Error: Missing automation client credentials!" 18 | exit 1 19 | elif [ -z "${program_uuids}" ] || [ -z "${dns_name}" ]; then 20 | echo "Error: Missing dns_name or program_uuids!" 21 | exit 2 22 | fi 23 | 24 | CREDENTIALS="client_id=${MASTERS_AUTOMATION_CLIENT_ID} client_secret=${MASTERS_AUTOMATION_CLIENT_SECRET}" 25 | PARAMS="program_uuids=${program_uuids} dns_name=${dns_name}" 26 | 27 | ansible-playbook --user ubuntu \ 28 | -i "${dns_name}.sandbox.edx.org," \ 29 | -e "${CREDENTIALS} ${PARAMS}" \ 30 | masters_sandbox_update.yml 31 | -------------------------------------------------------------------------------- /devops/resources/user-retirement-archiver.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set +u 4 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 5 | # creates a venv with its location stored in variable "venvpath" 6 | create_virtualenv --python=python3.8 --clear 7 | . "$venvpath/bin/activate" 8 | set -u 9 | 10 | env 11 | set -ex 12 | 13 | cd $WORKSPACE/configuration 14 | pip install -r util/jenkins/requirements.txt 15 | 16 | . util/jenkins/assume-role.sh 17 | 18 | # hide the sensitive information in the logs 19 | set +x 20 | 21 | CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "${SECRET_ARN}" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .) 22 | 23 | # Create a temporary file to store the YAML 24 | TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml) 25 | 26 | # Write the YAML data to the temporary file 27 | echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML" 28 | 29 | set -x 30 | 31 | assume-role ${ROLE_ARN} 32 | 33 | # prepare edx-platform 34 | cd $WORKSPACE/edx-platform 35 | pip install -r scripts/user_retirement/requirements/base.txt 36 | 37 | # In case this is being run without an explicit END_DATE, default to running with "now" - COOL_OFF_DAYS 38 | if [[ ! -v END_DATE ]]; then 39 | END_DATE=$(date --iso --date "$(date --iso) - $COOL_OFF_DAYS days") 40 | fi 41 | 42 | # Call the script to read the retirement statuses from the LMS, send them to S3, and delete them from the LMS. 43 | python scripts/user_retirement/retirement_archive_and_cleanup.py \ 44 | --config_file=$TEMP_CONFIG_YAML \ 45 | --cool_off_days=$COOL_OFF_DAYS \ 46 | --batch_size=$BATCH_SIZE \ 47 | --start_date=$START_DATE \ 48 | --end_date=$END_DATE \ 49 | --dry_run=$DRY_RUN 50 | 51 | # Remove the temporary file after processing 52 | rm -f "$TEMP_CONFIG_YAML" 53 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | jenkins_tools: 4 | image: edxops/tools_jenkins:latest 5 | container_name: jenkins_tools 6 | volumes: 7 | - jenkins_tools:/edx/var/jenkins 8 | - /var/run/docker.sock:/var/run/docker.sock 9 | ports: 10 | - "127.0.0.1:8080:8080" 11 | 12 | volumes: 13 | jenkins_tools: 14 | -------------------------------------------------------------------------------- /experiments/jobs/BackUpOptimizely.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | Pulls current state of experiment code in Optimizely and backs up to git history branch in edx/optimizely-experiments. 3 | 4 | Vars consumed for this job: 5 | * FOLDER_NAME: folder 6 | * NOTIFY_ON_FAILURE: alert@example.com 7 | * OPTIMIZELY_TOKEN: optimizely api token (required) 8 | * SECURE_GIT_CREDENTIALS: secure-bot-user (required) 9 | * SSH_AGENT_KEY : ssh-credential-name 10 | */ 11 | package experiments.jobs 12 | 13 | class BackUpOptimizely { 14 | 15 | public static def job = { dslFactory, extraVars -> 16 | 17 | dslFactory.job(extraVars.get("FOLDER_NAME","Experiments") + "/back-up-optimizely") { 18 | 19 | environmentVariables { 20 | env('GIT_AUTHOR_NAME', extraVars.get('GIT_AUTHOR_NAME')) 21 | env('GIT_AUTHOR_EMAIL', extraVars.get('GIT_AUTHOR_EMAIL')) 22 | env('GIT_COMMITTER_NAME', extraVars.get('GIT_AUTHOR_NAME')) 23 | env('GIT_COMMITTER_EMAIL', extraVars.get('GIT_AUTHOR_EMAIL')) 24 | } 25 | 26 | def gitCredentialId = extraVars.get('SECURE_GIT_CREDENTIALS','') 27 | 28 | wrappers { 29 | credentialsBinding { 30 | string('OPTIMIZELY_TOKEN', 'optimizely-token') 31 | } 32 | sshAgent(gitCredentialId) 33 | } 34 | 35 | multiscm { 36 | git { 37 | remote { 38 | url(extraVars.get('OPTIMIZELY_EXPERIMENTS_REPO')) 39 | branch("history") 40 | if (gitCredentialId) { 41 | credentials(gitCredentialId) 42 | } 43 | } 44 | extensions { 45 | cleanAfterCheckout() 46 | pruneBranches() 47 | relativeTargetDirectory('optimizely-experiments') 48 | } 49 | } 50 | git { 51 | remote { 52 | url("https://github.com/edx/py-opt-cli.git") 53 | branch('master') 54 | } 55 | extensions { 56 | cleanAfterCheckout() 57 | pruneBranches() 58 | relativeTargetDirectory('py-opt-cli') 59 | } 60 | } 61 | } 62 | 63 | 64 | triggers { 65 | // Trigger every hour 66 | cron("H * * * *") 67 | } 68 | 69 | steps { 70 | shell(dslFactory.readFileFromWorkspace("experiments/resources/back_up_optimizely.sh")) 71 | } 72 | 73 | // Alert on call revenue engineer on failure. 74 | publishers { 75 | mailer(extraVars.get('NOTIFY_ON_FAILURE'), false, false) 76 | } 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /experiments/resources/back_up_optimizely.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | set +u 5 | . /edx/var/jenkins/jobvenvs/virtualenv_tools.sh 6 | # creates a venv with its location stored in variable "venvpath" 7 | create_virtualenv --python=python3.6 --clear 8 | . "$venvpath/bin/activate" 9 | set -u 10 | 11 | # Required by click http://click.pocoo.org/5/python3/ 12 | export LC_ALL=C.UTF-8 13 | export LANG=C.UTF-8 14 | export GIT_AUTHOR_NAME 15 | export GIT_AUTHOR_EMAIL 16 | export GIT_COMMITTER_NAME 17 | export GIT_COMMITTER_EMAIL 18 | 19 | 20 | cd $WORKSPACE/py-opt-cli 21 | pip install -r requirements.txt 22 | pip install -e . 23 | env 24 | 25 | cd $WORKSPACE/optimizely-experiments 26 | 27 | # Retry 5 times before giving up on optimizely 28 | for i in {1..5} 29 | do 30 | # Don't sleep the first time through the loop 31 | [ $i -gt 1 ] && sleep 300 32 | 33 | # Save the return-code of the pull command 34 | py-opt-cli pull && s=0 && break || s=$? 35 | done 36 | # Fail the build with the return code of the pull command if it failed after 5 retries 37 | (exit $s) 38 | 39 | git add -A 40 | git diff --cached --quiet || git commit -am "history commit job # ${BUILD_ID}" 41 | git push origin HEAD:history 42 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | jobDslVersion=1.67 2 | jenkinsVersion=2.89.4 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edx/jenkins-job-dsl/af8587323226bf3b2a6f08dc3a272ffa4eed659f/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /openedx.yaml: -------------------------------------------------------------------------------- 1 | # This file describes this Open edX repo, as described in OEP-2: 2 | # https://open-edx-proposals.readthedocs.io/en/latest/oep-0002-bp-repo-metadata.html#specification 3 | owner: jdmulloy 4 | supporting_teams: 5 | - edx/devops 6 | oeps: 7 | oep-2: true 8 | tags: [tools] 9 | -------------------------------------------------------------------------------- /requirements/base.in: -------------------------------------------------------------------------------- 1 | -c constraints.txt 2 | 3 | docker-compose # For launching the Docker containers 4 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with python 3.8 3 | # To update, run: 4 | # 5 | # make upgrade 6 | # 7 | attrs==21.4.0 8 | # via jsonschema 9 | bcrypt==3.2.2 10 | # via paramiko 11 | certifi==2022.6.15 12 | # via requests 13 | cffi==1.15.0 14 | # via 15 | # bcrypt 16 | # cryptography 17 | # pynacl 18 | charset-normalizer==2.0.12 19 | # via requests 20 | cryptography==37.0.2 21 | # via paramiko 22 | distro==1.7.0 23 | # via docker-compose 24 | docker[ssh]==5.0.3 25 | # via docker-compose 26 | docker-compose==1.29.2 27 | # via -r requirements/base.in 28 | dockerpty==0.4.1 29 | # via docker-compose 30 | docopt==0.6.2 31 | # via docker-compose 32 | idna==3.3 33 | # via requests 34 | jsonschema==3.2.0 35 | # via docker-compose 36 | paramiko==2.11.0 37 | # via docker 38 | pycparser==2.21 39 | # via cffi 40 | pynacl==1.5.0 41 | # via paramiko 42 | pyrsistent==0.18.1 43 | # via jsonschema 44 | python-dotenv==0.20.0 45 | # via docker-compose 46 | pyyaml==5.4.1 47 | # via docker-compose 48 | requests==2.28.0 49 | # via 50 | # docker 51 | # docker-compose 52 | six==1.16.0 53 | # via 54 | # dockerpty 55 | # jsonschema 56 | # paramiko 57 | # websocket-client 58 | texttable==1.6.4 59 | # via docker-compose 60 | urllib3==1.26.9 61 | # via requests 62 | websocket-client==0.59.0 63 | # via 64 | # docker 65 | # docker-compose 66 | 67 | # The following packages are considered to be unsafe in a requirements file: 68 | # setuptools 69 | -------------------------------------------------------------------------------- /requirements/common_constraints.txt: -------------------------------------------------------------------------------- 1 | # A central location for most common version constraints 2 | # (across edx repos) for pip-installation. 3 | # 4 | # Similar to other constraint files this file doesn't install any packages. 5 | # It specifies version constraints that will be applied if a package is needed. 6 | # When pinning something here, please provide an explanation of why it is a good 7 | # idea to pin this package across all edx repos, Ideally, link to other information 8 | # that will help people in the future to remove the pin when possible. 9 | # Writing an issue against the offending project and linking to it here is good. 10 | # 11 | # Note: Changes to this file will automatically be used by other repos, referencing 12 | # this file from Github directly. It does not require packaging in edx-lint. 13 | 14 | 15 | # using LTS django version 16 | Django<4.0 17 | 18 | # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. 19 | # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html 20 | elasticsearch<7.14.0 21 | 22 | setuptools<60 23 | 24 | # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected 25 | django-simple-history==3.0.0 26 | -------------------------------------------------------------------------------- /requirements/constraints.txt: -------------------------------------------------------------------------------- 1 | # Version constraints for pip-installation. 2 | # 3 | # This file doesn't install any packages. It specifies version constraints 4 | # that will be applied if a package is needed. 5 | # 6 | # When pinning something here, please provide an explanation of why. Ideally, 7 | # link to other information that will help people in the future to remove the 8 | # pin when possible. Writing an issue against the offending project and 9 | # linking to it here is good. 10 | 11 | -c common_constraints.txt 12 | -------------------------------------------------------------------------------- /requirements/pip-tools.in: -------------------------------------------------------------------------------- 1 | -c constraints.txt 2 | 3 | pip-tools 4 | -------------------------------------------------------------------------------- /requirements/pip-tools.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with python 3.8 3 | # To update, run: 4 | # 5 | # make upgrade 6 | # 7 | click==8.1.3 8 | # via pip-tools 9 | pep517==0.12.0 10 | # via pip-tools 11 | pip-tools==6.6.2 12 | # via -r requirements/pip-tools.in 13 | tomli==2.0.1 14 | # via pep517 15 | wheel==0.37.1 16 | # via pip-tools 17 | 18 | # The following packages are considered to be unsafe in a requirements file: 19 | # pip 20 | # setuptools 21 | -------------------------------------------------------------------------------- /requirements/pip.in: -------------------------------------------------------------------------------- 1 | -c constraints.txt 2 | # Core dependencies for installing other packages 3 | 4 | pip 5 | setuptools 6 | wheel 7 | 8 | -------------------------------------------------------------------------------- /requirements/pip.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with python 3.8 3 | # To update, run: 4 | # 5 | # make upgrade 6 | # 7 | wheel==0.37.1 8 | # via -r requirements/pip.in 9 | 10 | # The following packages are considered to be unsafe in a requirements file: 11 | pip==22.1.2 12 | # via -r requirements/pip.in 13 | setuptools==59.8.0 14 | # via 15 | # -c requirements/common_constraints.txt 16 | # -r requirements/pip.in 17 | -------------------------------------------------------------------------------- /sample/jobs/sampleJob.groovy: -------------------------------------------------------------------------------- 1 | job ('SampleJenkinsJob') { 2 | 3 | logRotator { 4 | daysToKeep(10) 5 | numToKeep(-1) 6 | } 7 | steps { 8 | shell("echo 'hello world'") 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /sample/jobs/sampleProgrammaticBuild.groovy: -------------------------------------------------------------------------------- 1 | final String CONFIGURATION_REPO_URL = 'https://github.com/edx/configuration.git' 2 | final String CONFIGURATION_BRANCH = 'master' 3 | 4 | final String EDX_REPO_ROOT = 'https://github.com/edx/' 5 | final String EDX_REPO_BRANCH = 'release' 6 | 7 | def job = job('configuration-watcher') { 8 | // check out edx/configuration repository from GitHub 9 | scm { 10 | git { 11 | remote { 12 | url(CONFIGURATION_REPO_URL) 13 | relativeTargetDir('configuration') 14 | branch(CONFIGURATION_BRANCH) 15 | } 16 | } 17 | } 18 | // polls configuration repository for changes every 10 minutes 19 | triggers { 20 | scm('H/10 * * * *') 21 | } 22 | 23 | // run the trigger-builds shell script in a virtual environment called venv 24 | steps { 25 | virtualenv { 26 | pythonName('System-CPython-3.X') 27 | name('venv') 28 | nature('shell') 29 | command readFileFromWorkspace('sample/resources/trigger-builds.sh') 30 | } 31 | 32 | // inject environment variables defined in the temp_props file 33 | environmentVariables { 34 | propertiesFile('temp_props') 35 | } 36 | 37 | // trigger the jobs defined in the TO_BUILD environment variable; this is set via the trigger-builds script 38 | // and injected into the environment from the temp_props file 39 | downstreamParameterized { 40 | trigger('${TO_BUILD}') 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/groovy/org/edx/jenkins/dsl/JenkinsPublicConstants.groovy: -------------------------------------------------------------------------------- 1 | package org.edx.jenkins.dsl 2 | 3 | class JenkinsPublicConstants { 4 | 5 | public static final Closure GHPRB_CANCEL_BUILDS_ON_UPDATE(boolean override) { 6 | return { 7 | it / 'triggers' / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / 'extensions' / 'org.jenkinsci.plugins.ghprb.extensions.build.GhprbCancelBuildsOnUpdate' { 8 | overrideGlobal(override) 9 | } 10 | } 11 | } 12 | 13 | public static final Closure DEFAULT_VIEW = { 14 | return { 15 | status() 16 | weather() 17 | name() 18 | lastSuccess() 19 | lastFailure() 20 | lastDuration() 21 | buildButton() 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/groovy/devops/UserRetirementArchiverJobSpec.groovy: -------------------------------------------------------------------------------- 1 | package devops 2 | 3 | import groovy.util.slurpersupport.GPathResult 4 | import groovy.util.slurpersupport.Node 5 | import javaposse.jobdsl.plugin.JenkinsJobManagement 6 | import javaposse.jobdsl.dsl.GeneratedItems 7 | import javaposse.jobdsl.dsl.GeneratedJob 8 | import javaposse.jobdsl.dsl.DslScriptLoader 9 | import javaposse.jobdsl.dsl.JobManagement 10 | import org.junit.ClassRule 11 | import org.jvnet.hudson.test.JenkinsRule 12 | import spock.lang.Shared 13 | import spock.lang.Specification 14 | 15 | 16 | class UserRetirementArchiverJobSpec extends Specification { 17 | 18 | @Shared 19 | @ClassRule 20 | JenkinsRule jenkinsRule = new JenkinsRule() 21 | 22 | def @Shared DslScriptLoader loader 23 | def @Shared JobManagement jm 24 | 25 | /** 26 | * Seed a DSL script and verify that the job is created, without throwing 27 | * any exceptions 28 | **/ 29 | void 'test seeding dsl creates a job'() { 30 | 31 | setup: 32 | JenkinsJobManagement jm = new JenkinsJobManagement(System.out, [:], new File('.')) 33 | loader = new DslScriptLoader(jm) 34 | 35 | when: 36 | File dslTestSeederScriptPath = new File( 37 | "src/test/resources/devops/seeders/UserRetirementArchiverTestSeeder.groovy" 38 | ) 39 | GeneratedItems generatedItems = loader.runScript(dslTestSeederScriptPath.text) 40 | 41 | then: 42 | noExceptionThrown() 43 | generatedItems.jobs.size() == 1 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/resources/devops/seeders/UserRetirementArchiverTestSeeder.groovy: -------------------------------------------------------------------------------- 1 | import static devops.jobs.UserRetirementArchiver.job as UserRetirementArchiverJob 2 | 3 | Map jobVars = [ 4 | MAILING_LIST: 'email@example.com', 5 | SECURE_GIT_CREDENTIALS: 'fake-git-credential', 6 | ACCESS_CONTROL: ['test-group-1'], 7 | ADMIN_ACCESS_CONTROL: ['test-group-2'], 8 | DISABLED: true, 9 | ENVIRONMENT_DEPLOYMENT: 'env-deployment-1', 10 | CRON: '0 9 * * *' 11 | ] 12 | UserRetirementArchiverJob(this, jobVars) 13 | --------------------------------------------------------------------------------