├── playbooks ├── edx-east │ ├── roles │ ├── ec2.py │ ├── library │ ├── group_vars │ ├── inventory.ini │ ├── callback_plugins │ ├── mysql.yml │ ├── locust.yml │ ├── asqatasun.yml │ ├── sitespeedio.yml │ ├── tanaguru.yml │ ├── xsy.yml │ ├── aws.yml │ ├── edx_service_rds.yml │ ├── minos.yml │ ├── README.md │ ├── stop_all_edx_services.yml │ ├── flower.yml │ ├── notifier.yml │ ├── edx_ansible.yml │ ├── harstorage.yml │ ├── edx_vpc.yml │ ├── mongo.yml │ ├── antivirus.yml │ ├── edx_mirror.yml │ ├── restart_supervisor.yml │ ├── alton.yml │ ├── common.yml │ ├── demo.yml │ ├── aide.yml │ ├── certs.yml │ ├── snort.yml │ ├── testcourses.yml │ ├── create_cname.yml │ ├── oauth_client_setup.yml │ ├── splunk.yml │ ├── worker.yml │ ├── deploy_nginx_all_roles.yml │ ├── ecomworker.yml │ ├── xserver.yml │ ├── forum.yml │ ├── ansible.cfg │ ├── analytics_single.yml │ ├── notes.yml │ ├── insights.yml │ ├── analyticsapi.yml │ ├── discovery.yml │ ├── tools-gp.yml │ ├── xqwatcher.yml │ ├── vpc_admin.yml │ ├── ecommerce.yml │ ├── programs.yml │ ├── credentials.yml │ ├── jenkins_worker_sitespeedio.yml │ ├── set_hostname.yml │ ├── jenkins_admin.yml │ └── jenkins_tools.yml ├── continuous_delivery │ ├── roles │ └── templates │ │ └── local │ │ ├── key.pem.j2 │ │ ├── inventory.j2 │ │ ├── ami_template.yml.j2 │ │ └── launch_template.yml.j2 ├── inventory.ini ├── roles │ ├── shibboleth │ │ ├── templates │ │ │ ├── sp.key.j2 │ │ │ └── sp.pem.j2 │ │ ├── meta │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── common │ │ ├── templates │ │ │ ├── hostname.j2 │ │ │ ├── etc │ │ │ │ ├── cron.hourly │ │ │ │ │ └── logrotate.j2 │ │ │ │ └── logrotate.d │ │ │ │ │ └── hourly │ │ │ │ │ ├── edx_logrotate.j2 │ │ │ │ │ └── edx_logrotate_tracking_log.j2 │ │ │ └── hosts.j2 │ │ ├── files │ │ │ ├── ssh_key_forward │ │ │ └── bash_profile │ │ ├── meta │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── user │ │ ├── templates │ │ │ ├── restricted.bashrc.j2 │ │ │ ├── default.profile.j2 │ │ │ └── restricted.sudoers.conf.j2 │ │ └── meta │ │ │ └── main.yml │ ├── apache │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ ├── templates │ │ │ └── ports.conf.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── datadog │ │ ├── meta │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── demo │ │ └── meta │ │ │ └── main.yml │ ├── mongo │ │ ├── meta │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── templates │ │ │ ├── mongo-s3-backup-s3cfg.j2 │ │ │ └── create_root.js.j2 │ ├── munin_node │ │ └── files │ │ │ └── munin-edx │ ├── mysql │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ └── remove_mysql_experimental.yml │ ├── nginx │ │ ├── meta │ │ │ └── main.yml │ │ ├── README.md │ │ ├── templates │ │ │ ├── edx │ │ │ │ └── app │ │ │ │ │ └── nginx │ │ │ │ │ ├── robots.txt.j2 │ │ │ │ │ └── sites-available │ │ │ │ │ ├── robots.j2 │ │ │ │ │ ├── edx-release.j2 │ │ │ │ │ ├── certs.j2 │ │ │ │ │ ├── basic-auth.j2 │ │ │ │ │ └── s3_maintenance.j2 │ │ │ └── etc │ │ │ │ └── logrotate.d │ │ │ │ ├── edx_logrotate_nginx_error.j2 │ │ │ │ └── edx_logrotate_nginx_access.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── aide │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── certs │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── certs.auth.json.j2 │ │ │ ├── certs.env.json.j2 │ │ │ ├── git_ssh.sh.j2 │ │ │ └── certs.conf.j2 │ │ ├── files │ │ │ └── example-key-ownertrust.txt │ │ └── tasks │ │ │ └── tag_ec2.yml │ ├── edxapp │ │ ├── templates │ │ │ ├── .npmrc.j2 │ │ │ ├── cms.env.yaml.j2 │ │ │ ├── lms.env.yaml.j2 │ │ │ ├── cms.auth.yaml.j2 │ │ │ ├── lms.auth.yaml.j2 │ │ │ ├── edxapp.conf.j2 │ │ │ ├── course.xml.j2 │ │ │ ├── edx │ │ │ │ └── bin │ │ │ │ │ ├── edxapp-shell-cms.j2 │ │ │ │ │ ├── edxapp-shell-lms.j2 │ │ │ │ │ ├── edxapp-update-assets-cms.j2 │ │ │ │ │ ├── edxapp-update-assets-lms.j2 │ │ │ │ │ ├── edxapp-runserver-cms.j2 │ │ │ │ │ ├── edxapp-runserver-lms.j2 │ │ │ │ │ ├── edxapp_common.j2 │ │ │ │ │ ├── edxapp-migrate-cms.j2 │ │ │ │ │ └── edxapp-migrate-lms.j2 │ │ │ ├── edxapp_env.j2 │ │ │ ├── git_ssh.sh.j2 │ │ │ ├── cms.env.json.j2 │ │ │ ├── lms.env.json.j2 │ │ │ ├── lms.auth.json.j2 │ │ │ └── cms.auth.json.j2 │ │ └── meta │ │ │ └── main.yml │ ├── local_dev │ │ ├── files │ │ │ ├── x11_display │ │ │ ├── gitconfig │ │ │ └── ftplugin-python.vim │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ └── share_x11.j2 │ ├── newrelic │ │ └── meta │ │ │ └── main.yml │ ├── rabbitmq │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ ├── erlang.cookie.j2 │ │ │ ├── rabbitmq-env.conf.j2 │ │ │ └── etc │ │ │ ├── logrotate.d │ │ │ └── rabbitmq.j2 │ │ │ └── rabbitmq │ │ │ └── rabbitmq.config.j2 │ ├── supervisor │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ ├── edx │ │ │ └── bin │ │ │ │ └── supervisorctl.j2 │ │ │ └── etc │ │ │ └── init │ │ │ └── supervisor-upstart.conf.j2 │ ├── testcourses │ │ ├── meta │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── elasticsearch │ │ └── meta │ │ │ └── main.yml │ ├── jenkins_analytics │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ └── credentials_file.json.j2 │ ├── splunkforwarder │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ └── opt │ │ │ │ └── splunkforwarder │ │ │ │ └── etc │ │ │ │ └── system │ │ │ │ └── local │ │ │ │ └── outputs.conf.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── cassandra │ │ └── meta │ │ │ └── main.yml │ ├── edx_ansible │ │ ├── templates │ │ │ ├── dumpall.yml.j2 │ │ │ └── show-repo-heads.j2 │ │ └── meta │ │ │ └── main.yml │ ├── edxlocal │ │ └── meta │ │ │ └── main.yml │ ├── jenkins_worker │ │ ├── templates │ │ │ ├── .npmrc.j2 │ │ │ ├── jenkins_env.j2 │ │ │ └── pip-accel.conf.j2 │ │ ├── tasks │ │ │ ├── python.yml │ │ │ ├── test.yml │ │ │ ├── packer.yml │ │ │ ├── test_sitespeed_worker.yml │ │ │ └── main.yml │ │ └── meta │ │ │ └── main.yml │ ├── notifier │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── git_ssh_noauth.sh.j2 │ │ │ ├── git_ssh_auth.sh.j2 │ │ │ ├── settings_local.py.j2 │ │ │ ├── notifier-scheduler-supervisor.sh.j2 │ │ │ ├── notifier-celery-workers-supervisor.sh.j2 │ │ │ └── notifier_env.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── flower │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ └── edx │ │ │ │ └── app │ │ │ │ ├── flower │ │ │ │ ├── flower_env.j2 │ │ │ │ └── flowerconfig.py.j2 │ │ │ │ └── supervisor │ │ │ │ └── conf.d.available │ │ │ │ └── flower.conf.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── kibana │ │ ├── meta │ │ │ └── default.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── xqueue │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ ├── xqueue.auth.json.j2 │ │ │ ├── xqueue.env.json.j2 │ │ │ ├── xqueue_consumer.conf.j2 │ │ │ └── xqueue_gunicorn.py.j2 │ ├── xserver │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── xserver.env.json.j2 │ │ │ ├── 99-sandbox.j2 │ │ │ ├── git_ssh.sh.j2 │ │ │ ├── usr.bin.python-sandbox.j2 │ │ │ └── xserver_gunicorn.py.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── gh_mirror │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ └── orgs.yml.j2 │ ├── swapfile │ │ ├── handlers │ │ │ └── main.yml │ │ └── meta │ │ │ └── main.yml │ ├── ansible-role-django-ida │ │ └── templates │ │ │ ├── docker │ │ │ ├── build │ │ │ │ └── ROLE_NAME │ │ │ │ │ ├── inventory │ │ │ │ │ └── ansible_overrides.yml.j2 │ │ │ └── plays │ │ │ │ └── ROLE_NAME.yml.j2 │ │ │ ├── templates │ │ │ └── edx │ │ │ │ └── app │ │ │ │ ├── ROLE_NAME │ │ │ │ ├── ROLE_NAME_env.j2 │ │ │ │ └── ROLE_NAME_gunicorn.py.j2 │ │ │ │ └── supervisor │ │ │ │ └── conf.d.available │ │ │ │ └── ROLE_NAME.conf.j2 │ │ │ └── header.j2 │ ├── gitreload │ │ ├── templates │ │ │ └── edx │ │ │ │ └── app │ │ │ │ ├── gitreload │ │ │ │ ├── gr.env.json.j2 │ │ │ │ └── gitreload_gunicorn.py.j2 │ │ │ │ └── supervisor │ │ │ │ └── conf.available.d │ │ │ │ └── gitreload.conf.j2 │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── logstash │ │ ├── meta │ │ │ └── default.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── minos │ │ └── templates │ │ │ ├── edx │ │ │ └── etc │ │ │ │ └── minos │ │ │ │ └── conf.d │ │ │ │ ├── BellwetherVoter.yml.j2 │ │ │ │ ├── RolledTrackingLogVoter.yml.j2 │ │ │ │ ├── ZippedTrackingLogVoter.yml.j2 │ │ │ │ ├── ProccessQuiescenceVoterPython.yml.j2 │ │ │ │ └── TrackingLogVoter.yml.j2 │ │ │ └── tmp │ │ │ └── git-identity.sh.j2 │ ├── xqwatcher │ │ ├── templates │ │ │ ├── edx │ │ │ │ └── app │ │ │ │ │ └── xqwatcher │ │ │ │ │ ├── xqwatcher.json.j2 │ │ │ │ │ ├── conf.d │ │ │ │ │ └── course.json.j2 │ │ │ │ │ └── data │ │ │ │ │ └── requirements.txt.j2 │ │ │ └── etc │ │ │ │ └── sudoers.d │ │ │ │ ├── 95-course-sandbox.j2 │ │ │ │ └── 95-xqwatcher.j2 │ │ └── tasks │ │ │ └── deploy_courses.yml │ ├── browsermob-proxy │ │ ├── files │ │ │ └── browsermob-proxy │ │ └── defaults │ │ │ └── main.yml │ ├── harprofiler │ │ ├── templates │ │ │ ├── bashrc.j2 │ │ │ └── validate_harprofiler_install.sh.j2 │ │ ├── meta │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── mongo_3_0 │ │ ├── handlers │ │ │ └── main.yml │ │ ├── meta │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── create_root.js.j2 │ │ │ ├── log-mongo-serverStatus.sh.j2 │ │ │ └── mongo_logrotate.j2 │ │ └── files │ │ │ └── disable-transparent-hugepages.conf │ ├── oraclejdk │ │ ├── templates │ │ │ └── java.sh.j2 │ │ └── defaults │ │ │ └── main.yml │ ├── enhanced_networking │ │ └── defaults │ │ │ └── main.yml │ ├── edx_service │ │ ├── templates │ │ │ └── config.yml.j2 │ │ └── defaults │ │ │ └── main.yml │ ├── hive │ │ ├── templates │ │ │ └── hive-env.sh.j2 │ │ └── meta │ │ │ └── main.yml │ ├── mongo_mms │ │ ├── handlers │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── tools_jenkins │ │ ├── handlers │ │ │ └── main.yml │ │ └── meta │ │ │ └── main.yml │ ├── forum │ │ ├── templates │ │ │ ├── java.sh.j2 │ │ │ ├── forum_env.j2 │ │ │ ├── forum.conf.j2 │ │ │ ├── forum-supervisor.sh.j2 │ │ │ └── cs_comments_service.conf.j2 │ │ ├── meta │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ ├── tag_ec2.yml │ │ │ └── test.yml │ ├── hadoop_common │ │ ├── templates │ │ │ ├── hadoop_user_ssh_config.j2 │ │ │ ├── core-site.xml.j2 │ │ │ ├── hdfs.conf.j2 │ │ │ ├── yarn.conf.j2 │ │ │ ├── mapred-site.xml.j2 │ │ │ ├── hdfs-site.xml.j2 │ │ │ └── yarn-site.xml.j2 │ │ └── meta │ │ │ └── main.yml │ ├── security │ │ ├── templates │ │ │ └── etc │ │ │ │ └── apt │ │ │ │ └── apt.conf.d │ │ │ │ ├── 20unattended-upgrade │ │ │ │ └── 10periodic │ │ └── tasks │ │ │ └── main.yml │ ├── snort │ │ ├── templates │ │ │ └── etc │ │ │ │ └── cron.daily │ │ │ │ └── oinkmaster.j2 │ │ └── defaults │ │ │ └── main.yml │ ├── sqoop │ │ ├── templates │ │ │ └── sqoop-env.sh.j2 │ │ └── meta │ │ │ └── main.yml │ ├── xsy │ │ ├── templates │ │ │ ├── xsy-supervisor.sh.j2 │ │ │ ├── xsy_env.j2 │ │ │ └── xsy.conf.j2 │ │ ├── tasks │ │ │ └── tag_ec2.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── alton │ │ ├── templates │ │ │ ├── alton-supervisor.sh.j2 │ │ │ ├── boto.j2 │ │ │ ├── alton_env.j2 │ │ │ └── alton.conf.j2 │ │ ├── tasks │ │ │ └── tag_ec2.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── go-server │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ └── go-server │ │ │ └── password.txt.j2 │ ├── common_vars │ │ └── tasks │ │ │ └── main.yml │ ├── elasticsearch-monitor │ │ ├── templates │ │ │ └── plugins │ │ │ │ └── me.snov.newrelic-elasticsearch │ │ │ │ └── newrelic-elasticsearch-plugin-1.4.1 │ │ │ │ └── config │ │ │ │ └── plugin.json.j2 │ │ └── meta │ │ │ └── main.yml │ ├── ad_hoc_reporting │ │ ├── templates │ │ │ ├── edx │ │ │ │ └── bin │ │ │ │ │ └── mysql.sh.j2 │ │ │ └── etc │ │ │ │ └── mongorc.js.j2 │ │ └── meta │ │ │ └── main.yml │ ├── harstorage │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── harstorage │ │ │ └── harstorage.sh.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── harstorage.conf.j2 │ ├── discovery │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── discovery │ │ │ ├── discovery_env.j2 │ │ │ └── discovery_gunicorn.py.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── discovery.conf.j2 │ ├── ecommerce │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── ecommerce │ │ │ ├── ecommerce_env.j2 │ │ │ └── ecommerce_gunicorn.py.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── ecommerce.conf.j2 │ ├── insights │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── insights │ │ │ └── insights_env.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── insights.conf.j2 │ ├── memcache │ │ └── tasks │ │ │ └── main.yml │ ├── programs │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── programs │ │ │ ├── programs_env.j2 │ │ │ └── programs_gunicorn.py.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── programs.conf.j2 │ ├── analytics_pipeline │ │ ├── templates │ │ │ └── client.cfg.j2 │ │ └── meta │ │ │ └── main.yml │ ├── credentials │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── credentials │ │ │ ├── credentials_env.j2 │ │ │ └── credentials_gunicorn.py.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── credentials.conf.j2 │ ├── ecomworker │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── ecomworker │ │ │ └── ecomworker_env.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── ecomworker.conf.j2 │ ├── rbenv │ │ └── templates │ │ │ └── ruby_env.j2 │ ├── analytics_api │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── analytics_api │ │ │ ├── analytics_api_env.j2 │ │ │ └── analytics_api_gunicorn.py.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── analytics_api.conf.j2 │ ├── edx_notes_api │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ ├── edx_notes_api │ │ │ └── edx_notes_api_env.j2 │ │ │ └── supervisor │ │ │ └── conf.d.available │ │ │ └── edx_notes_api.conf.j2 │ ├── gluster │ │ └── defaults │ │ │ └── main.yml │ ├── go-agent-docker-server │ │ ├── README.md │ │ ├── templates │ │ │ └── edx │ │ │ │ └── app │ │ │ │ └── go-agent-docker-server │ │ │ │ └── autoregister.properties.j2 │ │ └── defaults │ │ │ └── main.yml │ ├── jenkins_master │ │ ├── tasks │ │ │ └── datadog.yml │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── browsers │ │ └── templates │ │ │ └── xvfb.conf.j2 │ ├── edxapp_common │ │ └── tasks │ │ │ └── main.yml │ ├── jenkins_admin │ │ ├── templates │ │ │ └── edx │ │ │ │ └── var │ │ │ │ └── jenkins │ │ │ │ ├── boto.j2 │ │ │ │ ├── aws_config.j2 │ │ │ │ └── hudson.plugins.s3.S3BucketPublisher.xml.j2 │ │ └── handlers │ │ │ └── main.yml │ ├── haproxy │ │ ├── templates │ │ │ ├── haproxy.rsyslog.j2 │ │ │ ├── haproxy.logrotate.j2 │ │ │ └── haproxy.cfg.j2 │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── vhost │ │ ├── templates │ │ │ └── etc │ │ │ │ └── init │ │ │ │ └── sync-on-stop.conf.j2 │ │ └── meta │ │ │ └── main.yml │ ├── ansible-role │ │ └── templates │ │ │ ├── meta │ │ │ └── main.yml.j2 │ │ │ ├── header.j2 │ │ │ ├── tasks │ │ │ └── main.yml.j2 │ │ │ └── defaults │ │ │ └── main.yml.j2 │ ├── go-agent │ │ └── templates │ │ │ └── edx │ │ │ └── app │ │ │ └── go-agent │ │ │ └── go-agent.j2 │ ├── jscover │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── openstack │ │ ├── templates │ │ │ └── log-sync-env.sh.j2 │ │ └── meta │ │ │ └── main.yml │ ├── splunk-server │ │ ├── templates │ │ │ └── opt │ │ │ │ └── splunk │ │ │ │ └── etc │ │ │ │ ├── apps │ │ │ │ └── search │ │ │ │ │ └── local │ │ │ │ │ ├── indexes.conf.j2 │ │ │ │ │ └── props.conf.j2 │ │ │ │ └── system │ │ │ │ └── local │ │ │ │ └── alert_actions.conf.j2 │ │ └── meta │ │ │ └── main.yml │ ├── postfix_queue │ │ └── handlers │ │ │ └── main.yml │ ├── asqatasun │ │ └── templates │ │ │ └── etc │ │ │ └── mysql │ │ │ └── conf.d │ │ │ └── asqatasun.cnf.j2 │ ├── server_utils │ │ └── meta │ │ │ └── main.yml │ ├── antivirus │ │ ├── meta │ │ │ └── main.yml │ │ └── handlers │ │ │ └── main.yml │ ├── hadoop_master │ │ ├── meta │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── aws │ │ └── meta │ │ │ └── main.yml │ ├── newrelic-npi │ │ └── meta │ │ │ └── main.yml │ ├── add_user │ │ └── meta │ │ │ └── main.yml │ ├── git_clone │ │ └── meta │ │ │ └── main.yml │ ├── redis │ │ ├── handlers │ │ │ └── main.yml │ │ └── meta │ │ │ └── main.yml │ ├── container │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── mariadb │ │ ├── meta │ │ │ └── main.yml │ │ └── templates │ │ │ └── etc │ │ │ └── mysql │ │ │ └── conf.d │ │ │ └── galera.cnf.j2 │ ├── docker │ │ ├── meta │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── edx_vpc │ │ └── defaults │ │ │ └── main.yml │ ├── test_build_server │ │ └── defaults │ │ │ └── main.yml │ ├── sitespeedio │ │ └── meta │ │ │ └── main.yml │ ├── nltk │ │ └── defaults │ │ │ └── main.yml │ └── mount_ebs │ │ └── defaults │ │ └── main.yml ├── vagrant │ ├── inventory.ini │ ├── ansible.cfg │ ├── vagrant-devstack-delta.yml │ └── vagrant-fullstack-delta.yml ├── security.yml ├── group_vars │ ├── README.md │ └── tag_environment_prod ├── run_role.yml ├── create_django_ida.yml ├── create_role.yml ├── analytics-jenkins.yml ├── go-agent.yml ├── go-agent-docker.yml ├── log_server.yml ├── go-server.yml └── ansible.cfg ├── pre-requirements.txt ├── docker ├── plays │ ├── roles │ ├── library │ ├── ansible.cfg │ ├── xqwatcher.yml │ ├── xqueue.yml │ ├── harstorage.yml │ ├── insights.yml │ ├── forum.yml │ ├── edxapp.yml │ ├── rabbitmq.yml │ ├── ecommerce.yml │ ├── analytics_api.yml │ ├── jenkins_analytics.yml │ ├── discovery.yml │ ├── credentials.yml │ └── nginx.yml └── build │ ├── discovery │ ├── inventory │ └── ansible_overrides.yml │ ├── analytics_api │ ├── analytics_api_tmp.yml │ └── ansible_overrides.yml │ ├── nginx │ ├── ansible_overrides.yml │ └── Dockerfile │ ├── harstorage │ └── ansible_overrides.yml │ ├── ecommerce │ ├── docker-run.sh │ ├── ansible_overrides.yml │ └── Dockerfile │ ├── xqueue │ ├── docker-run.sh │ ├── ansible_overrides.yml │ └── Dockerfile │ ├── edxapp │ └── ansible_overrides.yml │ ├── insights │ ├── insights-temp.yml │ ├── Dockerfile │ └── ansible_overrides.yml │ ├── forum │ ├── ansible_overrides.yml │ └── Dockerfile │ ├── go-agent │ └── files │ │ └── go-agent-env-vars │ ├── trusty-common │ └── Dockerfile │ ├── precise-common │ └── Dockerfile │ ├── rabbitmq │ └── Dockerfile │ ├── credentials │ └── ansible_overrides.yml │ ├── jenkins_analytics │ └── Dockerfile.noci │ ├── elasticsearch │ └── xDockerfile │ └── xqwatcher │ └── Dockerfile ├── vagrant └── base │ ├── devstack │ └── ansible.cfg │ ├── analyticstack │ └── ansible.cfg │ ├── fullstack │ └── ansible.cfg │ ├── test_role │ └── ansible.cfg │ ├── cluster │ ├── inventory.ini │ └── ansible.cfg │ └── test_playbook │ └── ansible.cfg ├── util ├── vpc-tools │ ├── requirements.txt │ └── sanitize-db-prod_grader.sql ├── jenkins │ ├── restart-xqueue.sh │ ├── restart-xqueue_consumer.sh │ ├── restart-xqwatcher.sh │ ├── ascii-convert.sh │ ├── change-enrollment-course.sh │ ├── change-enrollment.sh │ └── django-admin.sh ├── csmh-extended │ ├── slow-delete.sh │ ├── migrate-same-database-instance.sh │ └── migrate-separate-database-instances.sh └── parsefiles_config.yml ├── .dockerignore ├── terraform ├── .gitignore ├── README └── sitespeed │ └── terraform.tfvars.example ├── .gitignore ├── requirements.txt └── .github └── ISSUE_TEMPLATE.md /playbooks/edx-east/roles: -------------------------------------------------------------------------------- 1 | ../roles -------------------------------------------------------------------------------- /playbooks/edx-east/ec2.py: -------------------------------------------------------------------------------- 1 | ../ec2.py -------------------------------------------------------------------------------- /playbooks/edx-east/library: -------------------------------------------------------------------------------- 1 | ../library -------------------------------------------------------------------------------- /pre-requirements.txt: -------------------------------------------------------------------------------- 1 | pip==8.1.2 2 | -------------------------------------------------------------------------------- /docker/plays/roles: -------------------------------------------------------------------------------- 1 | ../../playbooks/roles/ -------------------------------------------------------------------------------- /playbooks/continuous_delivery/roles: -------------------------------------------------------------------------------- 1 | ../roles -------------------------------------------------------------------------------- /playbooks/edx-east/group_vars: -------------------------------------------------------------------------------- 1 | ../group_vars -------------------------------------------------------------------------------- /docker/plays/library: -------------------------------------------------------------------------------- 1 | ../../playbooks/library/ -------------------------------------------------------------------------------- /playbooks/edx-east/inventory.ini: -------------------------------------------------------------------------------- 1 | ../inventory.ini -------------------------------------------------------------------------------- /playbooks/inventory.ini: -------------------------------------------------------------------------------- 1 | [localhost] 2 | 127.0.0.1 3 | -------------------------------------------------------------------------------- /docker/build/discovery/inventory: -------------------------------------------------------------------------------- 1 | [local] 2 | localhost 3 | -------------------------------------------------------------------------------- /playbooks/edx-east/callback_plugins: -------------------------------------------------------------------------------- 1 | ../callback_plugins -------------------------------------------------------------------------------- /playbooks/roles/shibboleth/templates/sp.key.j2: -------------------------------------------------------------------------------- 1 | {{ shib.sp_key }} -------------------------------------------------------------------------------- /playbooks/roles/shibboleth/templates/sp.pem.j2: -------------------------------------------------------------------------------- 1 | {{ shib.sp_pem }} -------------------------------------------------------------------------------- /playbooks/vagrant/inventory.ini: -------------------------------------------------------------------------------- 1 | [vagrant] 2 | 192.168.33.10 3 | -------------------------------------------------------------------------------- /vagrant/base/devstack/ansible.cfg: -------------------------------------------------------------------------------- 1 | ../../../playbooks/ansible.cfg -------------------------------------------------------------------------------- /vagrant/base/analyticstack/ansible.cfg: -------------------------------------------------------------------------------- 1 | ../../../playbooks/ansible.cfg -------------------------------------------------------------------------------- /vagrant/base/fullstack/ansible.cfg: -------------------------------------------------------------------------------- 1 | ../../../playbooks/ansible.cfg -------------------------------------------------------------------------------- /vagrant/base/test_role/ansible.cfg: -------------------------------------------------------------------------------- 1 | ../../../playbooks/ansible.cfg -------------------------------------------------------------------------------- /playbooks/roles/common/templates/hostname.j2: -------------------------------------------------------------------------------- 1 | {{ COMMON_HOSTNAME }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/user/templates/restricted.bashrc.j2: -------------------------------------------------------------------------------- 1 | PATH=${HOME}/bin 2 | -------------------------------------------------------------------------------- /docker/build/analytics_api/analytics_api_tmp.yml: -------------------------------------------------------------------------------- 1 | ../../plays/analytics_api.yml -------------------------------------------------------------------------------- /docker/build/nginx/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | FLOCK_TLD: "edx" 4 | -------------------------------------------------------------------------------- /playbooks/roles/apache/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/datadog/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/demo/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/mongo/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/munin_node/files/munin-edx: -------------------------------------------------------------------------------- 1 | [iostat] 2 | env.SHOW_NUMBERED 1 -------------------------------------------------------------------------------- /playbooks/roles/mysql/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/user/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common_vars -------------------------------------------------------------------------------- /docker/build/harstorage/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | FLOCK_TLD: "edx" 4 | -------------------------------------------------------------------------------- /playbooks/roles/aide/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | AIDE_REPORT_EMAIL: 'root' 4 | -------------------------------------------------------------------------------- /playbooks/roles/certs/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - common 3 | - supervisor -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/.npmrc.j2: -------------------------------------------------------------------------------- 1 | registry={{ COMMON_NPM_MIRROR_URL }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/local_dev/files/x11_display: -------------------------------------------------------------------------------- 1 | Defaults env_keep+=DISPLAY 2 | -------------------------------------------------------------------------------- /playbooks/roles/newrelic/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/rabbitmq/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/shibboleth/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - apache 4 | -------------------------------------------------------------------------------- /playbooks/roles/supervisor/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/testcourses/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - demo 4 | -------------------------------------------------------------------------------- /playbooks/roles/common/files/ssh_key_forward: -------------------------------------------------------------------------------- 1 | Defaults env_keep+=SSH_AUTH_SOCK 2 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/cms.env.yaml.j2: -------------------------------------------------------------------------------- 1 | {{ cms_env_config | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/lms.env.yaml.j2: -------------------------------------------------------------------------------- 1 | {{ lms_env_config | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/elasticsearch/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_analytics/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - jenkins_master 3 | -------------------------------------------------------------------------------- /playbooks/roles/splunkforwarder/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | -------------------------------------------------------------------------------- /playbooks/roles/cassandra/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - oraclejdk -------------------------------------------------------------------------------- /playbooks/roles/certs/templates/certs.auth.json.j2: -------------------------------------------------------------------------------- 1 | {{ certs_auth_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/certs/templates/certs.env.json.j2: -------------------------------------------------------------------------------- 1 | {{ certs_env_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/edx_ansible/templates/dumpall.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {{ vars | to_nice_yaml }} 3 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/cms.auth.yaml.j2: -------------------------------------------------------------------------------- 1 | {{ cms_auth_config | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/lms.auth.yaml.j2: -------------------------------------------------------------------------------- 1 | {{ lms_auth_config | to_nice_yaml }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/edxlocal/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - mysql 5 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/templates/.npmrc.j2: -------------------------------------------------------------------------------- 1 | registry={{ COMMON_NPM_MIRROR_URL }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor -------------------------------------------------------------------------------- /playbooks/continuous_delivery/templates/local/key.pem.j2: -------------------------------------------------------------------------------- 1 | {{ ssh_key_register.key.private_key }} -------------------------------------------------------------------------------- /playbooks/roles/flower/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | -------------------------------------------------------------------------------- /playbooks/roles/kibana/meta/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - nginx 5 | -------------------------------------------------------------------------------- /playbooks/roles/rabbitmq/templates/erlang.cookie.j2: -------------------------------------------------------------------------------- 1 | {{rabbitmq_auth_config.erlang_cookie}} 2 | -------------------------------------------------------------------------------- /playbooks/roles/xqueue/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | -------------------------------------------------------------------------------- /playbooks/roles/xqueue/templates/xqueue.auth.json.j2: -------------------------------------------------------------------------------- 1 | {{ xqueue_auth_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/xqueue/templates/xqueue.env.json.j2: -------------------------------------------------------------------------------- 1 | {{ xqueue_env_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/templates/xserver.env.json.j2: -------------------------------------------------------------------------------- 1 | {{ xserver_env_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /util/vpc-tools/requirements.txt: -------------------------------------------------------------------------------- 1 | boto 2 | docopt 3 | requests 4 | python-simple-hipchat==0.2 5 | -------------------------------------------------------------------------------- /playbooks/roles/certs/files/example-key-ownertrust.txt: -------------------------------------------------------------------------------- 1 | A9F9EAD11A0A6E7E5A037BDC044089B6FEF8D954:6: 2 | -------------------------------------------------------------------------------- /playbooks/roles/gh_mirror/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | -------------------------------------------------------------------------------- /playbooks/roles/swapfile/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: reload sysctl 3 | command: sysctl -p 4 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/inventory: -------------------------------------------------------------------------------- 1 | [local] 2 | localhost 3 | -------------------------------------------------------------------------------- /playbooks/roles/gitreload/templates/edx/app/gitreload/gr.env.json.j2: -------------------------------------------------------------------------------- 1 | {{ gitreload_env | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/logstash/meta/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - elasticsearch 5 | -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/edx/etc/minos/conf.d/BellwetherVoter.yml.j2: -------------------------------------------------------------------------------- 1 | BellwetherVoter: 2 | config: -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/templates/edx/app/xqwatcher/xqwatcher.json.j2: -------------------------------------------------------------------------------- 1 | {{ XQWATCHER_CONFIG | to_nice_json }} -------------------------------------------------------------------------------- /playbooks/roles/jenkins_analytics/templates/credentials_file.json.j2: -------------------------------------------------------------------------------- 1 | {{ JENKINS_ANALYTICS_CREDENTIALS|to_json }} -------------------------------------------------------------------------------- /playbooks/roles/gh_mirror/templates/orgs.yml.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | {{ gh_mirror_orgs | to_nice_yaml }} 3 | -------------------------------------------------------------------------------- /playbooks/roles/browsermob-proxy/files/browsermob-proxy: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /etc/browsermob-proxy/bin/browsermob-proxy $* 3 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edxapp.conf.j2: -------------------------------------------------------------------------------- 1 | [group:edxapp] 2 | programs={{ ",".join(service_variants_enabled) }} 3 | -------------------------------------------------------------------------------- /playbooks/roles/shibboleth/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart shibd 3 | service: name=shibd state=restarted 4 | -------------------------------------------------------------------------------- /playbooks/security.yml: -------------------------------------------------------------------------------- 1 | - name: Apply security role 2 | hosts: all 3 | become: True 4 | roles: 5 | - security 6 | -------------------------------------------------------------------------------- /playbooks/roles/harprofiler/templates/bashrc.j2: -------------------------------------------------------------------------------- 1 | export DISPLAY=:1 2 | 3 | source {{ harprofiler_venv_dir }}/bin/activate 4 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart mongo 3 | service: name=mongod state=restarted 4 | 5 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/git_ssh_noauth.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec /usr/bin/ssh -o StrictHostKeyChecking=no "$@" 3 | -------------------------------------------------------------------------------- /playbooks/roles/oraclejdk/templates/java.sh.j2: -------------------------------------------------------------------------------- 1 | export JAVA_HOME="{{ oraclejdk_link }}" 2 | export PATH=$JAVA_HOME/bin:$PATH 3 | -------------------------------------------------------------------------------- /playbooks/roles/enhanced_networking/defaults/main.yml: -------------------------------------------------------------------------------- 1 | profile: edx 2 | compatible_instance_types: ['c3', 'c4', 'd2', 'i2', 'm4', 'r3'] -------------------------------------------------------------------------------- /playbooks/roles/logstash/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart logstash 3 | service: name=logstash state=restarted 4 | 5 | -------------------------------------------------------------------------------- /playbooks/roles/apache/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apache_ports: 3 | - 80 4 | apache_sites: 5 | - lms 6 | apache_template_dir: '.' 7 | -------------------------------------------------------------------------------- /playbooks/roles/edx_service/templates/config.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | # {{ ansible_managed }} 3 | 4 | {{ edx_service_config | to_nice_yaml }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/course.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /playbooks/roles/harprofiler/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - browsers 5 | - oraclejdk 6 | - browsermob-proxy 7 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/README.md: -------------------------------------------------------------------------------- 1 | * main.yml: installs nginx and will enable the basic nginx configuration for version introspection 2 | -------------------------------------------------------------------------------- /playbooks/edx-east/mysql.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy MySQL 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - mysql 7 | -------------------------------------------------------------------------------- /playbooks/roles/certs/templates/git_ssh.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ certs_git_identity }} "$@" 3 | -------------------------------------------------------------------------------- /playbooks/roles/hive/templates/hive-env.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export HIVE_HOME={{ HIVE_HOME }} 4 | export PATH=$PATH:$HIVE_HOME/bin 5 | -------------------------------------------------------------------------------- /playbooks/roles/mongo/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart mongo 3 | service: 4 | name: mongod 5 | state: restarted 6 | 7 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - role: mount_ebs 5 | volumes: "{{ MONGO_VOLUMES }}" 6 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_mms/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart mms 3 | service: name=mongodb-mms-monitoring-agent state=restarted 4 | -------------------------------------------------------------------------------- /playbooks/roles/tools_jenkins/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart Jenkins 3 | service: 4 | name: jenkins 5 | state: restarted -------------------------------------------------------------------------------- /playbooks/roles/forum/templates/java.sh.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | export JAVA_HOME="{{ java_home }}" 4 | export PATH=$JAVA_HOME/bin:$PATH -------------------------------------------------------------------------------- /playbooks/roles/local_dev/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: jscover 4 | jscover_version: "{{ localdev_jscover_version }}" 5 | -------------------------------------------------------------------------------- /playbooks/roles/rabbitmq/templates/rabbitmq-env.conf.j2: -------------------------------------------------------------------------------- 1 | RABBITMQ_NODE_PORT={{ rabbitmq_port }} 2 | RABBITMQ_NODE_IP_ADDRESS={{ rabbitmq_ip }} 3 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/templates/99-sandbox.j2: -------------------------------------------------------------------------------- 1 | www-data ALL=({{ xserver_sandbox_user }}) NOPASSWD:{{ xserver_venv_sandbox_dir }}/bin/python 2 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/git_ssh_auth.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ notifier_git_identity }} "$@" 3 | -------------------------------------------------------------------------------- /playbooks/continuous_delivery/templates/local/inventory.j2: -------------------------------------------------------------------------------- 1 | {% for instance in ec2_instance_register.instances %} 2 | {{ instance.private_ip }} 3 | {% endfor %} -------------------------------------------------------------------------------- /playbooks/roles/apache/templates/ports.conf.j2: -------------------------------------------------------------------------------- 1 | {%- for port in apache_ports -%} 2 | NameVirtualHost *:{{ port }} 3 | Listen {{ port }} 4 | {% endfor %} 5 | -------------------------------------------------------------------------------- /playbooks/roles/common/files/bash_profile: -------------------------------------------------------------------------------- 1 | # .bash_profile 2 | 3 | # Get the aliases and functions 4 | if [ -f ~/.bashrc ]; then 5 | . ~/.bashrc 6 | fi -------------------------------------------------------------------------------- /playbooks/roles/datadog/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart the datadog service 3 | service: 4 | name: datadog-agent 5 | state: restarted 6 | -------------------------------------------------------------------------------- /playbooks/roles/local_dev/files/gitconfig: -------------------------------------------------------------------------------- 1 | [branch] 2 | autosetupmerge = true 3 | autosetuprebase = always 4 | [push] 5 | default = current 6 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/hadoop_user_ssh_config.j2: -------------------------------------------------------------------------------- 1 | Host localhost 2 | StrictHostKeyChecking no 3 | 4 | Host 0.0.0.0 5 | StrictHostKeyChecking no -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/templates/edx/app/xqwatcher/conf.d/course.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "{{ item.QUEUE_NAME }}": 3 | {{ item.QUEUE_CONFIG | to_nice_json }} 4 | } -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !playbooks/ 3 | !docker/build/*/ansible_overrides.yml 4 | !docker/build/* 5 | docker/build/*/Dockerfile 6 | !docker/plays/ 7 | !util/install/ 8 | -------------------------------------------------------------------------------- /docker/plays/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | 3 | jinja2_extensions=jinja2.ext.do 4 | roles_path=../plays:../../playbooks/roles 5 | library=../../playbooks/library 6 | 7 | -------------------------------------------------------------------------------- /docker/plays/xqwatcher.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy xqwatcher 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | roles: 6 | - docker 7 | - xqwatcher 8 | -------------------------------------------------------------------------------- /playbooks/edx-east/locust.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Deploy Locust 3 | hosts: all 4 | become: True 5 | gather_facts: True 6 | roles: 7 | - aws 8 | - locust 9 | -------------------------------------------------------------------------------- /playbooks/roles/apache/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart apache 3 | service: 4 | name: apache2 5 | state: restarted 6 | tags: 7 | - deploy 8 | -------------------------------------------------------------------------------- /playbooks/roles/supervisor/templates/edx/bin/supervisorctl.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo -u {{ supervisor_service_user }} {{ supervisor_ctl }} -c {{ supervisor_cfg }} $* 3 | -------------------------------------------------------------------------------- /playbooks/edx-east/asqatasun.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Asqatasun 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - mysql 7 | - asqatasun 8 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/etc/cron.hourly/logrotate.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | test -x /usr/sbin/logrotate || exit 0 4 | /usr/sbin/logrotate /etc/logrotate.d/hourly 5 | -------------------------------------------------------------------------------- /playbooks/roles/mysql/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | remove_experimental_mysql: false 3 | 4 | mysql_debian_pkgs: 5 | - "mysql-server-5.6" 6 | - python-mysqldb 7 | 8 | -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- 1 | # Do not accidentally check in a file with secret info e.g. AWS credentials 2 | terraform.tfvars 3 | terraform.tfstate 4 | terraform.tfstate.backup 5 | -------------------------------------------------------------------------------- /docker/plays/xqueue.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy xqueue 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | roles: 6 | - common_vars 7 | - docker 8 | - xqueue 9 | -------------------------------------------------------------------------------- /playbooks/edx-east/sitespeedio.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Sitespeed.io 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - aws 7 | - sitespeedio 8 | -------------------------------------------------------------------------------- /playbooks/roles/security/templates/etc/apt/apt.conf.d/20unattended-upgrade: -------------------------------------------------------------------------------- 1 | 2 | Unattended-Upgrade::Allowed-Origins { 3 | "${distro_id} ${distro_codename}-security"; 4 | }; 5 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/templates/git_ssh.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec /usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ xserver_git_identity }} "$@" 3 | -------------------------------------------------------------------------------- /docker/plays/harstorage.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Harstorage 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | roles: 6 | - docker 7 | - mongo 8 | - harstorage 9 | -------------------------------------------------------------------------------- /playbooks/continuous_delivery/templates/local/ami_template.yml.j2: -------------------------------------------------------------------------------- 1 | ami_id: {{ ami_register.image_id }} 2 | ami_message: {{ ami_register.msg }} 3 | ami_state: {{ ami_register.state }} -------------------------------------------------------------------------------- /playbooks/edx-east/tanaguru.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Tanaguru 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - aws 7 | - mysql 8 | - tanaguru 9 | -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/edx/etc/minos/conf.d/RolledTrackingLogVoter.yml.j2: -------------------------------------------------------------------------------- 1 | RolledTrackingLogVoter: 2 | config: 3 | tracking_directory: '{{ COMMON_LOG_DIR }}/tracking' -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/edx/etc/minos/conf.d/ZippedTrackingLogVoter.yml.j2: -------------------------------------------------------------------------------- 1 | ZippedTrackingLogVoter: 2 | config: 3 | tracking_directory: '{{ COMMON_LOG_DIR }}/tracking' -------------------------------------------------------------------------------- /playbooks/roles/snort/templates/etc/cron.daily/oinkmaster.j2: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | oinkmaster -C /etc/oinkmaster.conf -o /etc/snort/rules/ > /dev/null 4 | service snort restart 5 | -------------------------------------------------------------------------------- /playbooks/roles/sqoop/templates/sqoop-env.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export SQOOP_HOME={{ SQOOP_HOME }} 4 | export SQOOP_LIB=$SQOOP_HOME/lib 5 | export PATH=$PATH:$SQOOP_HOME/bin 6 | -------------------------------------------------------------------------------- /playbooks/roles/xsy/templates/xsy-supervisor.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source {{ xsy_app_dir }}/xsy_env 4 | cd {{ xsy_code_dir }} 5 | 6 | {{ xsy_venv_bin }}/python run_will.py 7 | -------------------------------------------------------------------------------- /playbooks/roles/kibana/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart nginx 3 | service: name=nginx state=restarted 4 | 5 | - name: reload nginx 6 | service: name=nginx state=reloaded 7 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/robots.txt.j2: -------------------------------------------------------------------------------- 1 | {% for item in NGINX_ROBOT_RULES %} 2 | User-agent: {{ item.agent }} 3 | Disallow: {{ item.disallow }} 4 | 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/tmp/git-identity.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec /usr/bin/ssh -o StrictHostKeyChecking=no {% if MINOS_GIT_IDENTITY %}-i {{ minos_git_identity }}{% endif %} "$@" 3 | -------------------------------------------------------------------------------- /playbooks/roles/alton/templates/alton-supervisor.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source {{ alton_app_dir }}/alton_env 4 | cd {{ alton_code_dir }} 5 | 6 | {{ alton_venv_bin }}/python run_alton.py 7 | -------------------------------------------------------------------------------- /playbooks/roles/go-server/templates/edx/app/go-server/password.txt.j2: -------------------------------------------------------------------------------- 1 | {{ admin_user_password_line.stdout }} 2 | {{ backup_user_password_line.stdout }} 3 | {{ gomatic_user_password_line.stdout }} 4 | -------------------------------------------------------------------------------- /playbooks/roles/common_vars/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # There should never be any side-effecting tasks included in this role. 4 | # It is used solely for making shared variables available across roles. -------------------------------------------------------------------------------- /playbooks/roles/mongo/templates/mongo-s3-backup-s3cfg.j2: -------------------------------------------------------------------------------- 1 | [default] 2 | access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }} 3 | secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }} 4 | bucket_location = US 5 | -------------------------------------------------------------------------------- /playbooks/roles/elasticsearch-monitor/templates/plugins/me.snov.newrelic-elasticsearch/newrelic-elasticsearch-plugin-1.4.1/config/plugin.json.j2: -------------------------------------------------------------------------------- 1 | {{ elasticsearch_monitor_plugin_config | to_nice_json }} 2 | -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuiescenceVoterPython.yml.j2: -------------------------------------------------------------------------------- 1 | ProccessQuiescenceVoter: 2 | config: 3 | process_name: 'python' 4 | username: '{{ common_web_user }}' 5 | -------------------------------------------------------------------------------- /docker/build/ecommerce/docker-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | /usr/sbin/rsyslogd 5 | /edx/app/supervisor/venvs/supervisor/bin/supervisord --nodaemon --configuration /edx/app/supervisor/supervisord.conf 6 | -------------------------------------------------------------------------------- /docker/build/xqueue/docker-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | /usr/sbin/rsyslogd 5 | /edx/app/supervisor/venvs/supervisor/bin/supervisord --nodaemon --configuration /edx/app/supervisor/supervisord.conf 6 | -------------------------------------------------------------------------------- /playbooks/roles/ad_hoc_reporting/templates/edx/bin/mysql.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mysql -u {{ COMMON_MYSQL_READ_ONLY_USER }} -h {{ item.db_host }} -p"{{ COMMON_MYSQL_READ_ONLY_PASS }}" {{ item.db_name }} 3 | -------------------------------------------------------------------------------- /terraform/README: -------------------------------------------------------------------------------- 1 | This directory contains the remains of an original attempt at using terraform. 2 | 3 | Future work has been moved to a new repository. Work is planned to open source our terraform modules. 4 | -------------------------------------------------------------------------------- /playbooks/roles/harstorage/templates/edx/app/harstorage/harstorage.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # {{ ansible_managed }} 4 | 5 | {{ harstorage_venv_dir }}/bin/paster serve {{ harstorage_etc }}/production.ini 6 | -------------------------------------------------------------------------------- /playbooks/edx-east/xsy.yml: -------------------------------------------------------------------------------- 1 | # Configure an admin instance with jenkins and asgard. 2 | - name: Configure instance(s) 3 | hosts: all 4 | become: True 5 | gather_facts: True 6 | roles: 7 | - aws 8 | - xsy 9 | -------------------------------------------------------------------------------- /playbooks/roles/xsy/templates/xsy_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in xsy_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}='{{ value }}' 6 | {% endif %} 7 | {%- endfor %} 8 | 9 | -------------------------------------------------------------------------------- /playbooks/roles/common/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common_vars 4 | - server_utils 5 | - role: user 6 | user_info: "{{ COMMON_USER_INFO }}" 7 | - role: security 8 | when: COMMON_SECURITY_UPDATES 9 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart nginx 3 | service: 4 | name: nginx 5 | state: restarted 6 | 7 | - name: reload nginx 8 | service: 9 | name: nginx 10 | state: reloaded 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/aws.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy aws 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common 10 | - aws 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/edx_service_rds.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Build service RDS instances 4 | hosts: all 5 | connection: local 6 | # Needed for timestamps 7 | gather_facts: True 8 | roles: 9 | - edx_service_rds 10 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-shell-cms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py cms shell --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-shell-lms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py lms shell --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edxapp_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | {% for name,value in edxapp_environment.items() %} 3 | {%- if value %} 4 | export {{ name }}="{{ value }}" 5 | {%- endif %} 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/flower/templates/edx/app/flower/flower_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | {% for name,value in flower_environment.items() %} 3 | {%- if value %} 4 | export {{ name }}="{{ value }}" 5 | {%- endif %} 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/templates/edx/app/xqwatcher/data/requirements.txt.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for requirement in item.PYTHON_REQUIREMENTS %} 4 | {{ requirement.name }}=={{ requirement.version }} 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /docker/plays/insights.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Insights 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - docker 10 | - insights 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/minos.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edxapp 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - minos 11 | -------------------------------------------------------------------------------- /playbooks/group_vars/README.md: -------------------------------------------------------------------------------- 1 | After EC2 discovery variables in the files that match any 2 | of the discovered groups will be set. 3 | 4 | For convenience a single variable is set 5 | for every Group tag for conditional task execution. 6 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets-cms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets cms --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets-lms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets lms --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/sites-available/robots.j2: -------------------------------------------------------------------------------- 1 | {% if NGINX_ROBOT_RULES|length > 0 %} 2 | location /robots.txt { 3 | root {{ nginx_app_dir }}; 4 | try_files $uri /robots.txt =404; 5 | } 6 | {% endif %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/tools_jenkins/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - edxapp_common 5 | - role: jenkins_master 6 | jenkins_plugins: "{{ jenkins_tools_plugins }}" 7 | jenkins_version: "{{ jenkins_tools_version }}" 8 | -------------------------------------------------------------------------------- /playbooks/roles/user/templates/default.profile.j2: -------------------------------------------------------------------------------- 1 | umask 022 2 | # if running bash 3 | if [ -n "$BASH_VERSION" ]; then 4 | # include .bashrc if it exists 5 | if [ -f "$HOME/.bashrc" ]; then 6 | . "$HOME/.bashrc" 7 | fi 8 | fi 9 | -------------------------------------------------------------------------------- /playbooks/roles/user/templates/restricted.sudoers.conf.j2: -------------------------------------------------------------------------------- 1 | {% for user in user_info -%} 2 | {% if 'sudo_cmds' in user -%} 3 | {% for cmd in user['sudo_cmds'] -%} 4 | {{ user['name'] }} {{ cmd }} 5 | {% endfor %} 6 | {% endif %} 7 | {% endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/edx-east/README.md: -------------------------------------------------------------------------------- 1 | This directory contains playbooks used by edx-east 2 | for provisioning 3 | 4 | ``` 5 | ansible-playbook -c ssh -vvv --user=ubuntu -i ./ec2.py -e 'secure_dir=path/to/configuration-secure/ansible' 6 | ``` 7 | -------------------------------------------------------------------------------- /playbooks/roles/discovery/templates/edx/app/discovery/discovery_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in discovery_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in ecommerce_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/forum/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | - role: rbenv 6 | rbenv_user: "{{ forum_user }}" 7 | rbenv_dir: "{{ forum_app_dir }}" 8 | rbenv_ruby_version: "{{ forum_ruby_version }}" 9 | -------------------------------------------------------------------------------- /playbooks/roles/insights/templates/edx/app/insights/insights_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in insights_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/templates/jenkins_env.j2: -------------------------------------------------------------------------------- 1 | # Configure JavaScript coverage 2 | export JSCOVER_JAR=/usr/local/bin/JSCover-all-{{ jscover_version }}.jar 3 | 4 | # Set the display to the virtual frame buffer (Xvfb) 5 | export DISPLAY=:1 6 | -------------------------------------------------------------------------------- /playbooks/roles/memcache/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # Installs memcached 2 | 3 | - name: Install memcached 4 | apt: 5 | name: memcached 6 | state: present 7 | update_cache: yes 8 | tags: 9 | - install 10 | - install:memcache 11 | -------------------------------------------------------------------------------- /playbooks/roles/programs/templates/edx/app/programs/programs_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in programs_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /docker/plays/forum.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy forum 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - forum 12 | -------------------------------------------------------------------------------- /playbooks/roles/alton/templates/boto.j2: -------------------------------------------------------------------------------- 1 | {% for deployment, creds in ALTON_AWS_CREDENTIALS.iteritems() %} 2 | [profile {{ deployment }}] 3 | aws_access_key_id = {{ creds.access_id }} 4 | aws_secret_access_key = {{ creds.secret_key }} 5 | 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/swapfile/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: "Kamal Nasser" 4 | description: swapfile 5 | license: MIT 6 | min_ansible_version: 1.4 7 | version: 0.4 8 | categories: 9 | - system 10 | dependencies: [] 11 | -------------------------------------------------------------------------------- /docker/plays/edxapp.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edxapp 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - edxapp 12 | -------------------------------------------------------------------------------- /playbooks/edx-east/stop_all_edx_services.yml: -------------------------------------------------------------------------------- 1 | - name: Stop all services 2 | hosts: all 3 | become: True 4 | gather_facts: False 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - stop_all_edx_services 10 | -------------------------------------------------------------------------------- /playbooks/roles/analytics_pipeline/templates/client.cfg.j2: -------------------------------------------------------------------------------- 1 | [hadoop] 2 | version: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_VERSION }} 3 | command: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_COMMAND }} 4 | streaming-jar: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_STREAMING_JAR }} 5 | -------------------------------------------------------------------------------- /playbooks/roles/credentials/templates/edx/app/credentials/credentials_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in credentials_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in ecommerce_worker_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/git_ssh.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | {% if EDXAPP_USE_GIT_IDENTITY %} 3 | exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ edxapp_git_identity }} "$@" 4 | {% else %} 5 | exec /usr/bin/ssh -o StrictHostKeyChecking=no "$@" 6 | {% endif %} -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/tasks/python.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Requests library is required for both the github status 4 | # script, as well as the sitespeed cookie script. 5 | - name: Install requests Python library 6 | pip: name=requests state=present 7 | -------------------------------------------------------------------------------- /playbooks/roles/rbenv/templates/ruby_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in rbenv_environment.items() %} 4 | {% if value %} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {% endfor %} 8 | eval "$(rbenv init -)" 9 | -------------------------------------------------------------------------------- /playbooks/run_role.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Runs an ansible role 3 | # Usage: 4 | # ansible-playbook ./run_role.yml -i "hostname," -e role=my_awesome_role 5 | # 6 | - hosts: all 7 | become: True 8 | gather_facts: True 9 | roles: 10 | - "{{role}}" 11 | -------------------------------------------------------------------------------- /docker/plays/rabbitmq.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy rabbitmq 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - rabbitmq 12 | -------------------------------------------------------------------------------- /playbooks/edx-east/flower.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy celery flower (monitoring tool) 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - flower 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/notifier.yml: -------------------------------------------------------------------------------- 1 | - name: Configure notifier instance 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - notifier 11 | 12 | -------------------------------------------------------------------------------- /playbooks/group_vars/tag_environment_prod: -------------------------------------------------------------------------------- 1 | --- 2 | secure_dir: '../../configuration-secure/ansible' 3 | # this indicates the path to site-specific (with precedence) 4 | # things like nginx template files 5 | local_dir: '../../configuration-secure/ansible/local' 6 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/templates/pip-accel.conf.j2: -------------------------------------------------------------------------------- 1 | [pip-accel] 2 | auto-install = no 3 | data-directory = ~/.pip-accel 4 | download-cache = ~/.pip/download-cache 5 | s3-bucket = edx-platform.pip-accel-cache 6 | s3-prefix = precise64 7 | s3-readonly = no 8 | -------------------------------------------------------------------------------- /docker/plays/ecommerce.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy ecommerce 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - ecommerce 12 | -------------------------------------------------------------------------------- /playbooks/edx-east/edx_ansible.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy the edx_ansible on AWS 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common 10 | - edx_ansible 11 | -------------------------------------------------------------------------------- /playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in analytics_api_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/edx_notes_api/templates/edx/app/edx_notes_api/edx_notes_api_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in edx_notes_api_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-runserver-cms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py cms runserver {{ edxapp_cms_gunicorn_port }} --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-runserver-lms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py lms runserver {{ edxapp_lms_gunicorn_port }} --settings $EDX_PLATFORM_SETTINGS 4 | -------------------------------------------------------------------------------- /playbooks/roles/gluster/defaults/main.yml: -------------------------------------------------------------------------------- 1 | gluster_primary_ip: 127.0.0.1 2 | gluster_peers: 3 | gluster_volumes: 4 | - path: /mnt/gfsv0 5 | name: gfsv0 6 | replicas: 2 7 | cache_size: 128MB 8 | security: "*" 9 | mount_location: /mnt/data 10 | -------------------------------------------------------------------------------- /playbooks/roles/go-agent-docker-server/README.md: -------------------------------------------------------------------------------- 1 | ##In order to use this role you must use a specific set of AMIs 2 | [This role is for use with the AWS ECS AMIs listed here](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/launch_container_instance.html) 3 | -------------------------------------------------------------------------------- /docker/build/edxapp/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | DOCKER_TLD: "edx" 4 | 5 | # prevents Travis from giving up on the build 6 | COMMON_PIP_VERBOSITY: "-vvvv" 7 | 8 | EDXAPP_MYSQL_HOST: "db.{{ DOCKER_TLD }}" 9 | EDXAPP_MONGO_HOSTS: 10 | - "mongo.{{ DOCKER_TLD }}" 11 | -------------------------------------------------------------------------------- /docker/build/insights/insights-temp.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Insights 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - insights 12 | -------------------------------------------------------------------------------- /docker/plays/analytics_api.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Analytics API 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - analytics_api 12 | -------------------------------------------------------------------------------- /playbooks/edx-east/harstorage.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Harstorage 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | nginx_default_sites: 7 | - harstorage 8 | roles: 9 | - aws 10 | - mongo 11 | - nginx 12 | - harstorage 13 | -------------------------------------------------------------------------------- /playbooks/roles/forum/templates/forum_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in forum_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | 9 | eval "$(rbenv init -)" 10 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_master/tasks/datadog.yml: -------------------------------------------------------------------------------- 1 | - name: Enable jenkins datadog 2 | shell: cp /etc/dd-agent/conf.d/jenkins.yaml.example /etc/dd-agent/conf.d/jenkins.yaml 3 | args: 4 | creates: /etc/dd-agent/conf.d/jenkins.yaml 5 | notify: restart the datadog service 6 | -------------------------------------------------------------------------------- /playbooks/roles/security/templates/etc/apt/apt.conf.d/10periodic: -------------------------------------------------------------------------------- 1 | APT::Periodic::Enable "1"; 2 | APT::Periodic::Update-Package-Lists "1"; 3 | APT::Periodic::Download-Upgradeable-Packages "1"; 4 | APT::Periodic::AutocleanInterval "7"; 5 | APT::Periodic::Unattended-Upgrade "1"; 6 | -------------------------------------------------------------------------------- /playbooks/roles/browsers/templates/xvfb.conf.j2: -------------------------------------------------------------------------------- 1 | description "Xvfb X Server" 2 | start on (net-device-up and local-filesystems and runlevel [2345]) 3 | stop on runlevel [016] 4 | exec /usr/bin/Xvfb {{ browser_xvfb_display }} -screen 0 1024x768x24 5 | respawn 6 | respawn limit 15 5 7 | -------------------------------------------------------------------------------- /playbooks/vagrant/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | 3 | jinja2_extensions=jinja2.ext.do 4 | host_key_checking = False 5 | library=../library 6 | roles_path=../roles 7 | callback_plugins=../callback_plugins 8 | ansible_managed=This file is created and updated by ansible, edit at your peril 9 | -------------------------------------------------------------------------------- /playbooks/continuous_delivery/templates/local/launch_template.yml.j2: -------------------------------------------------------------------------------- 1 | keypair_id: {{ unique_key_name.stdout }} 2 | key_material_file: {{ artifact_path }}/key.pem 3 | instance_id: {{ ec2_instance_register.instances[0].id }} 4 | instance_ip: {{ ec2_instance_register.instances[0].private_ip }} -------------------------------------------------------------------------------- /playbooks/roles/edxapp_common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install system packages 3 | apt: 4 | name: "{{ item }}" 5 | state: present 6 | update_cache: yes 7 | with_items: "{{ edxapp_common_debian_pkgs }}" 8 | tags: 9 | - install 10 | - install:base 11 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_admin/templates/edx/var/jenkins/boto.j2: -------------------------------------------------------------------------------- 1 | {% for deployment, creds in JENKINS_ADMIN_AWS_CREDENTIALS.iteritems() %} 2 | [profile {{ deployment }}] 3 | aws_access_key_id = {{ creds.access_id }} 4 | aws_secret_access_key = {{ creds.secret_key }} 5 | 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/tasks/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Tests for this role 3 | 4 | 5 | ### Tests ### 6 | - name: Verify java cmd is using v 1.8 7 | shell: java -version 8 | register: java_version 9 | - assert: 10 | that: 11 | - "'1.8.0' in java_version.stderr" 12 | -------------------------------------------------------------------------------- /docker/plays/jenkins_analytics.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy the analytics jenkins 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - jenkins_analytics 12 | -------------------------------------------------------------------------------- /playbooks/roles/flower/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart flower 3 | supervisorctl: 4 | state: restarted 5 | supervisorctl_path: "{{ supervisor_ctl }}" 6 | config: "{{ supervisor_cfg }}" 7 | name: "{{ FLOWER_USER }}" 8 | sudo_user: "{{ supervisor_service_user }}" 9 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_admin/templates/edx/var/jenkins/aws_config.j2: -------------------------------------------------------------------------------- 1 | {% for deployment, creds in JENKINS_ADMIN_AWS_CREDENTIALS.iteritems() %} 2 | [profile {{ deployment }}] 3 | aws_access_key_id = {{ creds.access_id }} 4 | aws_secret_access_key = {{ creds.secret_key }} 5 | 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_master/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - role: oraclejdk 5 | tags: java 6 | oraclejdk_version: "7u51" 7 | oraclejdk_base: "jdk1.7.0_51" 8 | oraclejdk_build: "b13" 9 | oraclejdk_link: "/usr/lib/jvm/java-7-oracle" 10 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/tasks/packer.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Download packer 3 | shell: "curl -L {{ packer_url }} -o /var/tmp/packer.zip" 4 | args: 5 | creates: /var/tmp/packer.zip 6 | 7 | - name: Unzip packer 8 | unarchive: src=/var/tmp/packer.zip dest=/usr/local/bin copy=no 9 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/settings_local.py.j2: -------------------------------------------------------------------------------- 1 | from .settings import * 2 | 3 | FORUM_DIGEST_EMAIL_SUBJECT = '{{ NOTIFIER_DIGEST_EMAIL_SUBJECT }}' 4 | 5 | CUSTOM_THEME_DIR = '{{ NOTIFIER_CODE_DIR }}/{{ NOTIFIER_THEME_NAME }}/' 6 | TEMPLATE_DIRS = (CUSTOM_THEME_DIR + 'templates',) 7 | -------------------------------------------------------------------------------- /docker/plays/discovery.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Discovery 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - nginx 10 | - role: discovery 11 | nginx_default_sites: 12 | - discovery -------------------------------------------------------------------------------- /playbooks/create_django_ida.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Creates a new ansible role 3 | # Usage: 4 | # ansible-playbook ./create_django_ida.yml -i "localhost," -c local -e my_role_name=my_awesome_role 5 | # 6 | - hosts: localhost 7 | gather_facts: False 8 | roles: 9 | - ansible-role-django-ida 10 | -------------------------------------------------------------------------------- /playbooks/create_role.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Creates a new ansible role 3 | # Usage: 4 | # ansible-playbook -c local --limit "localhost," ./create_role.yml -i "localhost," -e my_role_name=my_awesome_role 5 | # 6 | - hosts: localhost 7 | gather_facts: False 8 | roles: 9 | - ansible-role 10 | -------------------------------------------------------------------------------- /playbooks/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Common variables are defined in the common_vars role on which this 4 | # role depends. This is to allow sharing vars without creating 5 | # side-effects. Any vars requred by this role should be added to 6 | # common_vars/defaults/main.yml 7 | -------------------------------------------------------------------------------- /playbooks/roles/go-agent-docker-server/templates/edx/app/go-agent-docker-server/autoregister.properties.j2: -------------------------------------------------------------------------------- 1 | agent.auto.register.key={{ GO_SERVER_AUTO_REGISTER_KEY }} 2 | agent.auto.register.resources={{ GO_AGENT_DOCKER_RESOURCES }} 3 | agent.auto.register.environments={{ GO_AGENT_DOCKER_ENVIRONMENT }} -------------------------------------------------------------------------------- /util/jenkins/restart-xqueue.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd configuration 4 | pip install -r requirements.txt 5 | env 6 | 7 | command="/edx/bin/supervisorctl restart xqueue" 8 | 9 | ansible tag_Name_${environment}-${deployment}-commoncluster -i playbooks/ec2.py -u ubuntu -s -a "$command" 10 | -------------------------------------------------------------------------------- /playbooks/roles/browsermob-proxy/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # browsermob-proxy 2 | browsermob_proxy_version: '2.0.0' 3 | browsermob_proxy_url: 'https://github.com/lightbody/browsermob-proxy/releases/download/browsermob-proxy-{{ browsermob_proxy_version }}/browsermob-proxy-{{ browsermob_proxy_version }}-bin.zip' 4 | -------------------------------------------------------------------------------- /playbooks/roles/kibana/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | KIBANA_SERVER_NAME: "192.168.33.10" 3 | KIBANA_NGINX_PORT: 80 4 | KIBANA_SSL_NGINX_PORT: 443 5 | 6 | kibana_app_dir: /edx/app/kibana 7 | kibana_file: kibana-3.0.0.tar.gz 8 | kibana_url: "https://download.elasticsearch.org/kibana/kibana/{{ kibana_file }}" 9 | -------------------------------------------------------------------------------- /playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2: -------------------------------------------------------------------------------- 1 | TrackingLogVoter: 2 | config: 3 | aws_profile: !!null 4 | local_directory: '{{ COMMON_LOG_DIR }}/tracking' 5 | s3_bucket: '{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}' 6 | bucket_path_prefix: 'logs/tracking' 7 | -------------------------------------------------------------------------------- /playbooks/roles/snort/defaults/main.yml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | SNORT_OINKCODE: 'oinkcode' 4 | SNORT_RULES_URL: [ 'http://www.snort.org/pub-bin/oinkmaster.cgi/{{ SNORT_OINKCODE }}/snortrules-snapshot-2931.tar.gz', 5 | 'http://rules.emergingthreats.net/open/snort-2.9.0/emerging.rules.tar.gz' ] 6 | -------------------------------------------------------------------------------- /playbooks/roles/forum/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart the forum service 3 | supervisorctl: 4 | name: forum 5 | supervisorctl_path: "{{ supervisor_ctl }}" 6 | config: "{{ supervisor_cfg }}" 7 | state: restarted 8 | when: forum_installed is defined and not disable_edx_services 9 | -------------------------------------------------------------------------------- /playbooks/analytics-jenkins.yml: -------------------------------------------------------------------------------- 1 | # Usage: ansible-playbook -i jenkins-host, -e@path/to/jenkins-extra-vars.yml -e@path/to/secure.yml -e 'COMMON_ENABLE_NEWRELIC=false' -e 'COMMON_ENABLE_DATADOG=false' 2 | 3 | - name: Configure instance(s) 4 | hosts: all 5 | become: True 6 | roles: 7 | - jenkins_analytics 8 | -------------------------------------------------------------------------------- /playbooks/roles/alton/templates/alton_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for name,value in alton_environment.items() -%} 4 | {%- if value -%} 5 | export {{ name }}="{{ value }}" 6 | {% endif %} 7 | {%- endfor %} 8 | 9 | export WILL_BOTO_PROFILES="{{ ALTON_AWS_CREDENTIALS|join(';') }}" 10 | 11 | -------------------------------------------------------------------------------- /playbooks/roles/haproxy/templates/haproxy.rsyslog.j2: -------------------------------------------------------------------------------- 1 | if ($programname == 'haproxy' and $syslogseverity-text == 'info') then -{{ COMMON_LOG_DIR }}/haproxy/haproxy-info.log 2 | & ~ 3 | if ($programname == 'haproxy' and $syslogseverity-text == 'notice') then -{{ COMMON_LOG_DIR }}/haproxy/haproxy-notice.log 4 | & ~ 5 | -------------------------------------------------------------------------------- /util/jenkins/restart-xqueue_consumer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd configuration 4 | pip install -r requirements.txt 5 | env 6 | 7 | command="/edx/bin/supervisorctl restart xqueue_consumer" 8 | 9 | ansible tag_Name_${environment}-${deployment}-commoncluster -i playbooks/ec2.py -u ubuntu -s -a "$command" 10 | -------------------------------------------------------------------------------- /playbooks/go-agent.yml: -------------------------------------------------------------------------------- 1 | # ansible-playbook -i 'admin.edx.org,' ./hotg.yml -e@/path/to/ansible/vars/edx.yml -e@/path/to/secure/ansible/vars/edx_admin.yml 2 | 3 | - name: Install go-agent 4 | hosts: all 5 | become: True 6 | gather_facts: True 7 | roles: 8 | - aws 9 | - supervisor 10 | - go-agent 11 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/cms.env.json.j2: -------------------------------------------------------------------------------- 1 | {% do cms_env_config.update(EDXAPP_ENV_EXTRA) %} 2 | {% for key, value in cms_env_config.iteritems() -%} 3 | {% if value == 'None' -%} 4 | {% do cms_env_config.update({key: None }) %} 5 | {%- endif %} 6 | {%- endfor %} 7 | {{ cms_env_config | to_nice_json }} 8 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/lms.env.json.j2: -------------------------------------------------------------------------------- 1 | {% do lms_env_config.update(EDXAPP_ENV_EXTRA) %} 2 | {% for key, value in lms_env_config.iteritems() -%} 3 | {% if value == 'None' -%} 4 | {% do lms_env_config.update({key: None }) %} 5 | {%- endif %} 6 | {%- endfor %} 7 | {{ lms_env_config | to_nice_json }} 8 | -------------------------------------------------------------------------------- /playbooks/roles/harprofiler/templates/validate_harprofiler_install.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | # This script confirms that harprofiler can successfully run on the 5 | # target machine. 6 | source {{ harprofiler_venv_dir }}/bin/activate 7 | cd {{ harprofiler_dir }} 8 | python harprofiler.py 9 | 10 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/templates/create_root.js.j2: -------------------------------------------------------------------------------- 1 | // Add super user 2 | conn = new Mongo(); 3 | db = conn.getDB("admin"); 4 | db.createUser( 5 | { 6 | "user": "{{ MONGO_ADMIN_USER }}", 7 | "pwd": "{{ MONGO_ADMIN_PASSWORD }}", 8 | "roles": ["root"] 9 | } 10 | ); 11 | 12 | 13 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/ansible_overrides.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {{ role_name }}_gunicorn_host: 127.0.0.1 3 | {{ role_name|upper }}_MYSQL: 'db' 4 | {{ role_name|upper }}_DJANGO_SETTINGS_MODULE: '{{ role_name }}.settings.devstack' 5 | {{ role_name|upper }}_MYSQL_MATCHER: '%' 6 | 7 | -------------------------------------------------------------------------------- /playbooks/roles/vhost/templates/etc/init/sync-on-stop.conf.j2: -------------------------------------------------------------------------------- 1 | start on stopped supervisor 2 | description "sync tracking logs on supervisor shutdown" 3 | script 4 | /usr/sbin/logrotate -f /etc/logrotate.d/hourly/tracking.log 5 | /usr/sbin/logrotate -f /etc/logrotate.d/hourly/edx-services 6 | end script 7 | -------------------------------------------------------------------------------- /playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2: -------------------------------------------------------------------------------- 1 | // we only ever connect to secondaries, avoid people needing to remember to type this 2 | rs.slaveOk(); 3 | 4 | // This uses the DB name rather than the replica set, which I think is more useful 5 | var prompt = function() { 6 | return db.getName() + "> "; 7 | } 8 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/hosts.j2: -------------------------------------------------------------------------------- 1 | 127.0.0.1 {{ COMMON_HOSTNAME }} localhost 2 | 3 | # The following lines are desirable for IPv6 capable hosts 4 | ::1 ip6-localhost ip6-loopback 5 | fe00::0 ip6-localnet 6 | ff00::0 ip6-mcastprefix 7 | ff02::1 ip6-allnodes 8 | ff02::2 ip6-allrouters 9 | ff02::3 ip6-allhosts 10 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/lms.auth.json.j2: -------------------------------------------------------------------------------- 1 | {% do lms_auth_config.update(EDXAPP_LMS_AUTH_EXTRA) %} 2 | {% for key, value in lms_auth_config.iteritems() -%} 3 | {% if value == 'None' -%} 4 | {% do lms_auth_config.update({key: None }) %} 5 | {%- endif %} 6 | {%- endfor %} 7 | {{ lms_auth_config | to_nice_json }} 8 | -------------------------------------------------------------------------------- /docker/plays/credentials.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Credentials 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - nginx 10 | - docker 11 | - role: credentials 12 | nginx_default_sites: 13 | - credentials 14 | -------------------------------------------------------------------------------- /playbooks/go-agent-docker.yml: -------------------------------------------------------------------------------- 1 | # ansible-playbook -i 'admin.edx.org,' ./hotg.yml -e@/path/to/ansible/vars/edx.yml -e@/path/to/secure/ansible/vars/edx_admin.yml 2 | 3 | - name: Install go-agent-docker-server 4 | hosts: all 5 | become: True 6 | gather_facts: True 7 | roles: 8 | - aws 9 | - go-agent-docker-server 10 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role/templates/meta/main.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {% include 'roles/ansible-role/templates/header.j2' %} 3 | # 4 | # Role includes for role {{ my_role_name }} 5 | # 6 | # Example: 7 | # 8 | # dependencies: 9 | # - { 10 | # role: my_role 11 | # my_role_var0: "foo" 12 | # my_role_var1: "bar" 13 | # } 14 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/cms.auth.json.j2: -------------------------------------------------------------------------------- 1 | {% do cms_auth_config.update(EDXAPP_CMS_AUTH_EXTRA) %} 2 | {% for key, value in cms_auth_config.iteritems() -%} 3 | {% if value == 'None' -%} 4 | {% do cms_auth_config.update({key: None }) %} 5 | {%- endif %} 6 | {%- endfor %} 7 | 8 | {{ cms_auth_config | to_nice_json }} 9 | -------------------------------------------------------------------------------- /playbooks/roles/haproxy/templates/haproxy.logrotate.j2: -------------------------------------------------------------------------------- 1 | {{ COMMON_LOG_DIR }}/haproxy/*.log { 2 | weekly 3 | missingok 4 | rotate 7 5 | compress 6 | delaycompress 7 | notifempty 8 | create 640 root adm 9 | sharedscripts 10 | postrotate 11 | /etc/init.d/haproxy reload > /dev/null 12 | endscript 13 | } 14 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_master/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart Jenkins 3 | service: 4 | name: jenkins 5 | state: restarted 6 | 7 | - name: start nginx 8 | service: 9 | name: nginx 10 | state: started 11 | 12 | - name: reload nginx 13 | service: 14 | name: nginx 15 | state: reloaded 16 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/files/disable-transparent-hugepages.conf: -------------------------------------------------------------------------------- 1 | description "set transparent hugepage to never" 2 | 3 | start on starting mongod 4 | task 5 | 6 | script 7 | echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled 8 | echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag 9 | end script 10 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/notifier-scheduler-supervisor.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source {{ notifier_app_dir }}/notifier_env 4 | cd {{ NOTIFIER_CODE_DIR }} 5 | 6 | export PID='/var/tmp/notifier-celery-workers.pid' 7 | export LANG=en_US.UTF-8 8 | 9 | {{ NOTIFIER_VENV_DIR }}/bin/python manage.py scheduler 10 | 11 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME_env.j2: -------------------------------------------------------------------------------- 1 | # {{ '{{' }} ansible_managed }} 2 | 3 | {{ '{%' }} for name,value in {{ role_name }}_environment.items() -%} 4 | {{ '{%' }}- if value -%} 5 | export {{ '{{' }} name }}="{{ '{{' }} value }}" 6 | {{ '{%' }} endif %} 7 | {{ '{%' }}- endfor %} 8 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx-release.j2: -------------------------------------------------------------------------------- 1 | server { 2 | listen 8099 default_server; 3 | 4 | location = /versions.html { 5 | alias {{ nginx_cfg.version_html }}; 6 | } 7 | 8 | location = /versions.json { 9 | alias {{ nginx_cfg.version_json }}; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /docker/build/xqueue/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | DOCKER_TLD: "xqueue" 4 | 5 | CONFIGURATION_REPO: "https://github.com/edx/configuration.git" 6 | CONFIGURATION_VERSION: "hack2015/docker" 7 | XQUEUE_SYSLOG_SERVER: "localhost" 8 | XQUEUE_RABBITMQ_HOSTNAME: "rabbit.{{ DOCKER_TLD }}" 9 | XQUEUE_MYSQL_HOST: "db.{{ DOCKER_TLD }}" 10 | -------------------------------------------------------------------------------- /playbooks/roles/go-agent/templates/edx/app/go-agent/go-agent.j2: -------------------------------------------------------------------------------- 1 | GO_SERVER={{ GO_AGENT_SERVER_IP }} 2 | export GO_SERVER 3 | GO_SERVER_PORT={{ GO_AGENT_SERVER_PORT }} 4 | export GO_SERVER_PORT 5 | AGENT_WORK_DIR=/var/lib/${SERVICE_NAME:-go-agent} 6 | export AGENT_WORK_DIR 7 | DAEMON=Y 8 | VNC=N 9 | export JAVA_HOME="{{ GO_AGENT_ORACLEJDK_LINK }}" -------------------------------------------------------------------------------- /playbooks/roles/rabbitmq/templates/etc/logrotate.d/rabbitmq.j2: -------------------------------------------------------------------------------- 1 | # We want to hit the top level queues and any vhost queues 2 | # such as notifier and fulfillment 3 | {{ rabbitmq_log_dir }}/*.log {{ rabbitmq_log_dir }}/*/*.log { 4 | compress 5 | dateext 6 | dateformat -%Y%m%d-%s 7 | missingok 8 | daily 9 | rotate 3 10 | } 11 | -------------------------------------------------------------------------------- /playbooks/roles/local_dev/templates/share_x11.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Change permissions on the X11 session cookie 4 | # so application users can use the same X11 session. 5 | # This is very insecure and should *only* be used for local VMs. 6 | if [ -f {{ localdev_xauthority }} ]; then 7 | chmod og+r {{ localdev_xauthority }} 8 | fi 9 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/templates/usr.bin.python-sandbox.j2: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /usr/bin/python-sandbox { 4 | #include 5 | 6 | /usr/bin/python-sandbox mr, 7 | /usr/include/python2.7/** r, 8 | /usr/local/lib/python2.7/** r, 9 | /usr/lib/python2.7** rix, 10 | 11 | /tmp/** rix, 12 | } 13 | 14 | -------------------------------------------------------------------------------- /playbooks/roles/datadog/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | DATADOG_API_KEY: "SPECIFY_KEY_HERE" 3 | 4 | datadog_agent_version: '1:5.1.1-546' 5 | 6 | datadog_apt_key: "0x226AE980C7A7DA52" 7 | datadog_debian_pkgs: 8 | - apparmor-utils 9 | - build-essential 10 | - curl 11 | - g++ 12 | - gcc 13 | - ipython 14 | - pkg-config 15 | - rsyslog 16 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/templates/log-mongo-serverStatus.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON 3 | /usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' 4 | -------------------------------------------------------------------------------- /playbooks/roles/rabbitmq/templates/etc/rabbitmq/rabbitmq.config.j2: -------------------------------------------------------------------------------- 1 | % {{ ansible_managed }} 2 | 3 | [{rabbit, [ 4 | {log_levels, [{connection, info}]}, 5 | {# 6 | Note: That these names should include the node name prefix. eg. 'rabbit@hostname' 7 | #} 8 | {cluster_nodes, {['{{ RABBITMQ_CLUSTERED_HOSTS|join("\',\'") }}'], disc}} 9 | ]}]. 10 | -------------------------------------------------------------------------------- /playbooks/edx-east/edx_vpc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Sample command: ansible-playbook -c local -i localhost, edx_vpc.yml -e@/Users/feanil/src/edx-secure/cloud_migrations/vpcs/test.yml -vvv 3 | - name: Create a simple empty vpc 4 | hosts: all 5 | connection: local 6 | gather_facts: False 7 | vars: 8 | vpc_state: present 9 | roles: 10 | - edx_vpc 11 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/notifier-celery-workers-supervisor.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source {{ notifier_app_dir }}/notifier_env 4 | cd {{ NOTIFIER_CODE_DIR }} 5 | 6 | export PID='/var/tmp/notifier-scheduler.pid' 7 | export LANG=en_US.UTF-8 8 | 9 | {{ NOTIFIER_VENV_DIR }}/bin/python manage.py celery worker -l {{ NOTIFIER_LOG_LEVEL }} 10 | 11 | -------------------------------------------------------------------------------- /playbooks/roles/forum/templates/forum.conf.j2: -------------------------------------------------------------------------------- 1 | [program:forum] 2 | command={{ forum_supervisor_wrapper }} 3 | priority=999 4 | user={{ common_web_user }} 5 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 6 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 7 | killasgroup=true 8 | stopasgroup=true 9 | stopsignal=QUIT 10 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/sites-available/certs.j2: -------------------------------------------------------------------------------- 1 | server { 2 | listen {{ CERTS_NGINX_PORT }} default_server; 3 | 4 | location / { 5 | root {{ CERTS_WEB_ROOT }}; 6 | {% if CERTS_ENABLE_BASIC_AUTH|bool %} 7 | {% include "basic-auth.j2" %} 8 | {% endif %} 9 | try_files $uri $uri/valid.html =404; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /playbooks/roles/xsy/templates/xsy.conf.j2: -------------------------------------------------------------------------------- 1 | [program:xsy] 2 | 3 | command={{ xsy_supervisor_wrapper }} 4 | priority=999 5 | user={{ common_web_user }} 6 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 7 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 8 | killasgroup=true 9 | stopasgroup=true 10 | stopsignal=QUIT 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/mongo.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy MongoDB 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - aws 7 | - mongo 8 | - role: datadog 9 | when: COMMON_ENABLE_DATADOG 10 | - role: splunkforwarder 11 | when: COMMON_ENABLE_SPLUNKFORWARDER 12 | - role: newrelic 13 | when: COMMON_ENABLE_NEWRELIC 14 | -------------------------------------------------------------------------------- /playbooks/roles/alton/templates/alton.conf.j2: -------------------------------------------------------------------------------- 1 | [program:alton] 2 | 3 | command={{ alton_supervisor_wrapper }} 4 | priority=999 5 | user={{ common_web_user }} 6 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 7 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 8 | killasgroup=true 9 | stopasgroup=true 10 | stopsignal=QUIT 11 | -------------------------------------------------------------------------------- /playbooks/roles/jscover/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Installs JSCover jar. 3 | # Java is a pre-requisite for JSCover. This role is not responsible 4 | # for installing Java. 5 | # 6 | jscover_role_name: jscover 7 | 8 | # JSCover direct download URL 9 | jscover_version: "1.0.2" 10 | jscover_url: "http://files.edx.org/testeng/JSCover-{{ jscover_version }}.zip" 11 | -------------------------------------------------------------------------------- /util/jenkins/restart-xqwatcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd configuration 4 | pip install -r requirements.txt 5 | env 6 | 7 | command="/edx/app/xqwatcher/venvs/supervisor/bin/supervisorctl -c /edx/app/xqwatcher/supervisor/supervisord.conf restart xqwatcher" 8 | 9 | ansible tag_Name_${environment}-${deployment}-xqwatcher -i playbooks/ec2.py -u ubuntu -s -a "$command" 10 | -------------------------------------------------------------------------------- /playbooks/log_server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Build a kibana/logstash/elasticsearch server for capturing and 3 | # analyzing logs. 4 | - name: Configure syslog server 5 | hosts: all 6 | become: True 7 | roles: 8 | - common 9 | - oraclejdk 10 | - elasticsearch 11 | - logstash 12 | - kibana 13 | - role: nginx 14 | nginx_sites: 15 | - kibana 16 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2: -------------------------------------------------------------------------------- 1 | {{ COMMON_LOG_DIR }}/*/edx.log { 2 | create 3 | compress 4 | copytruncate 5 | delaycompress 6 | dateext 7 | dateformat -%Y%m%d-%s 8 | missingok 9 | notifempty 10 | daily 11 | rotate 90 12 | size 1M 13 | postrotate 14 | /usr/bin/killall -HUP rsyslogd 15 | endscript 16 | } 17 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/core-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | fs.default.name 9 | hdfs://localhost:9000 10 | 11 | -------------------------------------------------------------------------------- /docker/build/discovery/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | discovery_gunicorn_host: 0.0.0.0 3 | DISCOVERY_MYSQL: 'db' 4 | DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack' 5 | DISCOVERY_ELASTICSEARCH_HOST: 'es' 6 | DISCOVERY_GUNICORN_EXTRA: '--reload' 7 | COMMON_MYSQL_MIGRATE_USER: '{{ DISCOVERY_MYSQL_USER }}' 8 | COMMON_MYSQL_MIGRATE_PASS: '{{ DISCOVERY_MYSQL_PASSWORD }}' 9 | -------------------------------------------------------------------------------- /docker/build/forum/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | FLOCK_TLD: "edx" 4 | 5 | FORUM_MONGO_HOSTS: 6 | - mongo.{{ FLOCK_TLD }} 7 | 8 | FORUM_ELASTICSEARCH_HOST: "es.{{ FLOCK_TLD }}" 9 | forum_source_repo: "https://github.com/open-craft/cs_comments_service.git" 10 | forum_version: "mongoid5" 11 | FORUM_USE_TCP: "true" 12 | FORUM_RACK_ENV: "staging" 13 | FORUM_SINATRA_ENV: "staging" -------------------------------------------------------------------------------- /docker/build/go-agent/files/go-agent-env-vars: -------------------------------------------------------------------------------- 1 | GO_SERVER=127.0.0.1 2 | export GO_SERVER 3 | GO_SERVER_PORT=8153 4 | export GO_SERVER_PORT 5 | GO_SERVER_SSL_PORT=8154 6 | export GO_SERVER_SSL_PORT 7 | AGENT_WORK_DIR=/var/lib/${SERVICE_NAME:-go-agent} 8 | export AGENT_WORK_DIR 9 | DAEMON=N 10 | VNC=N 11 | export JAVA_HOME="/usr/lib/jvm/java-7-oracle/jre" # SET_BY_GO_INSTALLER__DONT_REMOVE -------------------------------------------------------------------------------- /playbooks/edx-east/antivirus.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Antivirus Scanner 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - aws 7 | - antivirus 8 | - role: datadog 9 | when: COMMON_ENABLE_DATADOG 10 | - role: splunkforwarder 11 | when: COMMON_ENABLE_SPLUNKFORWARDER 12 | - role: newrelic 13 | when: COMMON_ENABLE_NEWRELIC 14 | -------------------------------------------------------------------------------- /playbooks/go-server.yml: -------------------------------------------------------------------------------- 1 | # This playbook installs go-server 2 | # https://www.go.cd/ 3 | # 4 | 5 | - name: Install go-server 6 | hosts: all 7 | become: True 8 | gather_facts: True 9 | roles: 10 | - aws 11 | - go-server 12 | - role: splunkforwarder 13 | when: COMMON_ENABLE_SPLUNKFORWARDER 14 | - role: newrelic 15 | when: COMMON_ENABLE_NEWRELIC 16 | -------------------------------------------------------------------------------- /playbooks/roles/openstack/templates/log-sync-env.sh.j2: -------------------------------------------------------------------------------- 1 | export OS_USERNAME='{{ SWIFT_LOG_SYNC_USERNAME }}' 2 | export OS_PASSWORD='{{ SWIFT_LOG_SYNC_PASSWORD }}' 3 | export OS_TENANT_ID='{{ SWIFT_LOG_SYNC_TENANT_ID }}' 4 | export OS_TENANT_NAME='{{ SWIFT_LOG_SYNC_TENANT_NAME }}' 5 | export OS_AUTH_URL='{{ SWIFT_LOG_SYNC_AUTH_URL }}' 6 | export OS_REGION_NAME='{{ SWIFT_LOG_SYNC_REGION_NAME }}' 7 | -------------------------------------------------------------------------------- /playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf.j2: -------------------------------------------------------------------------------- 1 | {% for name in SPLUNK_INDEXES %} 2 | [{{ name }}] 3 | coldPath = {{ splunk_cold_dir }}/{{ name }}/colddb 4 | homePath = {{ splunk_hot_dir }}/{{ name }}/db 5 | thawedPath = {{ splunk_thawed_dir }}/{{ name }}/thaweddb 6 | coldToFrozenDir = {{ splunk_frozen_dir }}/{{ name }}/frozendb 7 | 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/props.conf.j2: -------------------------------------------------------------------------------- 1 | {% for extraction in SPLUNK_FIELD_EXTRACTIONS %} 2 | {% if 'source' in extraction %} 3 | [source::{{ extraction.source }}] 4 | {% elif 'sourcetype' in extraction %} 5 | [{{ extraction.sourcetype }}] 6 | {% endif %} 7 | EXTRACT-{{ extraction.name }} = {{ extraction.regex }} 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /util/csmh-extended/slow-delete.sh: -------------------------------------------------------------------------------- 1 | MINID=0 2 | MAXID=1003426362 3 | STEP=20000 4 | MIGRATE_USER=migrate 5 | PASSWORD='secret' 6 | HOST='host' 7 | 8 | 9 | for ((i=$MINID-1; i<=$MAXID; i+=$STEP)); do 10 | echo -n "$i"; 11 | time mysql -u $MIGRATE_USER -p$PASSWORD -h $HOST wwc < 11 | name={{ supervisor_service }} 12 | state=restarted 13 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/templates/notifier_env.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% do notifier_env_vars.update(NOTIFIER_ENV_EXTRA) %} 4 | {% for name,value in notifier_env_vars.items() %} 5 | {% if value %} 6 | export {{ name }}="{{ value }}" 7 | {% endif %} 8 | {% endfor %} 9 | {% if NOTIFIER_THEME_NAME != "" %} 10 | export DJANGO_SETTINGS_MODULE=notifier.settings_local 11 | {% endif %} 12 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/header.j2: -------------------------------------------------------------------------------- 1 | # 2 | # edX Configuration 3 | # 4 | # github: https://github.com/edx/configuration 5 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 6 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 7 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 8 | # 9 | # 10 | -------------------------------------------------------------------------------- /playbooks/roles/forum/templates/cs_comments_service.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | description "Comments Service" 4 | 5 | start on runlevel [2345] 6 | stop on runlevel [!2345] 7 | 8 | env PID=/var/tmp/comments_service.pid 9 | 10 | chdir {{ forum_code_dir }} 11 | 12 | script 13 | . {{ forum_app_dir }}/forum_env 14 | {{ forum_app_dir }}/.rbenv/shims/ruby app.rb 15 | 16 | end script 17 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/hdfs.conf.j2: -------------------------------------------------------------------------------- 1 | description "hdfs" 2 | 3 | start on starting yarn 4 | stop on stopping yarn 5 | 6 | setuid {{ hadoop_common_user }} 7 | 8 | pre-start script 9 | . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh 10 | start-dfs.sh 11 | end script 12 | 13 | post-stop script 14 | . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh 15 | stop-dfs.sh 16 | end script 17 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp_common.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd {{ edxapp_code_dir }} 3 | source {{ edxapp_app_dir }}/edxapp_env 4 | 5 | # The default settings set in edxapp_env can be overridden 6 | # using the var $EDX_PLATFORM_SETTINGS_OVERRIDE 7 | 8 | if [[ -n "$EDX_PLATFORM_SETTINGS_OVERRIDE" ]]; then 9 | export EDX_PLATFORM_SETTINGS="$EDX_PLATFORM_SETTINGS_OVERRIDE" 10 | fi 11 | 12 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/yarn.conf.j2: -------------------------------------------------------------------------------- 1 | description "yarn" 2 | 3 | start on runlevel [2345] 4 | stop on runlevel [!2345] 5 | 6 | setuid {{ hadoop_common_user }} 7 | 8 | pre-start script 9 | . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh 10 | start-yarn.sh 11 | end script 12 | 13 | post-stop script 14 | . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh 15 | stop-yarn.sh 16 | end script 17 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role/templates/tasks/main.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {% include 'roles/ansible-role/templates/header.j2' %} 3 | 4 | # 5 | # Tasks for role {{ my_role_name }} 6 | # 7 | # Overview: 8 | # 9 | # 10 | # Dependencies: 11 | # 12 | # 13 | # Example play: 14 | # 15 | # 16 | 17 | - name: stub ansible task 18 | debug: msg="This is a stub task created by the ansible-role role" 19 | notify: notify me 20 | -------------------------------------------------------------------------------- /playbooks/roles/certs/tasks/tag_ec2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Get instance information 4 | action: ec2_facts 5 | 6 | - name: Tag instance 7 | ec2_tag: 8 | resource: "{{ ansible_ec2_instance_id }}" 9 | region: "{{ ansible_ec2_placement_region }}" 10 | tags: 11 | "version:certs" : "{{ CERT_REPO }} {{ certs_checkout.after|truncate(7,True,'') }}" 12 | when: certs_checkout.after is defined 13 | -------------------------------------------------------------------------------- /playbooks/roles/forum/tasks/tag_ec2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: get instance information 3 | action: ec2_facts 4 | 5 | - name: tag instance 6 | ec2_tag: 7 | resource: "{{ ansible_ec2_instance_id }}" 8 | region: "{{ ansible_ec2_placement_region }}" 9 | tags: 10 | "version:forum" : "{{ forum_source_repo }} {{ forum_checkout.after|truncate(7,True,'') }}" 11 | when: forum_checkout.after is defined 12 | -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-course-sandbox.j2: -------------------------------------------------------------------------------- 1 | {{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=({{ item.QUEUE.HANDLERS[0].CODEJAIL.user }}) SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE.HANDLERS[0].CODEJAIL.name }}/bin/python 2 | {{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/bin/kill 3 | {{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill 4 | -------------------------------------------------------------------------------- /playbooks/edx-east/alton.yml: -------------------------------------------------------------------------------- 1 | # Configure an admin instance with jenkins and asgard. 2 | # Usage: ansible-playbook alton.yml -i , -e /admin/edx_admin.yml -e /admin/admin.yml 3 | - name: Configure instance(s) 4 | hosts: all 5 | become: True 6 | gather_facts: True 7 | vars: 8 | serial_count: 1 9 | serial: "{{ serial_count }}" 10 | roles: 11 | - aws 12 | - alton 13 | -------------------------------------------------------------------------------- /playbooks/edx-east/common.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy common 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common 10 | - role: datadog 11 | when: COMMON_ENABLE_DATADOG 12 | - role: splunkforwarder 13 | when: COMMON_ENABLE_SPLUNKFORWARDER 14 | - role: newrelic 15 | when: COMMON_ENABLE_NEWRELIC 16 | -------------------------------------------------------------------------------- /playbooks/edx-east/demo.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy demo course 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - demo 10 | - role: datadog 11 | when: COMMON_ENABLE_DATADOG 12 | - role: splunkforwarder 13 | when: COMMON_ENABLE_SPLUNKFORWARDER 14 | - role: newrelic 15 | when: COMMON_ENABLE_NEWRELIC 16 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role/templates/defaults/main.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | {% include 'roles/ansible-role/templates/header.j2' %} 3 | # 4 | # Defaults for role {{ my_role_name }} 5 | # 6 | 7 | # 8 | # vars are namespaced with the module name. 9 | # 10 | {{ my_role_name }}_role_name: {{ my_role_name }} 11 | 12 | # 13 | # OS packages 14 | # 15 | 16 | {{ my_role_name }}_debian_pkgs: [] 17 | 18 | {{ my_role_name }}_redhat_pkgs: [] 19 | -------------------------------------------------------------------------------- /playbooks/roles/asqatasun/templates/etc/mysql/conf.d/asqatasun.cnf.j2: -------------------------------------------------------------------------------- 1 | [client] 2 | default-character-set={{ default_character_set }} 3 | 4 | [mysql] 5 | default-character-set={{ default_character_set }} 6 | 7 | [mysqld] 8 | collation-server = {{ collation_server }} 9 | init-connect={{ "\'" + init_connect + "\'" }} 10 | character-set-server = {{ character_set_server }} 11 | max_allowed_packet = {{ mysql_max_allowed_packet }} 12 | -------------------------------------------------------------------------------- /playbooks/roles/certs/templates/certs.conf.j2: -------------------------------------------------------------------------------- 1 | [program:certs] 2 | command={{ certs_venv_bin }}/python {{ certs_code_dir }}/certificate_agent.py 3 | priority=999 4 | environment=SERVICE_VARIANT="certs",HOME="/" 5 | user={{ common_web_user }} 6 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 7 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 8 | killasgroup=true 9 | stopasgroup=true 10 | -------------------------------------------------------------------------------- /util/jenkins/ascii-convert.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function ascii_convert { 4 | echo $1 | iconv -f utf8 -t ascii//TRANSLIT//IGNORE 5 | } 6 | 7 | # remove non-ascii chars from build user vars 8 | BUILD_USER_LAST_NAME=$(ascii_convert $BUILD_USER_LAST_NAME) 9 | BUILD_USER_FIRST_NAME=$(ascii_convert $BUILD_USER_FIRST_NAME) 10 | BUILD_USER_ID=$(ascii_convert $BUILD_USER_ID) 11 | BUILD_USER=$(ascii_convert $BUILD_USER) 12 | -------------------------------------------------------------------------------- /playbooks/roles/xserver/templates/xserver_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | 4 | {{ ansible_managed }} 5 | """ 6 | 7 | preload_app = True 8 | timeout = 30 9 | bind = "{{ xserver_gunicorn_host }}:{{ xserver_gunicorn_port }}" 10 | pythonpath = "{{ xserver_code_dir }}" 11 | workers = {{ xserver_gunicorn_workers }} 12 | 13 | {{ XSERVER_GUNICORN_EXTRA_CONF }} 14 | -------------------------------------------------------------------------------- /vagrant/base/cluster/inventory.ini: -------------------------------------------------------------------------------- 1 | [cluster] 2 | cluster1 ansible_ssh_host=192.168.33.100 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 3 | cluster2 ansible_ssh_host=192.168.33.110 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 4 | cluster3 ansible_ssh_host=192.168.33.120 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key 5 | -------------------------------------------------------------------------------- /playbooks/edx-east/aide.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy aide IDS 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - aide 11 | - role: datadog 12 | when: COMMON_ENABLE_DATADOG 13 | - role: splunkforwarder 14 | when: COMMON_ENABLE_SPLUNKFORWARDER 15 | - role: newrelic 16 | when: COMMON_ENABLE_NEWRELIC 17 | -------------------------------------------------------------------------------- /playbooks/edx-east/certs.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy certs 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - certs 11 | - role: datadog 12 | when: COMMON_ENABLE_DATADOG 13 | - role: splunkforwarder 14 | when: COMMON_ENABLE_SPLUNKFORWARDER 15 | - role: newrelic 16 | when: COMMON_ENABLE_NEWRELIC 17 | -------------------------------------------------------------------------------- /playbooks/edx-east/snort.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy snort IDS 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - snort 11 | - role: datadog 12 | when: COMMON_ENABLE_DATADOG 13 | - role: splunkforwarder 14 | when: COMMON_ENABLE_SPLUNKFORWARDER 15 | - role: newrelic 16 | when: COMMON_ENABLE_NEWRELIC 17 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_mms/defaults/main.yml: -------------------------------------------------------------------------------- 1 | 2 | base_url: "https://mms.mongodb.com/download/agent/monitoring" 3 | pkg_arch: "amd64" 4 | pkg_format: "deb" 5 | 6 | agents: 7 | - agent: mongodb-mms-monitoring-agent 8 | version: "5.1.0.323-1" 9 | config: "/etc/mongodb-mms/monitoring-agent.config" 10 | - agent: mongodb-mms-backup-agent 11 | version: "4.4.1.402-1" 12 | config: "/etc/mongodb-mms/backup-agent.config" 13 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/etc/logrotate.d/edx_logrotate_nginx_error.j2: -------------------------------------------------------------------------------- 1 | # Put in place by ansible 2 | 3 | {{ nginx_log_dir }}/*error.log { 4 | create 0640 www-data adm 5 | compress 6 | delaycompress 7 | dateext 8 | missingok 9 | notifempty 10 | daily 11 | rotate 90 12 | size 1M 13 | sharedscripts 14 | postrotate 15 | [ ! -f /var/run/nginx.pid ] || kill -HUP `cat /var/run/nginx.pid` 16 | endscript 17 | } 18 | -------------------------------------------------------------------------------- /util/parsefiles_config.yml: -------------------------------------------------------------------------------- 1 | roles_paths: 2 | - playbooks/roles 3 | aws_plays_paths: 4 | - playbooks 5 | - playbooks/edx-east 6 | docker_plays_paths: 7 | - docker/plays 8 | weights: 9 | - discovery: 6 10 | - go-agent: 3 11 | - xqwatcher: 3 12 | - analytics_api: 1 13 | - edxapp: 28 14 | - insights: 4 15 | - credentials: 8 16 | - forum: 7 17 | - nginx: 1 18 | - xqueue: 2 19 | - trusty-common: 5 20 | - precise-common: 4 -------------------------------------------------------------------------------- /playbooks/edx-east/testcourses.yml: -------------------------------------------------------------------------------- 1 | - name: Create courses for testing 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - testcourses 10 | - role: datadog 11 | when: COMMON_ENABLE_DATADOG 12 | - role: splunkforwarder 13 | when: COMMON_ENABLE_SPLUNKFORWARDER 14 | - role: newrelic 15 | when: COMMON_ENABLE_NEWRELIC 16 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/etc/logrotate.d/edx_logrotate_nginx_access.j2: -------------------------------------------------------------------------------- 1 | # Put in place by ansible 2 | 3 | {{ nginx_log_dir }}/*access.log { 4 | create 0640 www-data adm 5 | compress 6 | delaycompress 7 | dateext 8 | missingok 9 | notifempty 10 | daily 11 | rotate 90 12 | size 1M 13 | sharedscripts 14 | postrotate 15 | [ ! -f /var/run/nginx.pid ] || kill -HUP `cat /var/run/nginx.pid` 16 | endscript 17 | } 18 | 19 | -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-xqwatcher.j2: -------------------------------------------------------------------------------- 1 | {{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python 2 | {{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill 3 | {{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill 4 | -------------------------------------------------------------------------------- /docker/build/rabbitmq/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | USER root 5 | ADD . /edx/app/edx_ansible/edx_ansible 6 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 7 | 8 | # Role is currently untagged 9 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook rabbitmq.yml -c local \ 10 | -i '127.0.0.1,' 11 | 12 | USER rabbitmq 13 | # TBD what we want to run rabbit under 14 | EXPOSE 15672 5672 15 | -------------------------------------------------------------------------------- /playbooks/edx-east/create_cname.yml: -------------------------------------------------------------------------------- 1 | # Creates a cname for a sandbox ec2 instance 2 | - name: Creates a CNAME 3 | hosts: all 4 | gather_facts: False 5 | tasks: 6 | - name: Add DNS name 7 | route53: 8 | overwrite: yes 9 | command: create 10 | zone: "{{ dns_zone }}" 11 | type: CNAME 12 | ttl: 300 13 | record: "{{ dns_name }}.{{ dns_zone }}" 14 | value: "{{ sandbox }}" 15 | 16 | 17 | -------------------------------------------------------------------------------- /playbooks/roles/flower/templates/edx/app/supervisor/conf.d.available/flower.conf.j2: -------------------------------------------------------------------------------- 1 | [program:{{ FLOWER_USER }}] 2 | 3 | environment=PATH="{{ flower_deploy_path }}" 4 | user={{ common_web_user }} 5 | command={{ flower_venv_bin }}/celery flower --broker {{ flower_broker }} --conf={{ flower_conf_dir }}/flowerconfig.py 6 | stdout_logfile={{ supervisor_log_dir }}/{{ FLOWER_USER }}-stdout.log 7 | stderr_logfile={{ supervisor_log_dir }}/{{ FLOWER_USER }}-stderr.log 8 | -------------------------------------------------------------------------------- /playbooks/roles/server_utils/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role insights 12 | # 13 | 14 | 15 | -------------------------------------------------------------------------------- /playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/outputs.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | {% for server in SPLUNKFORWARDER_SERVERS|selectattr("default", "defined") %} 4 | [tcpout] 5 | defaultGroup = {{ server.target_group }} 6 | {% endfor %} 7 | 8 | # forwarder receivers 9 | {% for server in SPLUNKFORWARDER_SERVERS %} 10 | [tcpout:{{ server.target_group }}] 11 | server = {{ server.server }} 12 | 13 | {% endfor %} -------------------------------------------------------------------------------- /vagrant/base/cluster/ansible.cfg: -------------------------------------------------------------------------------- 1 | # config file for ansible -- http://ansible.github.com 2 | # nearly all parameters can be overridden in ansible-playbook or with command line flags 3 | # ansible will read ~/.ansible.cfg or /etc/ansible/ansible.cfg, whichever it finds first 4 | 5 | [defaults] 6 | 7 | jinja2_extensions=jinja2.ext.do 8 | host_key_checking = False 9 | roles_path=../../ansible-roles/roles:../../ansible-private/roles:../../ansible-roles/ 10 | -------------------------------------------------------------------------------- /playbooks/roles/local_dev/files/ftplugin-python.vim: -------------------------------------------------------------------------------- 1 | " Python specific syntax handling 2 | 3 | " indent according to pep-8 rules (4 char, all spaces) 4 | setlocal tabstop=8 5 | setlocal expandtab 6 | setlocal shiftwidth=4 7 | setlocal softtabstop=4 8 | setlocal smarttab 9 | setlocal smartindent 10 | setlocal cinwords=if,elif,else,for,while,with,try,except,finally,def,class 11 | 12 | " Don't auto-align block comments to column 1 13 | inoremap # X# 14 | -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/sites-available/basic-auth.j2: -------------------------------------------------------------------------------- 1 | satisfy any; 2 | 3 | allow 127.0.0.1; 4 | 5 | {% for cidr in COMMON_BASIC_AUTH_EXCEPTIONS %} 6 | allow {{ cidr }}; 7 | {% endfor %} 8 | 9 | deny all; 10 | 11 | auth_basic "Restricted"; 12 | auth_basic_user_file {{ nginx_htpasswd_file }}; 13 | 14 | index index.html 15 | proxy_set_header X-Forwarded-Proto https; 16 | -------------------------------------------------------------------------------- /playbooks/roles/supervisor/templates/etc/init/supervisor-upstart.conf.j2: -------------------------------------------------------------------------------- 1 | description "supervisord" 2 | 3 | {% if disable_edx_services and not devstack -%} 4 | start on stopped pre_supervisor 5 | {% else %} 6 | start on runlevel [2345] 7 | {% endif %} 8 | stop on runlevel [!2345] 9 | 10 | kill timeout 432000 11 | 12 | setuid {{ supervisor_service_user }} 13 | exec {{ supervisor_venv_dir }}/bin/supervisord -n --configuration {{ supervisor_cfg }} 14 | -------------------------------------------------------------------------------- /playbooks/roles/antivirus/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role antivirus 12 | # 13 | dependencies: 14 | - common -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-migrate-cms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | if [[ -z "$NO_EDXAPP_SUDO" ]]; then 4 | SUDO='sudo -E -u {{ edxapp_user }} env "PATH=$PATH"' 5 | fi 6 | 7 | {% for db in cms_auth_config.DATABASES.keys() %} 8 | {%- if db != 'read_replica' %} 9 | $SUDO {{ edxapp_venv_bin}}/python manage.py cms migrate --database {{ db }} --noinput --settings $EDX_PLATFORM_SETTINGS $@ 10 | {% endif %} 11 | {% endfor %} 12 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/templates/edx/bin/edxapp-migrate-lms.j2: -------------------------------------------------------------------------------- 1 | {% include "edxapp_common.j2" %} 2 | 3 | if [[ -z "$NO_EDXAPP_SUDO" ]]; then 4 | SUDO='sudo -E -u {{ edxapp_user }} env "PATH=$PATH"' 5 | fi 6 | 7 | {% for db in lms_auth_config.DATABASES.keys() %} 8 | {%- if db != 'read_replica' %} 9 | $SUDO {{ edxapp_venv_bin}}/python manage.py lms migrate --database {{ db }} --noinput --settings $EDX_PLATFORM_SETTINGS $@ 10 | {% endif %} 11 | {% endfor %} 12 | -------------------------------------------------------------------------------- /playbooks/roles/hive/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role hive 12 | 13 | dependencies: 14 | - hadoop_common 15 | -------------------------------------------------------------------------------- /playbooks/roles/insights/templates/edx/app/supervisor/conf.d.available/insights.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | [program:{{ insights_service_name }}] 4 | 5 | command={{ insights_app_dir }}/insights.sh 6 | user={{ common_web_user }} 7 | directory={{ insights_code_dir }} 8 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 9 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 10 | killasgroup=true 11 | stopasgroup=true 12 | -------------------------------------------------------------------------------- /playbooks/roles/testcourses/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role testcourses 12 | # 13 | TESTCOURSES_EXPORTS: [] 14 | -------------------------------------------------------------------------------- /playbooks/roles/vhost/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role vhost 12 | # 13 | dependencies: 14 | - common 15 | -------------------------------------------------------------------------------- /docker/build/credentials/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | credentials_gunicorn_host: 0.0.0.0 3 | CREDENTIALS_MYSQL: 'db' 4 | CREDENTIALS_DJANGO_SETTINGS_MODULE: 'credentials.settings.devstack' 5 | CREDENTIALS_GUNICORN_EXTRA: '--reload' 6 | CREDENTIALS_MYSQL_MATCHER: '%' 7 | CREDENTIALS_MYSQL_HOST: 'db' 8 | CREDENTIALS_MYSQL_PASSWORD: 'password' 9 | COMMON_MYSQL_MIGRATE_USER: '{{ CREDENTIALS_MYSQL_USER }}' 10 | COMMON_MYSQL_MIGRATE_PASS: '{{ CREDENTIALS_MYSQL_PASSWORD }}' 11 | -------------------------------------------------------------------------------- /playbooks/edx-east/oauth_client_setup.yml: -------------------------------------------------------------------------------- 1 | - name: Configure OAuth2 clients 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars_files: 6 | - "roles/common_vars/defaults/main.yml" 7 | - "roles/edxapp/defaults/main.yml" 8 | - "roles/insights/defaults/main.yml" 9 | - "roles/ecommerce/defaults/main.yml" 10 | - "roles/programs/defaults/main.yml" 11 | - "roles/credentials/defaults/main.yml" 12 | roles: 13 | - oauth_client_setup 14 | -------------------------------------------------------------------------------- /playbooks/edx-east/splunk.yml: -------------------------------------------------------------------------------- 1 | # Usage: ansible-playbook splunk.yml -e@/path/to/environment-deployment.yml 2 | 3 | - name: Deploy Splunk 4 | hosts: all 5 | become: True 6 | gather_facts: True 7 | vars: 8 | roles: 9 | - aws 10 | - splunk-server 11 | - role: datadog 12 | when: COMMON_ENABLE_DATADOG 13 | - role: splunkforwarder 14 | when: COMMON_ENABLE_SPLUNKFORWARDER 15 | - role: newrelic 16 | when: COMMON_ENABLE_NEWRELIC 17 | -------------------------------------------------------------------------------- /playbooks/edx-east/worker.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy worker 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | roles: 6 | - aws 7 | - role: edxapp 8 | celery_worker: True 9 | - role: datadog 10 | when: COMMON_ENABLE_DATADOG 11 | - role: splunkforwarder 12 | when: COMMON_ENABLE_SPLUNKFORWARDER 13 | - role: newrelic 14 | when: COMMON_ENABLE_NEWRELIC 15 | - role: minos 16 | when: COMMON_ENABLE_MINOS 17 | 18 | -------------------------------------------------------------------------------- /playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/alert_actions.conf.j2: -------------------------------------------------------------------------------- 1 | [email] 2 | mailserver = {{ SPLUNK_SMTP_SERVER }} 3 | pdf.header_left = none 4 | pdf.header_right = none 5 | auth_password = {{ SPLUNK_SMTP_PASSWORD }} 6 | auth_username = {{ SPLUNK_SMTP_USERNAME }} 7 | footer.text = {{ SPLUNK_EMAIL_FOOTER }} 8 | hostname = {{ SPLUNK_HOSTNAME }} 9 | from = {{ SPLUNK_FROM_ADDRESS }} 10 | pdf.footer_enabled = 0 11 | pdf.header_enabled = 0 12 | use_tls = 1 -------------------------------------------------------------------------------- /playbooks/roles/xsy/tasks/tag_ec2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: get instance information 4 | action: ec2_facts 5 | tags: 6 | - deploy 7 | 8 | - name: Tag instance 9 | ec2_tag: 10 | resource: "{{ ansible_ec2_instance_id }}" 11 | region: "{{ ansible_ec2_placement_region }}" 12 | tags: 13 | "version:xsy" : "{{ xsy_source_repo }} {{ xsy_checkout.after |truncate(7,True,'')}}" 14 | when: xsy_checkout.after is defined 15 | tags: 16 | - deploy 17 | -------------------------------------------------------------------------------- /docker/build/jenkins_analytics/Dockerfile.noci: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | USER root 5 | RUN apt-get update 6 | 7 | ADD . /edx/app/edx_ansible/edx_ansible 8 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 9 | COPY docker/build/jenkins_analytics/ansible_overrides.yml / 10 | RUN PYTHONUNBUFFERED=1 /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook -v jenkins_analytics.yml -i '127.0.0.1,' -c local -e@/ansible_overrides.yml 11 | -------------------------------------------------------------------------------- /playbooks/roles/alton/tasks/tag_ec2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: get instance information 4 | action: ec2_facts 5 | tags: 6 | - deploy 7 | 8 | - name: tag instance 9 | ec2_tag: 10 | resource: "{{ ansible_ec2_instance_id }}" 11 | region: "{{ ansible_ec2_placement_region }}" 12 | tags: 13 | "version:alton" : "{{ alton_source_repo }} {{ alton_checkout.after |truncate(7,True,'')}}" 14 | when: alton_checkout.after is defined 15 | tags: 16 | - deploy 17 | -------------------------------------------------------------------------------- /playbooks/roles/antivirus/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role antivirus 13 | # 14 | # Overview: 15 | # 16 | # 17 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role hadoop_common 12 | 13 | dependencies: 14 | - oraclejdk -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/tasks/test_sitespeed_worker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Tests for this role 3 | 4 | 5 | ### Tests ### 6 | 7 | # Sitespeed workers should have the latest version of firefox 8 | # Lite test. Ensures we are not using 9 | # the version of firefox specified in a different file. 10 | - name: Verify firefox version 11 | shell: firefox --version 12 | register: firefox_version 13 | - assert: 14 | that: 15 | - "'28.0' not in firefox_version.stdout" 16 | -------------------------------------------------------------------------------- /playbooks/roles/edx_ansible/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role edx_ansible 12 | 13 | dependencies: 14 | - common_vars 15 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_master/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role hadoop_master 12 | 13 | dependencies: 14 | - hadoop_common 15 | -------------------------------------------------------------------------------- /playbooks/roles/gitreload/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role gitreload 12 | # 13 | dependencies: 14 | - common 15 | - supervisor 16 | -------------------------------------------------------------------------------- /playbooks/roles/programs/templates/edx/app/supervisor/conf.d.available/programs.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ programs_service_name }}] 5 | 6 | command={{ programs_home }}/{{ programs_service_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ programs_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/edx-east/deploy_nginx_all_roles.yml: -------------------------------------------------------------------------------- 1 | - name: Configure instance(s) 2 | hosts: all 3 | become: True 4 | gather_facts: False 5 | vars_files: 6 | - roles/edxapp/defaults/main.yml 7 | - roles/xqueue/defaults/main.yml 8 | - roles/xserver/defaults/main.yml 9 | roles: 10 | - common 11 | - aws 12 | - role: nginx 13 | nginx_sites: 14 | - cms 15 | - lms 16 | - xqueue 17 | - xserver 18 | nginx_default_sites: 19 | - lms 20 | -------------------------------------------------------------------------------- /playbooks/roles/discovery/templates/edx/app/supervisor/conf.d.available/discovery.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ discovery_service_name }}] 5 | 6 | command={{ discovery_home }}/{{ discovery_service_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ discovery_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/roles/ecommerce/templates/edx/app/supervisor/conf.d.available/ecommerce.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ ecommerce_service_name }}] 5 | 6 | command={{ ecommerce_home }}/{{ ecommerce_service_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ ecommerce_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/roles/harstorage/templates/edx/app/supervisor/conf.d.available/harstorage.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ harstorage_role_name }}] 5 | 6 | command={{ harstorage_home }}/{{ harstorage_role_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ harstorage_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/edx-east/ecomworker.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Ecommerce Worker 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | roles: 10 | - aws 11 | - ecomworker 12 | - role: datadog 13 | when: COMMON_ENABLE_DATADOG 14 | - role: splunkforwarder 15 | when: COMMON_ENABLE_SPLUNKFORWARDER 16 | - role: newrelic 17 | when: COMMON_ENABLE_NEWRELIC 18 | -------------------------------------------------------------------------------- /playbooks/roles/openstack/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://github.com/edx/configuration/wiki 7 | # code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role openstack 12 | # 13 | dependencies: 14 | - role: vhost 15 | VHOST_NAME: "{{ vhost_name }}" 16 | -------------------------------------------------------------------------------- /playbooks/edx-east/xserver.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy xserver 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - aws 10 | - role: nginx 11 | nginx_sites: 12 | - xserver 13 | - xserver 14 | - role: datadog 15 | when: COMMON_ENABLE_DATADOG 16 | - role: splunkforwarder 17 | when: COMMON_ENABLE_SPLUNKFORWARDER 18 | - role: newrelic 19 | when: COMMON_ENABLE_NEWRELIC 20 | -------------------------------------------------------------------------------- /playbooks/roles/aws/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role aws 12 | # 13 | dependencies: 14 | - role: vhost 15 | VHOST_NAME: "{{ vhost_name }}" 16 | -------------------------------------------------------------------------------- /docker/build/elasticsearch/xDockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | WORKDIR /edx/app/edx_ansible 5 | WORKDIR /edx/app/edx_ansible/edx_ansible/playbooks/edx-east 6 | RUN sudo git checkout e0d/docker-latest 7 | RUN sudo git reset --hard origin/e0d/docker-latest 8 | RUN sudo git pull 9 | RUN sudo ansible-playbook elasticsearch-docker.yml -c local 10 | USER root 11 | WORKDIR /etc/elasticsearch 12 | CMD ["/usr/share/elasticsearch/bin/elasticsearch","-f"] 13 | EXPOSE 9200 9300 14 | -------------------------------------------------------------------------------- /playbooks/roles/analytics_api/templates/edx/app/supervisor/conf.d.available/analytics_api.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | [program:{{ analytics_api_service_name }}] 4 | 5 | command={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh 6 | user={{ common_web_user }} 7 | directory={{ analytics_api_code_dir }} 8 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 9 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 10 | killasgroup=true 11 | stopasgroup=true 12 | -------------------------------------------------------------------------------- /playbooks/roles/credentials/templates/edx/app/supervisor/conf.d.available/credentials.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ credentials_service_name }}] 5 | 6 | command={{ credentials_home }}/{{ credentials_service_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ credentials_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/roles/newrelic-npi/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role newrelic-npi 12 | # 13 | # Example: 14 | # 15 | dependencies: 16 | - role: newrelic 17 | -------------------------------------------------------------------------------- /playbooks/roles/programs/templates/edx/app/programs/programs_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | {{ ansible_managed }} 4 | """ 5 | 6 | timeout = {{ programs_gunicorn_timeout }} 7 | bind = "{{ programs_gunicorn_host }}:{{ programs_gunicorn_port }}" 8 | pythonpath = "{{ programs_code_dir }}" 9 | workers = {{ PROGRAMS_GUNICORN_WORKERS }} 10 | worker_class = "{{ PROGRAMS_GUNICORN_WORKER_CLASS }}" 11 | 12 | {{ PROGRAMS_GUNICORN_EXTRA_CONF }} 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | .idea/ 4 | .vagrant 5 | \#*\# 6 | *~ 7 | .#* 8 | 9 | vagrant/*/*/edx-platform 10 | vagrant/*/*/cs_comments_service 11 | vagrant/*/*/ora 12 | vagrant/*/*/analytics_api 13 | vagrant/*/*/insights 14 | vagrant/*/*/ecommerce 15 | vagrant/*/*/programs 16 | vagrant_ansible_inventory_default 17 | 18 | ### OS X artifacts 19 | *.DS_Store 20 | .AppleDouble 21 | :2e_* 22 | :2e# 23 | 24 | ## Make artifacts 25 | .build 26 | playbooks/edx-east/travis-test.yml 27 | 28 | ## Local virtualenv 29 | /venv 30 | -------------------------------------------------------------------------------- /playbooks/ansible.cfg: -------------------------------------------------------------------------------- 1 | # config file for ansible -- http://ansible.github.com 2 | # nearly all parameters can be overridden in ansible-playbook or with command line flags 3 | # ansible will read ~/.ansible.cfg or /etc/ansible/ansible.cfg, whichever it finds first 4 | 5 | [defaults] 6 | 7 | jinja2_extensions=jinja2.ext.do 8 | host_key_checking = False 9 | roles_path=../../ansible-roles/roles:../../ansible-private/roles:../../ansible-roles/ 10 | ansible_managed=This file is created and updated by ansible, edit at your peril -------------------------------------------------------------------------------- /playbooks/roles/add_user/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role add_user 12 | # 13 | 14 | # Allow this role to be duplicated in dependencies 15 | allow_duplicates: yes 16 | -------------------------------------------------------------------------------- /playbooks/roles/discovery/templates/edx/app/discovery/discovery_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | {{ ansible_managed }} 4 | """ 5 | 6 | timeout = {{ discovery_gunicorn_timeout }} 7 | bind = "{{ discovery_gunicorn_host }}:{{ discovery_gunicorn_port }}" 8 | pythonpath = "{{ discovery_code_dir }}" 9 | workers = {{ DISCOVERY_GUNICORN_WORKERS }} 10 | worker_class = "{{ DISCOVERY_GUNICORN_WORKER_CLASS }}" 11 | 12 | {{ DISCOVERY_GUNICORN_EXTRA_CONF }} 13 | -------------------------------------------------------------------------------- /playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | {{ ansible_managed }} 4 | """ 5 | 6 | timeout = {{ ecommerce_gunicorn_timeout }} 7 | bind = "{{ ecommerce_gunicorn_host }}:{{ ecommerce_gunicorn_port }}" 8 | pythonpath = "{{ ecommerce_code_dir }}" 9 | workers = {{ ECOMMERCE_GUNICORN_WORKERS }} 10 | worker_class = "{{ ECOMMERCE_GUNICORN_WORKER_CLASS }}" 11 | 12 | {{ ECOMMERCE_GUNICORN_EXTRA_CONF }} 13 | -------------------------------------------------------------------------------- /playbooks/roles/haproxy/templates/haproxy.cfg.j2: -------------------------------------------------------------------------------- 1 | # this config needs haproxy-1.1.28 or haproxy-1.2.1 2 | 3 | global 4 | log /dev/log local0 info 5 | log /dev/log local0 notice 6 | #log loghost local0 info 7 | maxconn 4096 8 | #chroot /usr/share/haproxy 9 | user haproxy 10 | group haproxy 11 | daemon 12 | #debug 13 | #quiet 14 | {{ haproxy_extra_global_config }} 15 | 16 | defaults 17 | {{ haproxy_default_config }} 18 | 19 | {%- for app in haproxy_applications -%} 20 | {{ app }} 21 | 22 | {%- endfor -%} 23 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - jscover 5 | - role: oraclejdk 6 | oraclejdk_version: "8u60" 7 | oraclejdk_base: "jdk1.8.0_60" 8 | oraclejdk_build: "b27" 9 | oraclejdk_link: "/usr/lib/jvm/java-8-oracle" 10 | 11 | # dependencies for edx-app jenkins worker: 12 | - role: edxapp_common 13 | when: platform_worker is defined 14 | 15 | # dependencies for sitespeed worker 16 | - role: sitespeedio 17 | when: sitespeed_worker is defined 18 | -------------------------------------------------------------------------------- /playbooks/roles/ecomworker/templates/edx/app/supervisor/conf.d.available/ecomworker.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | [program:{{ ecommerce_worker_service_name }}] 5 | 6 | command={{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}.sh 7 | user={{ common_web_user }} 8 | directory={{ ecommerce_worker_code_dir }} 9 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/roles/edx_notes_api/templates/edx/app/supervisor/conf.d.available/edx_notes_api.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ ansible_managed }} 3 | # 4 | 5 | [program:{{ edx_notes_api_service_name }}] 6 | 7 | command={{ edx_notes_api_home }}/{{ edx_notes_api_service_name }}.sh 8 | user={{ common_web_user }} 9 | directory={{ edx_notes_api_code_dir }} 10 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log 11 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log 12 | killasgroup=true 13 | stopasgroup=true 14 | -------------------------------------------------------------------------------- /playbooks/roles/git_clone/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role git_clone 12 | # 13 | 14 | # Allow this role to be duplicated in dependencies 15 | allow_duplicates: yes 16 | -------------------------------------------------------------------------------- /playbooks/edx-east/forum.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy forum 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | CLUSTER_NAME: 'forum' 8 | serial: "{{ serial_count }}" 9 | roles: 10 | - aws 11 | - role: nginx 12 | nginx_sites: 13 | - forum 14 | - forum 15 | - role: datadog 16 | when: COMMON_ENABLE_DATADOG 17 | - role: splunkforwarder 18 | when: COMMON_ENABLE_SPLUNKFORWARDER 19 | - role: newrelic 20 | when: COMMON_ENABLE_NEWRELIC 21 | -------------------------------------------------------------------------------- /playbooks/roles/forum/tasks/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: test that the required service are listening 3 | wait_for: 4 | port: "{{ item.port }}" 5 | host: "{{ item.host }}" 6 | timeout: 30 7 | with_items: "{{ forum_services }}" 8 | when: not disable_edx_services 9 | 10 | - name: test that mongo replica set members are listing 11 | wait_for: 12 | port: "{{ FORUM_MONGO_PORT }}" 13 | host: "{{ item }}" 14 | timeout: 30 15 | with_items: "{{ FORUM_MONGO_HOSTS }}" 16 | when: not disable_edx_services 17 | -------------------------------------------------------------------------------- /playbooks/roles/elasticsearch-monitor/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role elasticsearch-monitor 12 | # 13 | # Example: 14 | # 15 | dependencies: 16 | - role: newrelic-npi 17 | -------------------------------------------------------------------------------- /playbooks/edx-east/ansible.cfg: -------------------------------------------------------------------------------- 1 | # config file for ansible -- http://ansible.github.com 2 | # nearly all parameters can be overridden in ansible-playbook or with command line flags 3 | # ansible will read ~/.ansible.cfg or /etc/ansible/ansible.cfg, whichever it finds first 4 | 5 | [defaults] 6 | 7 | jinja2_extensions=jinja2.ext.do 8 | host_key_checking=False 9 | roles_path=../../../ansible-roles/roles:../../../ansible-private/roles:../../../ansible-roles/ 10 | ansible_managed=This file is created and updated by ansible, edit at your peril 11 | -------------------------------------------------------------------------------- /playbooks/roles/analytics_pipeline/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role analytics_pipeline 12 | 13 | dependencies: 14 | - common 15 | - hadoop_master 16 | - hive 17 | - sqoop 18 | -------------------------------------------------------------------------------- /playbooks/roles/credentials/templates/edx/app/credentials/credentials_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | {{ ansible_managed }} 4 | """ 5 | 6 | timeout = {{ credentials_gunicorn_timeout }} 7 | bind = "{{ credentials_gunicorn_host }}:{{ credentials_gunicorn_port }}" 8 | pythonpath = "{{ credentials_code_dir }}" 9 | workers = {{ CREDENTIALS_GUNICORN_WORKERS }} 10 | worker_class = "{{ CREDENTIALS_GUNICORN_WORKER_CLASS }}" 11 | 12 | {{ CREDENTIALS_GUNICORN_EXTRA_CONF }} 13 | -------------------------------------------------------------------------------- /playbooks/roles/harprofiler/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 6 | 7 | harprofiler_role_name: harprofiler 8 | 9 | harprofiler_user: "harprofiler" 10 | harprofiler_github_url: https://github.com/edx/harprofiler 11 | harprofiler_version: master 12 | harprofiler_dir: /edx/app/harprofiler 13 | harprofiler_venv_dir: "{{ harprofiler_dir }}/venvs/harprofiler" 14 | harprofiler_validation_script: validate_harprofiler_install.sh 15 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/mapred-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | mapreduce.framework.name 7 | yarn 8 | 9 | 10 | {% if mapred_site_config is defined %} 11 | {% for key,value in mapred_site_config.iteritems() %} 12 | 13 | {{ key }}} 14 | {{ value }} 15 | 16 | {% endfor %} 17 | {% endif %} 18 | 19 | -------------------------------------------------------------------------------- /playbooks/roles/mysql/tasks/remove_mysql_experimental.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Stop mysql service 3 | service: 4 | name: mysql 5 | state: stopped 6 | 7 | - name: Remove experimental apt repository 8 | apt_repository: 9 | repo: 'deb http://ppa.launchpad.net/ondrej/mysql-experimental/ubuntu precise main' 10 | state: absent 11 | 12 | - name: Remove experimental version of mysql 13 | apt: 14 | name: "{{ item }}" 15 | state: absent 16 | purge: yes 17 | with_items: 18 | - mysql-server-5.6 19 | - mysql-server -------------------------------------------------------------------------------- /vagrant/base/test_playbook/ansible.cfg: -------------------------------------------------------------------------------- 1 | # config file for ansible -- http://ansible.github.com 2 | # nearly all parameters can be overridden in ansible-playbook or with command line flags 3 | # ansible will read ~/.ansible.cfg or /etc/ansible/ansible.cfg, whichever it finds first 4 | 5 | [defaults] 6 | 7 | jinja2_extensions=jinja2.ext.do 8 | host_key_checking = False 9 | roles_path=../../ansible-roles/roles:../../ansible-private/roles:../../ansible-roles/ 10 | ansible_managed=This file is created and updated by ansible, edit at your peril 11 | -------------------------------------------------------------------------------- /playbooks/vagrant/vagrant-devstack-delta.yml: -------------------------------------------------------------------------------- 1 | - name: Update devstack to a specific intermediate revision 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | devstack: true 7 | disable_edx_services: true 8 | mongo_enable_journal: false 9 | COMMON_MOTD_TEMPLATE: 'devstack_motd.tail.j2' 10 | COMMON_SSH_PASSWORD_AUTH: "yes" 11 | EDXAPP_LMS_BASE: 127.0.0.1:8000 12 | EDXAPP_OAUTH_ENFORCE_SECURE: false 13 | EDXAPP_LMS_BASE_SCHEME: http 14 | roles: 15 | - common 16 | - vhost 17 | - edxapp 18 | -------------------------------------------------------------------------------- /playbooks/vagrant/vagrant-fullstack-delta.yml: -------------------------------------------------------------------------------- 1 | - name: Update fullstack to a specific intermediate revision 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | disable_edx_services: true 7 | mongo_enable_journal: false 8 | COMMON_MOTD_TEMPLATE: 'devstack_motd.tail.j2' 9 | COMMON_SSH_PASSWORD_AUTH: "yes" 10 | EDXAPP_LMS_BASE: 127.0.0.1:8000 11 | EDXAPP_OAUTH_ENFORCE_SECURE: false 12 | EDXAPP_LMS_BASE_SCHEME: http 13 | roles: 14 | - common 15 | - vhost 16 | - edxapp 17 | - xqueue 18 | -------------------------------------------------------------------------------- /playbooks/roles/xqwatcher/tasks/deploy_courses.yml: -------------------------------------------------------------------------------- 1 | # Iterates over the data structure documented in tasks/main.yml 2 | # checking out the grader code from the repository specified on 3 | # a per queue basis. 4 | 5 | - name: Checkout grader code 6 | git_2_0_1: 7 | repo: "{{ item.GIT_REPO }}" 8 | dest: "{{ xqwatcher_app_dir }}/data/{{ item.COURSE }}" 9 | version: "{{ item.GIT_REF }}" 10 | ssh_opts: "{{ xqwatcher_course_git_ssh_opts }}" 11 | with_items: "{{ XQWATCHER_COURSES }}" 12 | tags: 13 | - install 14 | - install:code 15 | -------------------------------------------------------------------------------- /playbooks/edx-east/analytics_single.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy all analytics services to a single node 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | migrate_db: "yes" 7 | disable_edx_services: false 8 | ENABLE_DATADOG: False 9 | ENABLE_SPLUNKFORWARDER: False 10 | ENABLE_NEWRELIC: False 11 | roles: 12 | - aws 13 | - mysql 14 | - edxlocal 15 | - memcache 16 | - analytics_api 17 | - analytics_pipeline 18 | - role: nginx 19 | nginx_sites: 20 | - insights 21 | - insights 22 | -------------------------------------------------------------------------------- /playbooks/roles/gitreload/templates/edx/app/gitreload/gitreload_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | 4 | {{ ansible_managed }} 5 | """ 6 | import multiprocessing 7 | 8 | preload_app = True 9 | timeout = 10 10 | bind = "{{ gitreload_gunicorn_host }}:{{ gitreload_gunicorn_port }}" 11 | 12 | workers = {{ gitreload_gunicorn_workers }} 13 | 14 | {{ common_close_all_caches }} 15 | 16 | def post_fork(server, worker): 17 | close_all_caches() 18 | 19 | {{ GITRELOAD_GUNICORN_EXTRA_CONF }} 20 | -------------------------------------------------------------------------------- /playbooks/roles/notifier/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart notifier-scheduler 3 | supervisorctl: 4 | name: "notifier-scheduler" 5 | state: restarted 6 | config: "{{ supervisor_cfg }}" 7 | supervisorctl_path: "{{ supervisor_ctl }}" 8 | when: not disable_edx_services 9 | 10 | - name: restart notifier-celery-workers 11 | supervisorctl: 12 | name: "notifier-celery-workers" 13 | state: restarted 14 | config: "{{ supervisor_cfg }}" 15 | supervisorctl_path: "{{ supervisor_ctl }}" 16 | when: not disable_edx_services 17 | -------------------------------------------------------------------------------- /playbooks/roles/redis/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role redis 13 | # 14 | # Overview: 15 | # 16 | # 17 | - name: reload redis 18 | service: 19 | name: redis-server 20 | state: restarted 21 | -------------------------------------------------------------------------------- /util/jenkins/change-enrollment-course.sh: -------------------------------------------------------------------------------- 1 | cd configuration 2 | pip install -r requirements.txt 3 | env 4 | 5 | ansible="ansible first_in_tag_Name_${environment}-${deployment}-worker -i playbooks/ec2.py -u ubuntu -s -U www-data -m shell -a" 6 | manage="cd /edx/app/edxapp/edx-platform && /edx/bin/python.edxapp /edx/bin/manage.edxapp lms change_enrollment --settings aws" 7 | 8 | if [ "$noop" = true ]; then 9 | $ansible "$manage --noop --course $course --to $to --from $from" 10 | else 11 | $ansible "$manage --course $course --to $to --from $from" 12 | fi 13 | -------------------------------------------------------------------------------- /playbooks/edx-east/notes.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Notes API 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: True 9 | roles: 10 | - aws 11 | - role: nginx 12 | nginx_sites: 13 | - edx_notes_api 14 | - edx_notes_api 15 | - role: datadog 16 | when: COMMON_ENABLE_DATADOG 17 | - role: splunkforwarder 18 | when: COMMON_ENABLE_SPLUNKFORWARDER 19 | - role: newrelic 20 | when: COMMON_ENABLE_NEWRELIC 21 | -------------------------------------------------------------------------------- /playbooks/roles/oraclejdk/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | oraclejdk_version: "8u65" 4 | # what the archive unpacks to 5 | oraclejdk_base: "jdk1.8.0_65" 6 | oraclejdk_build: "b17" 7 | oraclejdk_platform: "linux" 8 | oraclejdk_arch: "x64" 9 | oraclejdk_file: "jdk-{{ oraclejdk_version }}-{{ oraclejdk_platform }}-{{ oraclejdk_arch }}.tar.gz" 10 | oraclejdk_url: "http://download.oracle.com/otn-pub/java/jdk/{{ oraclejdk_version }}-{{ oraclejdk_build }}/{{ oraclejdk_file }}" 11 | oraclejdk_link: "/usr/lib/jvm/java-8-oracle" 12 | 13 | oraclejdk_debian_pkgs: 14 | - curl 15 | -------------------------------------------------------------------------------- /playbooks/roles/gitreload/templates/edx/app/supervisor/conf.available.d/gitreload.conf.j2: -------------------------------------------------------------------------------- 1 | [program:gitreload] 2 | 3 | user={{ common_web_user }} 4 | directory={{ gitreload_dir }} 5 | umask=002 6 | 7 | command={{ gitreload_venv }}/bin/gunicorn -c {{ gitreload_dir }}/gitreload_gunicorn.py {{ GITRELOAD_GUNICORN_EXTRA }} gitreload.web:app 8 | 9 | environment=PID=/var/tmp/gitreload.pid 10 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 11 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 12 | killasgroup=true 13 | stopasgroup=true 14 | -------------------------------------------------------------------------------- /util/jenkins/change-enrollment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd configuration 4 | pip install -r requirements.txt 5 | env 6 | 7 | ansible="ansible -v first_in_tag_Name_${environment}-${deployment}-worker -i playbooks/ec2.py -u ubuntu -s -U www-data -m shell -a" 8 | manage="cd /edx/app/edxapp/edx-platform && /edx/bin/python.edxapp /edx/bin/manage.edxapp lms change_enrollment --settings=aws" 9 | noop_switch="" 10 | 11 | if [ "$noop" = true ]; then 12 | noop_switch="--noop" 13 | fi 14 | 15 | $ansible "$manage $noop_switch --course $course --user $name --to $to --from $from" 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/edx/ansible.git@stable-1.9.3-rc1-edx#egg=ansible==1.9.3-edx 2 | PyYAML==3.11 3 | Jinja2==2.8 4 | MarkupSafe==0.23 5 | boto==2.32.1 6 | ecdsa==0.11 7 | paramiko==1.15.1 8 | pycrypto==2.6.1 9 | wsgiref==0.1.2 10 | docopt==0.6.1 11 | python-simple-hipchat==0.2 12 | prettytable==0.7.2 13 | awscli==1.10.28 14 | requests==2.9.1 15 | datadog==0.8.0 16 | networkx==1.11 17 | pathlib2==2.1.0 18 | 19 | # Needed for the mongo_* modules (playbooks/library/mongo_*) 20 | pymongo==3.1 21 | 22 | # Needed for the mysql_db module 23 | MySQL-python==1.2.5 24 | -------------------------------------------------------------------------------- /playbooks/roles/container/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role container 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/hdfs-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | dfs.replication 7 | 1 8 | 9 | 10 | 11 | dfs.namenode.name.dir 12 | file:{{ HADOOP_COMMON_DATA }}/namenode 13 | 14 | 15 | 16 | dfs.datanode.data.dir 17 | file:{{ HADOOP_COMMON_DATA }}/datanode 18 | 19 | 20 | -------------------------------------------------------------------------------- /playbooks/roles/splunk-server/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role splunk-server 12 | dependencies: 13 | - role: user 14 | user_info: "{{ COMMON_USER_INFO }}" 15 | - role: mount_ebs 16 | volumes: "{{ SPLUNK_VOLUMES }}" 17 | -------------------------------------------------------------------------------- /playbooks/edx-east/insights.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Insights 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: True 9 | CLUSTER_NAME: 'insights' 10 | roles: 11 | - aws 12 | - role: nginx 13 | nginx_sites: 14 | - insights 15 | - insights 16 | - role: datadog 17 | when: COMMON_ENABLE_DATADOG 18 | - role: splunkforwarder 19 | when: COMMON_ENABLE_SPLUNKFORWARDER 20 | - role: newrelic 21 | when: COMMON_ENABLE_NEWRELIC 22 | -------------------------------------------------------------------------------- /playbooks/roles/ad_hoc_reporting/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role bastion 12 | # 13 | dependencies: 14 | - common 15 | - role: user 16 | user_info: "{{ AD_HOC_REPORTING_USER_INFO }}" 17 | tags: 18 | - users 19 | 20 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_admin/templates/edx/var/jenkins/hudson.plugins.s3.S3BucketPublisher.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ JENKINS_ADMIN_S3_PROFILE.name }} 6 | {{ JENKINS_ADMIN_S3_PROFILE.access_key }} 7 | {{ JENKINS_ADMIN_S3_PROFILE.secret_key }} 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /docker/build/analytics_api/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | DOCKER_TLD: "edx" 4 | 5 | ANALYTICS_API_DATABASES: 6 | # rw user 7 | default: 8 | ENGINE: 'django.db.backends.mysql' 9 | NAME: '{{ ANALYTICS_API_DEFAULT_DB_NAME }}' 10 | USER: 'api001' 11 | PASSWORD: 'password' 12 | HOST: 'db.{{ DOCKER_TLD }}' 13 | PORT: '3306' 14 | # read-only user 15 | reports: 16 | ENGINE: 'django.db.backends.mysql' 17 | NAME: '{{ ANALYTICS_API_REPORTS_DB_NAME }}' 18 | USER: 'reports001' 19 | PASSWORD: 'password' 20 | HOST: "db.{{ DOCKER_TLD }}" 21 | PORT: '3306' 22 | -------------------------------------------------------------------------------- /playbooks/roles/mongo_3_0/templates/mongo_logrotate.j2: -------------------------------------------------------------------------------- 1 | {{ mongo_log_dir }}/serverStatus.log { 2 | create 3 | compress 4 | copytruncate 5 | delaycompress 6 | dateext 7 | dateformat -%Y%m%d-%s 8 | missingok 9 | notifempty 10 | daily 11 | rotate 90 12 | size 1M 13 | } 14 | 15 | {{ mongo_log_dir }}/mongodb.log { 16 | create 17 | compress 18 | copytruncate 19 | delaycompress 20 | dateext 21 | dateformat -%Y%m%d-%s 22 | missingok 23 | notifempty 24 | daily 25 | rotate 90 26 | size 1M 27 | postrotate 28 | /usr/bin/killall -USR1 mongod 29 | endscript 30 | } 31 | -------------------------------------------------------------------------------- /playbooks/roles/splunkforwarder/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role splunk 13 | # 14 | # Overview: 15 | # 16 | # 17 | 18 | # Restart Splunk 19 | - name: restart splunkforwarder 20 | service: 21 | name: splunk 22 | state: restarted 23 | -------------------------------------------------------------------------------- /docker/plays/nginx.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy nginx 2 | hosts: all 3 | sudo: True 4 | gather_facts: True 5 | vars: 6 | serial_count: 1 7 | serial: "{{ serial_count }}" 8 | roles: 9 | - common_vars 10 | - docker 11 | - role: nginx 12 | nginx_sites: 13 | - lms 14 | - cms 15 | - xqueue 16 | - certs 17 | - forum 18 | nginx_default_sites: 19 | - lms 20 | nginx_extra_sites: "{{ NGINX_EDXAPP_EXTRA_SITES }}" 21 | nginx_extra_configs: "{{ NGINX_EDXAPP_EXTRA_CONFIGS }}" 22 | nginx_redirects: "{{ NGINX_EDXAPP_CUSTOM_REDIRECTS }}" 23 | -------------------------------------------------------------------------------- /playbooks/edx-east/analyticsapi.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy Analytics API 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | CLUSTER_NAME: 'analytics-api' 10 | roles: 11 | - aws 12 | - role: nginx 13 | nginx_sites: 14 | - analytics_api 15 | - analytics_api 16 | - role: datadog 17 | when: COMMON_ENABLE_DATADOG 18 | - role: splunkforwarder 19 | when: COMMON_ENABLE_SPLUNKFORWARDER 20 | - role: newrelic 21 | when: COMMON_ENABLE_NEWRELIC 22 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/supervisor/conf.d.available/ROLE_NAME.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # {{ '{{' }} ansible_managed }} 3 | # 4 | [program:{{ '{{' }} {{ role_name }}_service_name }}] 5 | 6 | command={{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}.sh 7 | user={{ '{{' }} common_web_user }} 8 | directory={{ '{{' }} {{ role_name }}_code_dir }} 9 | stdout_logfile={{ '{{' }} supervisor_log_dir }}/%(program_name)s-stdout.log 10 | stderr_logfile={{ '{{' }} supervisor_log_dir }}/%(program_name)s-stderr.log 11 | killasgroup=true 12 | stopasgroup=true 13 | -------------------------------------------------------------------------------- /playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | 4 | {{ ansible_managed }} 5 | """ 6 | 7 | timeout = {{ analytics_api_gunicorn_timeout }} 8 | bind = "{{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }}" 9 | pythonpath = "{{ analytics_api_code_dir }}" 10 | workers = {{ ANALYTICS_API_GUNICORN_WORKERS }} 11 | 12 | {{ common_close_all_caches }} 13 | 14 | def post_fork(server, worker): 15 | close_all_caches() 16 | 17 | {{ ANALYTICS_API_GUNICORN_EXTRA_CONF }} 18 | -------------------------------------------------------------------------------- /playbooks/roles/mariadb/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role mariadb 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | dependencies: 22 | - common 23 | -------------------------------------------------------------------------------- /playbooks/roles/redis/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role redis 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - common 24 | -------------------------------------------------------------------------------- /util/vpc-tools/sanitize-db-prod_grader.sql: -------------------------------------------------------------------------------- 1 | SET FOREIGN_KEY_CHECKS=0; 2 | 3 | /* 4 | Grader has its own django core tables. 5 | */ 6 | 7 | UPDATE auth_user 8 | set 9 | email = concat('success+',cast(id AS CHAR),'@simulator.amazonses.com'), 10 | username = concat('user-',cast(id AS CHAR)), 11 | first_name = concat('user-',cast(id AS CHAR)), 12 | last_name = concat('user-',cast(id AS CHAR)), 13 | password = null, 14 | last_login = null, 15 | date_joined = null 16 | where email not like ('%@edx.org'); 17 | 18 | SET FOREIGN_KEY_CHECKS=1; 19 | -------------------------------------------------------------------------------- /playbooks/edx-east/discovery.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Course Discovery Service 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | CLUSTER_NAME: 'discovery' 10 | roles: 11 | - aws 12 | - role: nginx 13 | nginx_default_sites: 14 | - discovery 15 | - discovery 16 | - role: datadog 17 | when: COMMON_ENABLE_DATADOG 18 | - role: splunkforwarder 19 | when: COMMON_ENABLE_SPLUNKFORWARDER 20 | - role: newrelic 21 | when: COMMON_ENABLE_NEWRELIC 22 | -------------------------------------------------------------------------------- /playbooks/roles/docker/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role docker 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - container 24 | -------------------------------------------------------------------------------- /playbooks/roles/haproxy/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role haproxy 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - common 24 | -------------------------------------------------------------------------------- /playbooks/roles/security/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Tasks for role security 13 | # 14 | # Overview: 15 | # 16 | # 17 | # Dependencies: 18 | # 19 | # 20 | # Example play: 21 | # 22 | # 23 | 24 | - include: security-ubuntu.yml 25 | when: ansible_distribution == 'Ubuntu' 26 | -------------------------------------------------------------------------------- /playbooks/roles/shibboleth/defaults/main.yml: -------------------------------------------------------------------------------- 1 | shib: 2 | sp_entity_id: 'https://myownshibsp.org/shibboleth' 3 | #defaultng to use testshib.org 4 | idp_entity_id: 'https://idp.testshib.org/idp/shibboleth' 5 | idp_metadata_url: 'http://www.testshib.org/metadata/testshib-providers.xml' 6 | support_email: 'support@myownshibsp.org' 7 | sp_key: | 8 | YOU NEED TO GENERATE A REAL KEY HERE USING OPENSSL 9 | sp_pem: | 10 | THE CORRESPONDING CERTIFICATE PEM GOES HERE 11 | shib_template_dir: '.' 12 | shib_metadata_backup_url: "https://idp.stanford.edu/Stanford-metadata.xml" 13 | shib_download_metadata: true 14 | -------------------------------------------------------------------------------- /docker/build/xqueue/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | USER root 5 | RUN apt-get update 6 | ADD . /edx/app/edx_ansible/edx_ansible 7 | COPY docker/build/xqueue/ansible_overrides.yml / 8 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 9 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook xqueue.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml 10 | 11 | COPY docker/build/xqueue/docker-run.sh / 12 | ENTRYPOINT ["/docker-run.sh"] 13 | EXPOSE 8110 18110 14 | -------------------------------------------------------------------------------- /playbooks/edx-east/tools-gp.yml: -------------------------------------------------------------------------------- 1 | # Creating a new ad hoc reporting box: 2 | # ansible-playbook -i 'reporting.example.com,' ./tools-gp.yml -e@/var/path/common.yml -e@/vars/path/environnment-deployment.yml 3 | # Updating the users on an existing box: 4 | # ansible-playbook -i 'reporting.example.com,' ./tools-gp.yml --tags users -e@/var/path/users.yml -e@/vars/path/environnment-deployment.yml 5 | - name: Deploy Ad Hoc Reporting Scripts 6 | hosts: all 7 | become: True 8 | gather_facts: True 9 | vars: 10 | serial_count: 1 11 | serial: "{{ serial_count }}" 12 | roles: 13 | - aws 14 | - ad_hoc_reporting 15 | -------------------------------------------------------------------------------- /playbooks/edx-east/xqwatcher.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy xqueue-watcher 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | COMMON_APP_DIR: "/edx/app" 7 | common_web_group: "www-data" 8 | ENABLE_DATADOG: False 9 | ENABLE_SPLUNKFORWARDER: False 10 | ENABLE_NEWRELIC: False 11 | serial_count: 1 12 | serial: "{{ serial_count }}" 13 | roles: 14 | - aws 15 | - xqwatcher 16 | - role: datadog 17 | when: COMMON_ENABLE_DATADOG 18 | - role: splunkforwarder 19 | when: COMMON_ENABLE_SPLUNKFORWARDER 20 | - role: newrelic 21 | when: COMMON_ENABLE_NEWRELIC 22 | -------------------------------------------------------------------------------- /playbooks/roles/edx_vpc/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role edx_vpc 12 | # 13 | 14 | # 15 | # vars are namespace with the module name. 16 | # 17 | vpc_role_name: vpc 18 | 19 | # 20 | # OS packages 21 | # 22 | 23 | vpc_debian_pkgs: [] 24 | 25 | vpc_redhat_pkgs: [] 26 | -------------------------------------------------------------------------------- /playbooks/roles/mariadb/templates/etc/mysql/conf.d/galera.cnf.j2: -------------------------------------------------------------------------------- 1 | {%- set hosts= [] -%} 2 | {%- for host in hostvars.keys()|sort -%} 3 | {% do hosts.append(host) %} 4 | {%- endfor %} 5 | [mysqld] 6 | binlog_format=ROW 7 | innodb_autoinc_lock_mode=2 8 | innodb_doublewrite=1 9 | query_cache_size=0 10 | 11 | wsrep_provider=/usr/lib/galera/libgalera_smm.so 12 | wsrep_cluster_address=gcomm://{{ hosts|join(',') }}?pc.wait_prim=no 13 | wsrep_sst_auth={{ MARIADB_CLUSTER_USER_ADMIN }}:{{ MARIADB_CLUSTER_PASSWORD_ADMIN }} 14 | 15 | {% if vagrant_cluster|bool %} 16 | wsrep_node_address={{ ansible_ssh_host }} 17 | {% endif %} 18 | -------------------------------------------------------------------------------- /playbooks/roles/sqoop/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role sqoop 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - hadoop_common 24 | -------------------------------------------------------------------------------- /playbooks/roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Tasks for role docker 13 | # 14 | # Overview: 15 | # 16 | # This is a special role for providing common, docker specific tasks. 17 | # 18 | # Dependencies: 19 | # 20 | # Inherits from the container role via meta/main.yml 21 | # 22 | -------------------------------------------------------------------------------- /playbooks/roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role docker 12 | # 13 | 14 | # 15 | # vars are namespaced with the module name. 16 | # 17 | docker_role_name: docker 18 | 19 | # 20 | # OS packages 21 | # 22 | 23 | docker_debian_pkgs: [] 24 | 25 | docker_redhat_pkgs: [] 26 | -------------------------------------------------------------------------------- /playbooks/roles/xsy/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role xsy 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - common 24 | - supervisor 25 | -------------------------------------------------------------------------------- /playbooks/roles/test_build_server/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role test_build_server 12 | # 13 | 14 | # 15 | # vars are namespace with the module name. 16 | # 17 | test_build_server_user: jenkins 18 | test_build_server_repo_path: /home/jenkins 19 | test_edx_platform_version: master 20 | -------------------------------------------------------------------------------- /playbooks/roles/edx_service/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role edx_service 12 | # 13 | 14 | # 15 | # vars are namespace with the module name. 16 | # 17 | edx_service_name: edx_service 18 | 19 | # 20 | # OS packages 21 | # 22 | edx_service_packages: 23 | debian: [] 24 | redhat: [] 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | GitHub issues are not meant for support questions. Please use the 2 | [mailing list](https://groups.google.com/forum/#!forum/openedx-ops) 3 | or [Slack channels](https://open.edx.org/blog/open-edx-slack) to get help. 4 | 5 | Please open issues here to report bugs in the ansible scripts themselves. 6 | 7 | When reporting an issue, please include the following information. 8 | 9 | If `/edx/bin/show-repo-heads` is available on your system, it can provide much of this information: 10 | 11 | - Configuration ref: 12 | - edx-platform ref: 13 | - other refs: 14 | 15 | #Steps to replicate: 16 | - one 17 | - two 18 | - three 19 | -------------------------------------------------------------------------------- /docker/build/forum/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 5 | ADD . /edx/app/edx_ansible/edx_ansible 6 | COPY docker/build/forum/ansible_overrides.yml / 7 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook forum.yml \ 8 | -i '127.0.0.1,' -c local \ 9 | -t "install:base,install:configuration,install:app-requirements,install:code" \ 10 | -e@/ansible_overrides.yml 11 | WORKDIR /edx/app 12 | CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"] 13 | EXPOSE 4567 14 | -------------------------------------------------------------------------------- /playbooks/edx-east/vpc_admin.yml: -------------------------------------------------------------------------------- 1 | # ansible-playbook -vvv -c ssh -i admin_url, vpc_admin.yml -e "@path_to_common_overrides" -e "@path_to_deployment_specific_overrides" 2 | # Configure an admin instance with jenkins and asgard. 3 | - name: Configure instance(s) 4 | hosts: all 5 | become: True 6 | gather_facts: True 7 | roles: 8 | - aws 9 | - edx_ansible 10 | - user 11 | - jenkins_admin 12 | - hotg 13 | - alton 14 | - role: datadog 15 | when: COMMON_ENABLE_DATADOG 16 | - role: splunkforwarder 17 | when: COMMON_ENABLE_SPLUNKFORWARDER 18 | - role: newrelic 19 | when: COMMON_ENABLE_NEWRELIC 20 | -------------------------------------------------------------------------------- /playbooks/edx-east/ecommerce.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Ecommerce 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | CLUSTER_NAME: 'ecommerce' 10 | roles: 11 | - aws 12 | - role: nginx 13 | nginx_sites: 14 | - ecommerce 15 | nginx_default_sites: 16 | - ecommerce 17 | - ecommerce 18 | - role: datadog 19 | when: COMMON_ENABLE_DATADOG 20 | - role: splunkforwarder 21 | when: COMMON_ENABLE_SPLUNKFORWARDER 22 | - role: newrelic 23 | when: COMMON_ENABLE_NEWRELIC 24 | -------------------------------------------------------------------------------- /playbooks/edx-east/programs.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Programs Service 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | CLUSTER_NAME: 'programs' 10 | roles: 11 | - aws 12 | - role: nginx 13 | nginx_sites: 14 | - programs 15 | nginx_default_sites: 16 | - programs 17 | - programs 18 | - role: datadog 19 | when: COMMON_ENABLE_DATADOG 20 | - role: splunkforwarder 21 | when: COMMON_ENABLE_SPLUNKFORWARDER 22 | - role: newrelic 23 | when: COMMON_ENABLE_NEWRELIC 24 | -------------------------------------------------------------------------------- /playbooks/roles/container/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role container 12 | # 13 | 14 | # 15 | # vars are namespaced with the module name. 16 | # 17 | container_role_name: container 18 | 19 | # 20 | # OS packages 21 | # 22 | 23 | container_debian_pkgs: [] 24 | 25 | container_redhat_pkgs: [] 26 | -------------------------------------------------------------------------------- /playbooks/roles/container/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Tasks for role container 13 | # 14 | # Overview: 15 | # 16 | # This is a special role for providing common, container specific 17 | # tasks and serving as a common ancestor for specific container 18 | # providers, say, docker and rocket. 19 | # 20 | 21 | -------------------------------------------------------------------------------- /playbooks/roles/edxapp/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - common 4 | - supervisor 5 | - edxapp_common 6 | - nltk 7 | - role: user 8 | user_info: 9 | - name: "{{ EDXAPP_AUTOMATOR_NAME }}" 10 | sudo_cmds: "{{ EDXAPP_AUTOMATOR_SUDO_CMDS }}" 11 | type: restricted 12 | authorized_keys: "{{ EDXAPP_AUTOMATOR_AUTHORIZED_KEYS }}" 13 | when: EDXAPP_AUTOMATOR_AUTHORIZED_KEYS|length != 0 14 | - role: edx_themes 15 | theme_users: 16 | - "{{ edxapp_user }}" 17 | additional_theme_dirs: 18 | - "{{ EDXAPP_COMPREHENSIVE_THEME_DIRS }}" 19 | when: "{{ EDXAPP_ENABLE_COMPREHENSIVE_THEMING }}" 20 | -------------------------------------------------------------------------------- /playbooks/roles/alton/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role alton 12 | # 13 | # Example: 14 | # 15 | # dependencies: 16 | # - { 17 | # role: my_role 18 | # my_role_var0: "foo" 19 | # my_role_var1: "bar" 20 | # } 21 | 22 | dependencies: 23 | - common 24 | - supervisor 25 | - redis 26 | 27 | -------------------------------------------------------------------------------- /docker/build/xqwatcher/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/trusty-common:v3 2 | MAINTAINER edxops 3 | 4 | ADD . /edx/app/edx_ansible/edx_ansible 5 | COPY docker/build/xqwatcher/ansible_overrides.yml / 6 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 7 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook xqwatcher.yml \ 8 | -i '127.0.0.1,' -c local \ 9 | -t "install:base,install:configuration,install:system-requirements,install:app-requirements,install:code" \ 10 | -e@/ansible_overrides.yml 11 | WORKDIR /edx/app 12 | CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"] 13 | -------------------------------------------------------------------------------- /playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2: -------------------------------------------------------------------------------- 1 | [program:xqueue_consumer] 2 | 3 | command={{ xqueue_venv_bin }}/django-admin.py run_consumer --pythonpath={{ xqueue_code_dir }} --settings=xqueue.{{ XQUEUE_SETTINGS }} $WORKERS_PER_QUEUE 4 | 5 | user={{ common_web_user }} 6 | directory={{ xqueue_code_dir }} 7 | 8 | environment=LANG={{ XQUEUE_LANG }},WORKERS_PER_QUEUE={{xqueue_env_config.XQUEUE_WORKERS_PER_QUEUE}},SERVICE_VARIANT="xqueue" 9 | 10 | stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log 11 | stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log 12 | killasgroup=true 13 | stopasgroup=true 14 | startsecs=0 15 | 16 | -------------------------------------------------------------------------------- /util/jenkins/django-admin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd configuration 4 | pip install -r requirements.txt 5 | env 6 | 7 | ansible="ansible first_in_tag_Name_${environment}-${deployment}-worker -i playbooks/ec2.py -u ubuntu -s -U www-data -m shell -a" 8 | manage="cd /edx/app/edxapp/edx-platform && /edx/bin/python.edxapp ./manage.py chdir=/edx/app/edxapp/edx-platform" 9 | 10 | if [ "$service_variant" != "UNSET" ]; then 11 | manage="$manage $service_variant --settings aws" 12 | fi 13 | 14 | if [ "$help" = "true" ]; then 15 | manage="$manage help" 16 | fi 17 | 18 | echo "Running $ansible \"$manage $command $options\"" 19 | $ansible "$manage $command $options" 20 | -------------------------------------------------------------------------------- /docker/build/insights/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | ADD . /edx/app/edx_ansible/edx_ansible 5 | COPY docker/build/insights/ansible_overrides.yml / 6 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 7 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook insights.yml \ 8 | -i '127.0.0.1,' -c local \ 9 | -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" \ 10 | -e@/ansible_overrides.yml 11 | CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"] 12 | EXPOSE 8110 18110 13 | -------------------------------------------------------------------------------- /docker/build/insights/ansible_overrides.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | DOCKER_TLD: "edx" 4 | 5 | INSIGHTS_LMS_BASE: "http://lms.{{ DOCKER_TLD }}:8000" 6 | INSIGHTS_CMS_BASE: "http://cms.{{ DOCKER_TLD }}:8010" 7 | INSIGHTS_BASE_URL: "http://insights.{{ DOCKER_TLD }}:8110" 8 | INSIGHTS_MEMCACHE: 9 | - "memcache.{{ DOCKER_TLD }}:11211" 10 | ANALYTICS_API_ENDPOINT: "http://analtyicsapi.{{ DOCKER_TLD }}:8100/api/v0" 11 | 12 | INSIGHTS_DATABASES: 13 | # rw user 14 | default: 15 | ENGINE: 'django.db.backends.mysql' 16 | NAME: '{{ INSIGHTS_DATABASE_NAME }}' 17 | USER: 'rosencrantz' 18 | PASSWORD: 'secret' 19 | HOST: "db.{{ DOCKER_TLD }}" 20 | PORT: '3306' 21 | -------------------------------------------------------------------------------- /playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | {{ '{{' }} ansible_managed }} 4 | """ 5 | 6 | timeout = {{ '{{' }} {{ role_name }}_gunicorn_timeout }} 7 | bind = "{{ '{{' }} {{ role_name }}_gunicorn_host }}:{{ '{{' }} {{ role_name }}_gunicorn_port }}" 8 | pythonpath = "{{ '{{' }} {{ role_name }}_code_dir }}" 9 | workers = {{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKERS }} 10 | worker_class = "{{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKER_CLASS }}" 11 | 12 | {{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA_CONF }} 13 | -------------------------------------------------------------------------------- /playbooks/roles/logstash/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | LOGSTASH_DAYS_TO_KEEP: 30 3 | LOGSTASH_ROTATE: true 4 | logstash_app_dir: /edx/app/logstash 5 | logstash_log_dir: /edx/var/log/logstash 6 | logstash_data_dir: /edx/var/logstash/file_logs 7 | logstash_syslog_port: 514 8 | logstash_file: logstash-1.3.3-flatjar.jar 9 | logstash_url: "https://download.elasticsearch.org/logstash/logstash/{{ logstash_file }}" 10 | logstash_python_requirements: 11 | - pyes==0.19.0 12 | logstash_scripts_repo: https://github.com/crashdump/logstash-elasticsearch-scripts 13 | logstash_rotate_cron: 14 | hour: 5 15 | minute: 42 16 | logstash_optimize_cron: 17 | hour: 6 18 | minute: 15 19 | -------------------------------------------------------------------------------- /playbooks/roles/sitespeedio/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Role includes for role sitespeedio 12 | # 13 | 14 | dependencies: 15 | - common 16 | - role: oraclejdk 17 | oraclejdk_version: "8u60" 18 | oraclejdk_base: "jdk1.8.0_60" 19 | oraclejdk_build: "b27" 20 | oraclejdk_link: "/usr/lib/jvm/java-8-oracle" 21 | 22 | -------------------------------------------------------------------------------- /docker/build/ecommerce/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | RUN apt-get update 5 | 6 | ADD . /edx/app/edx_ansible/edx_ansible 7 | COPY docker/build/ecommerce/ansible_overrides.yml / 8 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 9 | 10 | COPY docker/build/ecommerce/ansible_overrides.yml / 11 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook ecommerce.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml 12 | COPY docker/build/xqueue/docker-run.sh / 13 | ENTRYPOINT ["/docker-run.sh"] 14 | EXPOSE 8130 15 | -------------------------------------------------------------------------------- /playbooks/edx-east/credentials.yml: -------------------------------------------------------------------------------- 1 | - name: Deploy edX Credentials Service 2 | hosts: all 3 | become: True 4 | gather_facts: True 5 | vars: 6 | ENABLE_DATADOG: False 7 | ENABLE_SPLUNKFORWARDER: False 8 | ENABLE_NEWRELIC: False 9 | CLUSTER_NAME: 'credentials' 10 | roles: 11 | - role: nginx 12 | nginx_sites: 13 | - credentials 14 | nginx_default_sites: 15 | - credentials 16 | - aws 17 | - credentials 18 | - role: datadog 19 | when: COMMON_ENABLE_DATADOG 20 | - role: splunkforwarder 21 | when: COMMON_ENABLE_SPLUNKFORWARDER 22 | - role: newrelic 23 | when: COMMON_ENABLE_NEWRELIC 24 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_common/templates/yarn-site.xml.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | yarn.nodemanager.aux-services 6 | mapreduce_shuffle 7 | 8 | 9 | 10 | yarn.nodemanager.aux-services.mapreduce.shuffle.class 11 | org.apache.hadoop.mapred.ShuffleHandler 12 | 13 | 14 | {% if yarn_site_config is defined %} 15 | {% for key,value in yarn_site_config.iteritems() %} 16 | 17 | {{ key }}} 18 | {{ value }} 19 | 20 | {% endfor %} 21 | {% endif %} 22 | 23 | -------------------------------------------------------------------------------- /playbooks/roles/hadoop_master/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | ## 11 | # Defaults for role hadoop_master 12 | # 13 | 14 | # 15 | # vars are namespace with the module name. 16 | # 17 | hadoop_master_role_name: hadoop_master 18 | 19 | # 20 | # OS packages 21 | # 22 | 23 | hadoop_master_debian_pkgs: [] 24 | 25 | hadoop_master_redhat_pkgs: [] 26 | -------------------------------------------------------------------------------- /playbooks/roles/jscover/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Download JSCover 4 | get_url: url={{ jscover_url }} dest=/var/tmp/jscover.zip 5 | 6 | - name: Unzip JSCover 7 | shell: unzip /var/tmp/jscover.zip -d /var/tmp/jscover 8 | creates=/var/tmp/jscover 9 | 10 | - name: Install JSCover JAR 11 | command: cp /var/tmp/jscover/target/dist/JSCover-all.jar /usr/local/bin/JSCover-all-{{ jscover_version }}.jar 12 | creates=/usr/local/bin/JSCover-all-{{ jscover_version }}.jar 13 | 14 | - name: Set JSCover permissions 15 | file: path="/usr/local/bin/JSCover-all-{{ jscover_version }}.jar" state=file 16 | owner=root group=root mode=0755 17 | 18 | -------------------------------------------------------------------------------- /playbooks/roles/mongo/templates/create_root.js.j2: -------------------------------------------------------------------------------- 1 | // Add super user 2 | conn = new Mongo(); 3 | db = conn.getDB("admin"); 4 | db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}'); 5 | 6 | if(db.getUser('{{ MONGO_ADMIN_USER }}') == null) { 7 | db.createUser( 8 | { 9 | "user": "{{ MONGO_ADMIN_USER }}", 10 | "pwd": "{{ MONGO_ADMIN_PASSWORD }}", 11 | "roles": ["root"] 12 | } 13 | ); 14 | } else { 15 | db.updateUser( 16 | "{{ MONGO_ADMIN_USER }}", 17 | { 18 | "pwd": "{{ MONGO_ADMIN_PASSWORD }}", 19 | "roles": ["root"] 20 | } 21 | ); 22 | } 23 | 24 | -------------------------------------------------------------------------------- /playbooks/roles/aide/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # install and configure aide IDS 3 | # 4 | - name: Install aide 5 | apt: 6 | name: aide 7 | state: present 8 | 9 | - name: Configure aide defaults 10 | template: 11 | src: etc/default/aide.j2 12 | dest: /etc/default/aide 13 | owner: root 14 | group: root 15 | mode: "0644" 16 | 17 | - name: Open read permissions on aide logs 18 | file: 19 | name: /var/log/aide 20 | recurse: yes 21 | state: directory 22 | mode: "0755" 23 | 24 | - name: Aide initial scan (this can take a long time) 25 | command: "aideinit -y -f" 26 | args: 27 | creates: "/var/lib/aide/aide.db" 28 | become: yes -------------------------------------------------------------------------------- /playbooks/roles/nginx/templates/edx/app/nginx/sites-available/s3_maintenance.j2: -------------------------------------------------------------------------------- 1 | {% if NGINX_EDXAPP_ENABLE_S3_MAINTENANCE %} 2 | location @maintenance { 3 | rewrite ^(.*) {{ NGINX_EDXAPP_S3_MAINTENANCE_FILE }} break; 4 | proxy_http_version 1.1; 5 | proxy_set_header Host s3.amazonaws.com; 6 | proxy_set_header Authorization ''; 7 | proxy_hide_header x-amz-id-2; 8 | proxy_hide_header x-amz-request-id; 9 | proxy_hide_header Set-Cookie; 10 | proxy_ignore_headers "Set-Cookie"; 11 | proxy_buffering off; 12 | proxy_intercept_errors on; 13 | proxy_pass https://s3.amazonaws.com; 14 | } 15 | {% endif %} 16 | -------------------------------------------------------------------------------- /playbooks/roles/xqueue/templates/xqueue_gunicorn.py.j2: -------------------------------------------------------------------------------- 1 | """ 2 | gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html 3 | 4 | {{ ansible_managed }} 5 | """ 6 | import multiprocessing 7 | 8 | preload_app = True 9 | timeout = 300 10 | bind = "{{ xqueue_gunicorn_host }}:{{ xqueue_gunicorn_port }}" 11 | pythonpath = "{{ xqueue_code_dir }}" 12 | 13 | {% if XQUEUE_WORKERS %} 14 | workers = {{ XQUEUE_WORKERS }} 15 | {% else %} 16 | workers = (multiprocessing.cpu_count()-1) * 2 + 2 17 | {% endif %} 18 | 19 | {{ common_close_all_caches }} 20 | 21 | def post_fork(server, worker): 22 | close_all_caches() 23 | 24 | {{ XQUEUE_GUNICORN_WORKERS_EXTRA_CONF }} 25 | -------------------------------------------------------------------------------- /playbooks/roles/xsy/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role xsy 13 | # 14 | # Overview: 15 | # 16 | # 17 | - name: restart xsy 18 | supervisorctl: 19 | name: xsy 20 | supervisorctl_path: "{{ supervisor_ctl }}" 21 | config: "{{ supervisor_cfg }}" 22 | state: restarted 23 | when: not disable_edx_services 24 | -------------------------------------------------------------------------------- /util/csmh-extended/migrate-same-database-instance.sh: -------------------------------------------------------------------------------- 1 | MINID=0 2 | MAXID=1003426362 3 | STEP=10000 4 | MIGRATE_USER=migrate 5 | PASSWORD='secret-password' 6 | HOST='my-database-instance' 7 | 8 | 9 | for ((i=0; i<=$MAXID; i+=$STEP)); do 10 | echo -n "$i"; 11 | mysql -u $MIGRATE_USER -p$PASSWORD -h $HOST wwc < 19 | name=gitreload 20 | supervisorctl_path={{ supervisor_ctl }} 21 | config={{ supervisor_cfg }} 22 | state=restarted 23 | when: not disable_edx_services 24 | -------------------------------------------------------------------------------- /playbooks/roles/haproxy/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role haproxy 13 | # 14 | # Overview: 15 | # 16 | # 17 | - name: restart haproxy 18 | service: name=haproxy state=restarted 19 | 20 | - name: reload haproxy 21 | service: name=haproxy state=reloaded 22 | 23 | - name: restart rsyslog 24 | service: name=rsyslog state=restarted 25 | -------------------------------------------------------------------------------- /docker/build/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM edxops/precise-common:latest 2 | MAINTAINER edxops 3 | 4 | USER root 5 | ADD . /edx/app/edx_ansible/edx_ansible 6 | COPY docker/build/nginx/ansible_overrides.yml / 7 | WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays 8 | 9 | RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook nginx.yml -c local \ 10 | -i '127.0.0.1,' \ 11 | -e@roles/edxapp/defaults/main.yml \ 12 | -e@roles/xqueue/defaults/main.yml \ 13 | -e@roles/certs/defaults/main.yml \ 14 | -e@roles/forum/defaults/main.yml 15 | 16 | RUN echo "\ndaemon off;" >> /etc/nginx/nginx.conf 17 | WORKDIR /etc/nginx 18 | CMD ["/usr/sbin/nginx"] 19 | EXPOSE 18000 48000 18010 48010 18020 20 | -------------------------------------------------------------------------------- /playbooks/roles/nltk/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | NLTK_DATA_DIR: "/usr/local/share/nltk_data" 3 | 4 | # Once the file is downloaded, it won't be downloaded again, 5 | # so if you need to version the data files, you should upload 6 | # your own version of the files with the version appended to the filename. 7 | NLTK_DATA: 8 | - { path: "taggers/maxent_treebank_pos_tagger", 9 | url: "http://nltk.github.io/nltk_data/packages/taggers/maxent_treebank_pos_tagger.zip" } 10 | - { path: "corpora/stopwords", 11 | url: "http://nltk.github.io/nltk_data/packages/corpora/stopwords.zip" } 12 | - { path: "corpora/wordnet", 13 | url: "http://nltk.github.io/nltk_data/packages/corpora/wordnet.zip" } 14 | -------------------------------------------------------------------------------- /playbooks/edx-east/set_hostname.yml: -------------------------------------------------------------------------------- 1 | # This is a utility play to set a hostname 2 | # on a server 3 | 4 | - name: Set hostname 5 | hosts: all 6 | become: True 7 | gather_facts: False 8 | tasks: 9 | - name: "Ensure we have a hostname" 10 | fail: msg="you must pass a hostname_fqdn var into this play" 11 | when: hostname_fqdn is not defined 12 | - name: Set hostname 13 | hostname: name={{ hostname_fqdn.split('.')[0] }} 14 | - name: Update /etc/hosts 15 | lineinfile: > 16 | dest=/etc/hosts 17 | regexp="^127\.0\.1\.1" 18 | line="127.0.1.1{{'\t'}}{{ hostname_fqdn.split('.')[0] }}{{'\t'}}{{ hostname_fqdn }}{{'\t'}}localhost" 19 | state=present 20 | -------------------------------------------------------------------------------- /playbooks/edx-east/jenkins_admin.yml: -------------------------------------------------------------------------------- 1 | # Configure an instance with the admin jenkins. 2 | - name: Configure instance(s) 3 | hosts: all 4 | become: True 5 | gather_facts: True 6 | vars: 7 | serial_count: 1 8 | serial: "{{ serial_count }}" 9 | roles: 10 | - aws 11 | - jenkins_admin 12 | # This requires an override of the following form: 13 | # SPLUNKFORWARDER_LOG_ITEMS: 14 | # - source: /edx/var/jenkins/jobs/*/builds/*/log 15 | # index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-jenkins' 16 | # sourcetype: jenkins_build 17 | # followSymlink: false 18 | # crcSalt: 19 | - role: splunkforwarder 20 | when: COMMON_ENABLE_SPLUNKFORWARDER 21 | -------------------------------------------------------------------------------- /playbooks/roles/go-agent-docker-server/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # Defaults for role go-agent-docker-server 12 | # 13 | 14 | # key for go-agents to autoregister with the go-server 15 | GO_SERVER_AUTO_REGISTER_KEY: "dev-only-override-this-key" 16 | GO_AGENT_DOCKER_RESOURCES: "tubular,python" 17 | GO_AGENT_DOCKER_ENVIRONMENT: "sandbox" 18 | GO_AGENT_DOCKER_CONF_HOME: "/tmp/go-agent/conf" -------------------------------------------------------------------------------- /playbooks/edx-east/jenkins_tools.yml: -------------------------------------------------------------------------------- 1 | # Configure an instance with the tool jenkins. 2 | - name: Configure Jenkins instance(s) 3 | hosts: all 4 | become: True 5 | gather_facts: True 6 | vars: 7 | serial_count: 1 8 | serial: "{{ serial_count }}" 9 | roles: 10 | - aws 11 | - tools_jenkins 12 | # This requires an override of the following form: 13 | # SPLUNKFORWARDER_LOG_ITEMS: 14 | # - source: /edx/var/jenkins/jobs/*/builds/*/log 15 | # index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-jenkins' 16 | # sourcetype: jenkins_build 17 | # followSymlink: false 18 | # crcSalt: 19 | - role: splunkforwarder 20 | when: COMMON_ENABLE_SPLUNKFORWARDER 21 | -------------------------------------------------------------------------------- /playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2: -------------------------------------------------------------------------------- 1 | {{ COMMON_LOG_DIR }}/tracking/tracking.log { 2 | {% for config in COMMON_TRACKING_LOG_ROTATION %} 3 | {{ config }} 4 | {% endfor %} 5 | compress 6 | create 7 | dateext 8 | dateformat -%Y%m%d-%s 9 | missingok 10 | nodelaycompress 11 | notifempty 12 | rotate 16000 13 | postrotate 14 | /usr/bin/killall -HUP rsyslogd 15 | endscript 16 | lastaction 17 | {% if COMMON_OBJECT_STORE_LOG_SYNC -%} 18 | {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" 19 | {% endif -%} 20 | endscript 21 | } 22 | -------------------------------------------------------------------------------- /playbooks/roles/mount_ebs/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # Expects a list of dicts with these keys 2 | # - device: /dev/xvdk 3 | # mount: /edx/var/mongo 4 | # options: "defaults,noatime" 5 | # fstype: ext4 6 | # While mount, options and fstype are pretty standard in our app, the device names 7 | # will be highly dependent on how you stand up your instances. 8 | # 9 | # Additionally - order is important if you have child directories. If you want to mount 10 | # /edx/var/mongo and /edx/var/mongo/mongodb/journal, you must specify them in that order, 11 | # otherwise this role will mount /edx/var/mongo over the top of /edx/var/mongo/mongodb/journal 12 | # which is not what you wanted. 13 | volumes: [] 14 | 15 | UNMOUNT_DISKS: false 16 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_admin/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # edX Configuration 4 | # 5 | # github: https://github.com/edx/configuration 6 | # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS 7 | # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions 8 | # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT 9 | # 10 | # 11 | # 12 | # Handlers for role jenkins_admin 13 | # 14 | # Overview: 15 | # 16 | 17 | # Have to use shell here because supervisorctl doesn't support 18 | # process groups. 19 | - name: restart nat monitor 20 | shell: "{{ jenkins_supervisor_ctl }} -c {{ jenkins_supervisor_cfg }} restart nat_monitor:*" 21 | when: not disable_edx_services 22 | -------------------------------------------------------------------------------- /playbooks/roles/jenkins_worker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # jenkins 3 | # 4 | # Provision a Jenkins worker instance. 5 | # - When sitespeed_worker is set, only apply the configuraiton necessary for running sitespeed.io 6 | # - When platform_worker is set, the resulting instance can run edx-platform tests 7 | 8 | # All jenkins workers 9 | - include: packer.yml 10 | - include: system.yml 11 | - include: python.yml 12 | 13 | # only platform workers 14 | - include: python_platform_worker.yml 15 | when: platform_worker is defined 16 | 17 | # Run appropriate tests 18 | - include: test.yml 19 | - include: test_platform_worker.yml 20 | when: platform_worker is defined 21 | - include: test_sitespeed_worker.yml 22 | when: sitespeed_worker is defined 23 | --------------------------------------------------------------------------------