├── .gitignore ├── README.md ├── Vagrantfile └── provisioning └── salt ├── minion ├── backup │ └── server ├── monitoring │ └── server └── web │ ├── mirror │ ├── pypi │ ├── pypi_dev │ └── pypi_log └── roots ├── pillar ├── backup │ └── server.sls ├── monitoring │ ├── client │ │ └── init.sls │ └── server │ │ └── init.sls ├── networking.sls ├── postgresql │ └── postgresql.sls ├── pypi-deploys │ └── pypi-dev.sls ├── pypi-mirror │ └── init.sls ├── pypi │ ├── log.sls │ └── web.sls ├── salt-master │ └── init.sls ├── secrets │ ├── backup │ │ └── pypi-dev.sls │ ├── monitoring.sls │ └── pypi-dev.sls ├── sudoers │ └── init.sls ├── top.sls ├── users │ ├── admin.sls │ └── init.sls ├── warehouse-deploys │ └── warehouse-dev.sls └── warehouse │ └── web.sls ├── prod-pillar ├── backup │ └── server.sls ├── monitoring │ ├── client │ │ ├── dfw.sls │ │ └── init.sls │ └── server │ │ └── init.sls ├── networking.sls ├── postgresql │ └── postgresql.sls ├── pypi-deploys │ ├── pypi.sls │ └── testpypi.sls ├── pypi-mirror │ └── init.sls ├── pypi │ ├── log.sls │ └── web.sls ├── salt-master │ └── init.sls ├── sudoers │ └── init.sls ├── top.sls ├── users │ ├── admin.sls │ └── init.sls ├── warehouse-deploys │ ├── pypi.sls │ └── testpypi.sls └── warehouse │ └── web.sls └── salt ├── _grains └── xenstore_datadog_tags.py ├── _modules └── ip_picker.py ├── auto-security └── init.sls ├── backup ├── base.sls ├── client │ ├── README.md │ ├── init.sls │ └── templates │ │ ├── backup.bash.jinja │ │ └── cron.jinja └── server │ ├── README.md │ ├── init.sls │ └── templates │ └── cron.jinja ├── base ├── auto-highstate.sls ├── config │ └── salt-logrotate.conf └── sanity.sls ├── datadog ├── config │ └── RPM-GPG-KEY-DATADOG ├── files │ └── datadog-nginx-status-codes.py └── init.sls ├── elasticsearch └── init.sls ├── firewall ├── config │ └── iptables.jinja └── init.sls ├── monitoring ├── client │ ├── base.sls │ ├── config │ │ ├── bucky.conf.jinja │ │ ├── bucky.ini │ │ ├── collectd.conf.jinja │ │ └── collectd.d │ │ │ ├── base.conf │ │ │ ├── nginx.conf │ │ │ ├── postgresql.conf.jinja │ │ │ ├── pypi-backend.conf │ │ │ ├── pypi-mirror.conf │ │ │ └── redis.conf │ ├── init.sls │ ├── nginx.sls │ ├── plugins │ │ └── python │ │ │ ├── pypi_backend.py │ │ │ ├── pypi_mirror.py │ │ │ ├── redis_info.py │ │ │ └── uwsgi_stats.py │ ├── postgresql.sls │ ├── pypi-backend.sls │ ├── pypi-mirror.sls │ └── redis.sls └── server │ ├── carbon.sls │ ├── config │ ├── carbon.conf.jinja │ ├── graphite-web-app.conf.jinja │ ├── graphite-web-nginx.conf.jinja │ ├── graphite-web.ini.jinja │ ├── local_settings.py.jinja │ ├── riemann.config.jinja │ └── storage-schemas.conf.jinja │ ├── graphite-web.sls │ ├── init.sls │ └── riemann.sls ├── nginx ├── config │ ├── RPM-GPG-KEY-NGINX │ ├── nginx.conf.jinja │ ├── nginx.logrotate │ └── nginx.ssl.conf.jinja └── init.sls ├── openvpn └── routing.sls ├── pkg └── git.sls ├── postgresql └── 93 │ ├── RPM-GPG-KEY-PGDG-93 │ ├── init.sls │ └── repo.sls ├── pypi-mirror ├── config │ ├── bandersnatch.conf.jinja │ ├── bandersnatch.logrotate.conf │ ├── bandersnatch.nginx.app.conf │ ├── bandersnatch.nginx.conf │ ├── crontab.jinja │ └── index.html.jinja └── init.sls ├── pypi ├── base.sls ├── config │ ├── pypi-cdn-log-archiver-wrapper.sh.jinja │ ├── pypi-docs-proxy.ini.jinja │ ├── pypi-syslog.logrotate.conf │ ├── pypi-worker.ini.jinja │ ├── pypi.ini.jinja │ ├── pypi.initd.jinja │ ├── pypi.logrotate.conf │ ├── pypi.nginx.app.conf.jinja │ ├── pypi.nginx.conf.jinja │ ├── pypi.nginx.ratelimit.conf.jinja │ ├── pypi.rsyslog.conf.jinja │ └── test-pg_hba.conf ├── dev-db.sls ├── files │ ├── dogstatsd_plugin.so │ └── error.html ├── log.sls └── web.sls ├── python ├── 27 │ ├── init.sls │ └── virtualenv.sls ├── 34 │ └── init.sls ├── 35 │ └── init.sls └── copr │ ├── config │ └── ewdurbin-pythons-el6.gpgkey │ └── init.sls ├── redis ├── copr │ ├── config │ │ └── jlaska-redis-28.gpgkey │ └── init.sls └── init.sls ├── sudoers ├── config │ └── salt.jinja └── init.sls ├── supervisor ├── config │ ├── supervisord.conf │ └── supervisord.init └── init.sls ├── tls └── init.sls ├── top.sls ├── users ├── config │ └── authorized_keys.jinja └── init.sls └── warehouse ├── base.sls ├── config ├── nginx.app.conf.jinja ├── nginx.conf.jinja ├── supervisor.ini.jinja └── unicorn.py.template └── web.sls /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | 3 | /provisioning/salt/roots/prod-pillar/secrets 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pypi-salt 2 | 3 | **note** 4 | 5 | data in `provisioning/salt/roots/pillar` is strictly for development environments. 6 | 7 | ## OSX Setup 8 | 9 | - Download and Install [Virtual Box](https://www.virtualbox.org/wiki/Downloads). 10 | 11 | - Download and Install [Vagrant](http://downloads.vagrantup.com/) version 1.3.0 or greater. 12 | 13 | - Start some servers! 14 | - An all in one PyPI development instance: 15 | - `vagrant up` 16 | - navigate to http://192.168.57.9/pypi 17 | - to hop into the environment: 18 | - `vagrant ssh` 19 | - `sudo -u devpypi -i` 20 | - `cd src` 21 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.provider :virtualbox do |vb| 6 | vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 7 | vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"] 8 | end 9 | 10 | config.vm.box_url = "https://github.com/CommanderK5/packer-centos-template/releases/download/0.6.8/vagrant-centos-6.8.box" 11 | config.vm.box = "centos-min" 12 | config.vm.provision "shell", inline: "yum -y update nss\*" 13 | 14 | # Fix for https://bugs.centos.org/view.php?id=9212 15 | config.vm.provision "shell", inline: "yum -y update python" 16 | 17 | config.vm.synced_folder "provisioning/salt/roots/", "/srv/" 18 | 19 | unless ENV['VAGRANT_SKIP_PYPI_DEV'] == '1' 20 | config.vm.define "pypi_dev" do |pypi_dev| 21 | 22 | pypi_dev.vm.hostname = "pypi-dev" 23 | pypi_dev.vm.network "private_network", ip: "192.168.57.9" 24 | pypi_dev.vm.network "private_network", ip: "172.16.57.9" 25 | 26 | pypi_dev.vm.provision :salt do |s| 27 | s.verbose = true 28 | s.minion_config = "provisioning/salt/minion/web/pypi_dev" 29 | s.run_highstate = true 30 | end 31 | end 32 | end 33 | 34 | if ENV['VAGRANT_BACKUP'] == '1' 35 | 36 | config.vm.define "backup" do |backup| 37 | backup.vm.network "private_network", ip: "192.168.57.201" 38 | backup.vm.network "private_network", ip: "172.16.57.201" 39 | 40 | backup.vm.provider :virtualbox do |vm| 41 | file_to_disk = '.vagrant/tmp/backup.vdi' 42 | vm.customize ['createhd', '--filename', file_to_disk, '--size', 10 * 1024] 43 | vm.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', file_to_disk] 44 | end 45 | 46 | backup.vm.provision :salt do |s| 47 | s.verbose = true 48 | s.install_type = "git v0.17.2" 49 | s.minion_config = "provisioning/salt/minion/backup/server" 50 | s.run_highstate = true 51 | end 52 | end 53 | 54 | end 55 | 56 | if ENV['VAGRANT_MONITORING'] == '1' 57 | config.vm.define "monitoring_server" do |monitoring_server| 58 | 59 | monitoring_server.vm.hostname = "pypi-monitoring" 60 | monitoring_server.vm.network "private_network", ip: "192.168.57.200" 61 | monitoring_server.vm.network "private_network", ip: "172.16.57.200" 62 | 63 | monitoring_server.vm.provision :salt do |s| 64 | s.verbose = true 65 | s.install_type = "git v0.17.2" 66 | s.minion_config = "provisioning/salt/minion/monitoring/server" 67 | s.run_highstate = true 68 | end 69 | end 70 | end 71 | 72 | if ENV['VAGRANT_PYPI'] == '1' 73 | config.vm.define "pypi" do |pypi| 74 | 75 | pypi.vm.network "private_network", ip: "192.168.57.10" 76 | pypi.vm.network "private_network", ip: "172.16.57.10" 77 | 78 | pypi.vm.provision :salt do |s| 79 | s.verbose = true 80 | s.install_type = "git v0.17.2" 81 | s.minion_config = "provisioning/salt/minion/web/pypi" 82 | s.run_highstate = true 83 | end 84 | end 85 | 86 | config.vm.define "pypi_log" do |pypi_log| 87 | 88 | pypi_log.vm.network "private_network", ip: "192.168.57.15" 89 | pypi_log.vm.network "private_network", ip: "172.16.57.15" 90 | 91 | pypi_log.vm.provision :salt do |s| 92 | s.verbose = true 93 | s.install_type = "git v0.17.2" 94 | s.minion_config = "provisioning/salt/minion/web/pypi_log" 95 | s.run_highstate = true 96 | end 97 | end 98 | 99 | config.vm.define "mirror" do |mirror| 100 | 101 | mirror.vm.network "private_network", ip: "192.168.57.20" 102 | 103 | mirror.vm.provision :salt do |s| 104 | s.verbose = true 105 | s.install_type = "git v0.17.2" 106 | s.minion_config = "provisioning/salt/minion/web/mirror" 107 | s.run_highstate = true 108 | end 109 | end 110 | end 111 | 112 | end 113 | -------------------------------------------------------------------------------- /provisioning/salt/minion/backup/server: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - backup_server 8 | -------------------------------------------------------------------------------- /provisioning/salt/minion/monitoring/server: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - monitoring_server 8 | -------------------------------------------------------------------------------- /provisioning/salt/minion/web/mirror: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - develop 8 | - pypi-mirror 9 | -------------------------------------------------------------------------------- /provisioning/salt/minion/web/pypi: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - pypi 8 | -------------------------------------------------------------------------------- /provisioning/salt/minion/web/pypi_dev: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - develop 8 | - pypi 9 | datadog_tags: 10 | - foo:bar 11 | - fizz:buzz 12 | - wu 13 | -------------------------------------------------------------------------------- /provisioning/salt/minion/web/pypi_log: -------------------------------------------------------------------------------- 1 | file_client: local 2 | file_roots: 3 | base: 4 | - /srv/salt 5 | grains: 6 | roles: 7 | - develop 8 | - pypi_log 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/backup/server.sls: -------------------------------------------------------------------------------- 1 | 2 | backup-server: 3 | volumes: 4 | /dev/sdb: /backup 5 | backups: 6 | devpypi-files: 7 | directory: /backup/devpypi/files 8 | user: devpypi 9 | increment_retention: 10m 10 | authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxqABhu46JE5zT68uNwpBdmqI/eYfepfA9PU8AxbGR96u5rPd8xyKJ2ZzTUgjrCKWLim4jutFgAtfCwkfjm/FrHQWNJZhZtQ1gT8a/bpIjHddjOFlELja1vFKkQsU6dsmYGTZxwByeMQl1+TWE5xmGrmqsgNnK+Twi01TCUmYs36M8OQGXle664XAapfc+4cAU4Clf3mH7xFIBJnCS12v5ES1J4pmnfBVMT85lGJMDW/vDI5Yx2A85bKOCwle1XpGsxLLfvvg7faNiCBjpj//pSEAI52tSq0hLtYSS5WJBz1tfJQPECCXVxa6LHKJB6RDT3eYyMQSugFSCrHxQ8j/F 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/monitoring/client/init.sls: -------------------------------------------------------------------------------- 1 | 2 | bucky: 3 | graphite_ip: 172.16.57.200 4 | graphite_port: 2002 5 | graphite_max_reconnects: 3 6 | graphite_reconnect_delay: 5 7 | 8 | monitoring_client: 9 | collectd_host: 127.0.0.1 10 | collectd_port: 25826 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/monitoring/server/init.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - secrets.monitoring: 3 | defaults: 4 | server_names: 5 | - '_' 6 | secret_key: deadbeef 7 | key: monitoring_secrets 8 | 9 | firewall: 10 | riemann_ports: 11 | port: 5555:5556 12 | source: 172.16.57.0/24 13 | riemann_udp_ports: 14 | port: 5556 15 | source: 172.16.57.0/24 16 | riemann_graphite: 17 | port: 2002 18 | source: 172.16.57.0/24 19 | graphite_ports: 20 | port: 2003:2004 21 | source: 172.16.57.0/24 22 | graphite_query_port: 23 | port: 7002 24 | source: 172.16.57.0/24 25 | http: 26 | port: 80 27 | https: 28 | port: 443 29 | 30 | riemann: 31 | host: 0.0.0.0 32 | graphite_host: 0.0.0.0 33 | graphite_port: 2002 34 | graphite_downstream_host: 127.0.0.1 35 | graphite_downstream_port: 2003 36 | 37 | graphite: 38 | allowed_hosts: '*' 39 | sqlite3_path: /var/lib/graphite-web/graphite.db 40 | https_only: true 41 | 42 | carbon: 43 | line_receiver_interface: 0.0.0.0 44 | line_reciver_port: 2003 45 | pickle_receiver_interface: 0.0.0.0 46 | pickle_receiver_port: 2004 47 | cache_query_interface: 0.0.0.0 48 | cache_query_port: 7002 49 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/networking.sls: -------------------------------------------------------------------------------- 1 | psf_internal_network: 192.168.57.0/24 2 | pypi_internal_network: 172.16.57.0/24 3 | vpn0_internal_network: 10.8.0.0/24 4 | vpn1_internal_network: 10.9.0.0/24 5 | rackspace_iad_service_net: 10.0.0.0/8 6 | 7 | psf_internal_vpn_gateway: 192.168.57.1 8 | pypi_internal_vpn_gateway: 172.16.57.1 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/postgresql/postgresql.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | postgresql: 4 | port: 5432 5 | 6 | 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls: -------------------------------------------------------------------------------- 1 | 2 | pypi-deploy-devpypi: 3 | name: devpypi 4 | user: devpypi 5 | group: devpypi 6 | user_uid: 6666 7 | group_gid: 6666 8 | 9 | source_uri: https://github.com/pypa/pypi-legacy.git 10 | source_rev: master 11 | 12 | fastly_syslog_name: pypi-dev 13 | path: /opt/devpypi 14 | data_mount: /data/devpypi 15 | data_device: 16 | type: local 17 | uri: None 18 | 19 | docs_bucket: pypi-docs 20 | files_bucket: pypi-files 21 | 22 | server_names: 23 | - 192.168.57.9 24 | tls_port: 8999 25 | docs_port: 8989 26 | internal_pypi_port: 40713 27 | internal_docs_port: 40715 28 | url: https://192.168.57.9 29 | 30 | statuspage_id: 928bjjg42vzc 31 | 32 | rate_limit: 33 | enable: True 34 | zone_size: 30m 35 | max_rate: 5r/s 36 | burst: 10 37 | nodelay: False 38 | 39 | cdn_log_archiver: 40 | pypi_log_bucket: testpypi-cdn-logs 41 | s3_host: objects.dreamhost.com 42 | debug: true 43 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/pypi-mirror/init.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | http: 4 | port: 80 5 | https: 6 | port: 443 7 | pypi-mirror: 8 | port: 9000 9 | testpypi-mirror: 10 | port: 9001 11 | 12 | bandersnatch: 13 | pypi: 14 | mirror: 15 | directory: /data/pypi-mirror 16 | master: https://pypi.python.org 17 | timeout: 60 18 | workers: 20 19 | stop-on-error: false 20 | delete-packages: true 21 | statistics: 22 | access-log-pattern: /var/log/nginx/pypi-mirror/access*.log 23 | server_names: 24 | - 192.168.57.20 25 | - pypi.python.org 26 | tls_port: 9000 27 | testpypi: 28 | mirror: 29 | directory: /data/testpypi-mirror 30 | master: https://testpypi.python.org 31 | timeout: 30 32 | workers: 20 33 | stop-on-error: false 34 | delete-packages: true 35 | statistics: 36 | access-log-pattern: /var/log/nginx/testpypi-mirror/access*.log 37 | server_names: 38 | - testpypi.python.org 39 | tls_port: 9001 40 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/pypi/log.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | syslog_tcp_72: 4 | source: 199.27.72.0/24 5 | port: 514 6 | syslog_tcp_77: 7 | source: 199.27.77.0/24 8 | port: 514 9 | syslog_udp_72: 10 | source: 199.27.72.0/24 11 | port: 514 12 | protocol: udp 13 | syslog_udp_77: 14 | source: 199.27.77.0/24 15 | port: 514 16 | protocol: udp 17 | redis: 18 | source: 172.16.57.0/24 19 | port: 6379 20 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/pypi/web.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_tags: 3 | - lol:wut 4 | - tang 5 | - wu 6 | 7 | datadog_dogstreams: 8 | - /var/log/nginx/pypi/access.log:/usr/share/datadog/datadog-nginx-status-codes.py:parse 9 | 10 | firewall: 11 | http: 12 | port: 80 13 | https: 14 | port: 443 15 | http_dev: 16 | port: 8999 17 | http_docs: 18 | port: 8989 19 | internal-pypi-port: 20 | port: 40713 21 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 22 | internal-testpypi-port: 23 | port: 40714 24 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 25 | internal-docs-port: 26 | port: 40715 27 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 28 | internal-testdocs-port: 29 | port: 40716 30 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 31 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/salt-master/init.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | salt_master: 4 | port: 4505:4506 5 | source: 172.16.57.0/24 6 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/secrets/backup/pypi-dev.sls: -------------------------------------------------------------------------------- 1 | 2 | backup: 3 | directories: 4 | pypi-data: 5 | source_directory: /data/devpypi 6 | target_host: 172.16.57.201 7 | target_directory: /backup/devpypi/files 8 | target_user: devpypi 9 | frequency: hourly 10 | user: devpypi 11 | ssh_key: | 12 | -----BEGIN RSA PRIVATE KEY----- 13 | MIIEowIBAAKCAQEAsagAYbuOiROc0+vLjcKQXZqiP3mH3qXwPT1PAMWxkferuaz3 14 | fMciidmc01II6wili4puI7rRYALXwsJH45vxax0FjSWYWbUNYE/Gv26SIx3XYzhZ 15 | RC42tbxSpELFOnbJmBk2ccAcnjEJdfk1hOcZhq5qrIDZyvk8ItNUwlJmLN+jPDkB 16 | l5XuuuFwGqX3PuHAFOApX95h+8RSASZwktdr+REtSeKZp3wVTE/OZRiTA1v7wyOW 17 | MdgPOWyjgsJXtV6RrMSy3774O32jYggY6Y//6UhACOdrUqtIS7WEkuViQc9bXyUD 18 | xAgl1cWuixyiQekQ093mMjEEroBUgqx8UPI/xQIDAQABAoIBACigXa374SWRuZxw 19 | 4LTDWJY/RXk0hpCw69ZlTcrEas4RkFC+sD31n/1cKVPd/7IX4RufBX7gOv80xzh/ 20 | i0cOo0+2bE2R2lwxXiS3OaEPXRXwvg+vlCJWWyaGMXPk3Qt4nLNOmLe8kg7O8fXr 21 | joSdAKZe/oACW0viYREpuMlTZJBAFpqCq95j1t2dCA0moPQVjxhR4QAhLkT4siHv 22 | Ed3qgYT4Vj9nTiXW6cGbz9O8WjTg9ttW0llWBpRLsChLfx8fa0yiCv5qMNGOnH0D 23 | POf+0rAHq6ijuqqOBDeKqH/887/Tqx3mEEYjQhynIGhSOlFCp43MGegfMmwfvTVm 24 | LzvaHoUCgYEA2YUDnUFIEjHyGFMs4AriyJaByFAb1K5jJJkcp31IJ7pmLj6WQz2O 25 | usBosNuuK9c4Xm00848ukSOXdOLMXViyG0XK2eXbObAUt19RD5+soC7F/mTiYPPA 26 | EOUqqgZ+VcxqII6tLIxKeS4HQpz0Zc2zncl8BPAR3eOqg9IG+GSJRR8CgYEA0RWq 27 | Fgjcm2Dhcb1q4Ie812Htr+0zeILym3OZUxGOY1MvbNknlf0Q33uQLiou8YLkmwZF 28 | CehDPolmwn/ahVxFhbkzSCbPjM7BNxtvwbaZaqVoNmVS9gE7fHOPhCgKijdU0L7s 29 | IN6eJ6SzApkqf0BQ1Yqrik9OixbYUguvHxLn2psCgYBzyooFAUZjYTEV39kInuLg 30 | krYdsv9NtVNTnSoSwu9RLrnMLkcBHljHczuHwjmyXsxD//BrIzJP0tmCQGU338pY 31 | GEwGuIR97gzpHJVjMsXLM3r0lDGqGLeKhuOyROiltb5c/HaVO009utHklPbI5rqR 32 | 6Trayg1IyDPyHjDVs3cbUwKBgQCQnUxsOyri6WplMh9HN3tc+aXdxdGQ6/mDnbwR 33 | 4ZW7i2DFB5nCuyu9d4hs9c5MSz11ICwGQzine3+wzZ/GF+EaMdOPdxCdErA/PmHY 34 | +UQ5qDhhT0nHT2jmlkNQpCVOHiEy1KsbvP5k6xzJkkj7hO+kE2q8mkf4Gg/7B4vT 35 | kU7+OwKBgEfjMJIvGt2xhS3tZZgYKVlh3/gymzD0eoa4DPwVmjWnQgLVtRmV25X2 36 | 5FWJLq+ryEbJNI6Rl3LG3c/K15I8jNQpuVRgeK7ydf2U3g4pzFPOlsb5UgJEETdi 37 | pqMI3F6ButHsMcwjZotivkf3baM1FmnjIJ4oeTFPyagaLZdRCuiT 38 | -----END RSA PRIVATE KEY----- 39 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/secrets/monitoring.sls: -------------------------------------------------------------------------------- 1 | secret_key: foobarbaz 2 | server_names: 3 | - 172.16.57.200 4 | - wow.so.secret.com 5 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/secrets/pypi-dev.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_api_key: bizzfuzz 3 | 4 | secrets-pypi-deploy-devpypi: 5 | database_url: postgresql://testpypi:testpypi@localhost/testpypi 6 | database_download_statistics_url: postgresql://testpypi:testpypi@localhost/testpypi 7 | elasticsearch: 8 | host: localhost 9 | port: 9200 10 | postgresql: 11 | host: 127.0.0.1 12 | database: testpypi 13 | user: testpypi 14 | password: testpypi 15 | redis: 16 | queue_redis_url: redis://localhost:6379/0 17 | count_redis_url: redis://localhost:6379/1 18 | cache_redis_url: redis://localhost:6379/2 19 | aws: 20 | aws_access_key_id: deadbeef 21 | aws_secret_access_key: deadbeef 22 | webui: 23 | cheescake_password: secret 24 | reset_secret: secret 25 | logging: 26 | fromaddr: foo@bar.com 27 | toaddrs: fiz@bar.com 28 | sentry: 29 | dsn: '' 30 | fastly: 31 | api_key: '' 32 | service_id: '' 33 | uwsgi: 34 | processes: 2 35 | smtp: 36 | hostname: localhost 37 | starttls: 'off' 38 | auth: 'off' 39 | login: None 40 | password: None 41 | pubkey: | 42 | -----BEGIN PUBLIC KEY----- 43 | publiclolnope 44 | -----END PUBLIC KEY----- 45 | privkey: | 46 | -----BEGIN DSA PRIVATE KEY----- 47 | privatelolnope 48 | -----END DSA PRIVATE KEY----- 49 | cdn_log_archiver: 50 | secret_key: lmsecretao 51 | access_key: accesslol 52 | authomatic: 53 | secure: 'true' 54 | secret: randodataz 55 | google: 56 | client_id: deadbeef-deadbeef.googleusercontent.com 57 | client_secret: deadbeef 58 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/sudoers/init.sls: -------------------------------------------------------------------------------- 1 | sudoer_groups: 2 | psf-admin: 3 | commands: 4 | - "ALL=(ALL) NOPASSWD: ALL" 5 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/top.sls: -------------------------------------------------------------------------------- 1 | base: 2 | 3 | '*': 4 | - networking 5 | - users 6 | - sudoers 7 | - monitoring.client 8 | 9 | 'roles:salt-master': 10 | - match: grain 11 | - salt-master 12 | 13 | 'roles:pypi-mirror': 14 | - match: grain 15 | - pypi-mirror 16 | 17 | 'G@roles:pypi not G@roles:develop': 18 | - match: compound 19 | - pypi.web 20 | - pypi-deploys.testpypi 21 | - secrets.testpypi 22 | 23 | 'G@roles:pypi_log not G@roles:develop': 24 | - match: compound 25 | - pypi.log 26 | - pypi-deploys.testpypi 27 | - secrets.testpypi 28 | 29 | 'G@roles:pypi and G@roles:develop': 30 | - match: compound 31 | - pypi.web 32 | - pypi-deploys.pypi-dev 33 | - secrets.pypi-dev 34 | - secrets.backup.pypi-dev 35 | - warehouse-deploys.warehouse-dev 36 | 37 | 'G@roles:pypi_log and G@roles:develop': 38 | - match: compound 39 | - pypi.log 40 | - pypi-deploys.pypi-dev 41 | - secrets.pypi-dev 42 | - warehouse-deploys.warehouse-dev 43 | 44 | 'roles:monitoring_server': 45 | - match: grain 46 | - monitoring.server 47 | 48 | 'roles:backup_server': 49 | - match: grain 50 | - backup.server 51 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/users/admin.sls: -------------------------------------------------------------------------------- 1 | users-admin: 2 | vagrant-user: 3 | add_group: 4 | - psf-admin 5 | ssh_keys: 6 | - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/users/init.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - users.admin 3 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/warehouse-deploys/warehouse-dev.sls: -------------------------------------------------------------------------------- 1 | warehouse-deploy-devpypi: 2 | name: devwarehouse 3 | user: devwarehouse 4 | group: devwarehouse 5 | user_uid: 7666 6 | group_gid: 7666 7 | path: /opt/devwarehouse 8 | 9 | port: 9000 10 | 11 | source_uri: https://github.com/pypa/warehouse.git 12 | source_rev: werkzeug 13 | 14 | config: 15 | debug: true 16 | site: 17 | name: Warehouse (dev) 18 | assets: 19 | directory: "data/static" 20 | database: 21 | url: "{{ salt['pillar.get']('secrets-pypi-deploy-devpypi:database_url', "postgresql://testpypi:testpypi@localhost/testpypi") }}" 22 | download_statistics_url: "{{ salt['pillar.get']('secrets-pypi-deploy-devpypi:database_download_statistics_url', "postgresql://testpypi:testpypi@localhost/testpypi") }}" 23 | redis: 24 | url: "{{ salt['pillar.get']('secrets-pypi-deploy-devpypi:redis:count_redis_url', "redis://localhost:6379/1") }}" 25 | search: 26 | hosts: 27 | - host: {{ salt['pillar.get']('secrets-pypi-deploy-devpypi:elasticsearch:host', "127.0.0.1") }} 28 | port: {{ salt['pillar.get']('secrets-pypi-deploy-devpypi:elasticsearch:port', "9200") }} 29 | paths: 30 | documentation: "/data/devpypi/packagedocs" 31 | packages: "/data/devpypi/packages" 32 | urls: 33 | documentation: "http://pythonhosted.org/" 34 | security: 35 | csp: 36 | default-src: 37 | - "'self'" 38 | style-src: 39 | - "'self'" 40 | - cloud.typography.com 41 | font-src: 42 | - "'self'" 43 | - "data:" 44 | logging: 45 | formatters: 46 | console: 47 | format: '[%(asctime)s %(levelname)s] %(message)s' 48 | datefmt: '%Y-%m-%d %H:%M:%S' 49 | handlers: 50 | console: 51 | class: logging.StreamHandler 52 | formatter: console 53 | level: DEBUG 54 | stream: ext://sys.stdout 55 | loggers: 56 | warehouse: 57 | level: DEBUG 58 | elasticsearch: 59 | level: DEBUG 60 | elasticsearch.trace: 61 | level: DEBUG 62 | root: 63 | level: INFO 64 | handlers: [console] 65 | -------------------------------------------------------------------------------- /provisioning/salt/roots/pillar/warehouse/web.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | http_test: 4 | port: 9000 5 | http_pypi: 6 | port: 9001 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/backup/server.sls: -------------------------------------------------------------------------------- 1 | 2 | backup-server: 3 | volumes: 4 | /dev/xvdb: /backup 5 | backups: {} 6 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/monitoring/client/dfw.sls: -------------------------------------------------------------------------------- 1 | 2 | bucky: 3 | graphite_ip: 162.242.222.99 4 | graphite_port: 2002 5 | graphite_max_reconnects: 3 6 | graphite_reconnect_delay: 5 7 | 8 | monitoring_client: 9 | collectd_host: 127.0.0.1 10 | collectd_port: 25826 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/monitoring/client/init.sls: -------------------------------------------------------------------------------- 1 | 2 | bucky: 3 | graphite_ip: 172.16.57.10 4 | graphite_port: 2002 5 | graphite_max_reconnects: 3 6 | graphite_reconnect_delay: 5 7 | 8 | monitoring_client: 9 | collectd_host: 127.0.0.1 10 | collectd_port: 25826 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/monitoring/server/init.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - secrets.monitoring: 3 | defaults: 4 | server_names: 5 | - '_' 6 | secret_key: deadbeef 7 | key: monitoring_secrets 8 | 9 | datadog_tags: 10 | - service:pypi 11 | - role:monitoring 12 | 13 | firewall: 14 | riemann_ports: 15 | port: 5555:5556 16 | source: 172.16.57.0/24 17 | riemann_udp_ports: 18 | port: 5556 19 | source: 172.16.57.0/24 20 | riemann_graphite: 21 | port: 2002 22 | source: 172.16.57.0/24 23 | graphite_ports: 24 | port: 2003:2004 25 | source: 172.16.57.0/24 26 | graphite_query_port: 27 | port: 7002 28 | source: 172.16.57.0/24 29 | http: 30 | port: 80 31 | https: 32 | port: 443 33 | riemann_graphite_from_backup: 34 | port: 2002 35 | source: 166.78.184.219 36 | graphite_ports_from_backup: 37 | port: 2003:2004 38 | source: 166.78.184.219 39 | riemann_graphite_from_ord_mirror: 40 | port: 2002 41 | source: 23.253.174.176 42 | graphite_ports_from_ord_mirror: 43 | port: 2003:2004 44 | source: 23.253.174.176 45 | 46 | 47 | riemann: 48 | host: 0.0.0.0 49 | graphite_host: 0.0.0.0 50 | graphite_port: 2002 51 | graphite_downstream_host: 127.0.0.1 52 | graphite_downstream_port: 2003 53 | 54 | graphite: 55 | allowed_hosts: '*' 56 | sqlite3_path: /var/lib/graphite-web/graphite.db 57 | https_only: true 58 | 59 | carbon: 60 | line_receiver_interface: 0.0.0.0 61 | line_reciver_port: 2003 62 | pickle_receiver_interface: 0.0.0.0 63 | pickle_receiver_port: 2004 64 | cache_query_interface: 0.0.0.0 65 | cache_query_port: 7002 66 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/networking.sls: -------------------------------------------------------------------------------- 1 | psf_internal_network: 192.168.5.0/24 2 | pypi_internal_network: 172.16.57.0/24 3 | vpn0_internal_network: 10.8.0.0/24 4 | vpn1_internal_network: 10.9.0.0/24 5 | rackspace_iad_service_net: 10.0.0.0/8 6 | 7 | psf_internal_vpn_gateway: 192.168.5.10 8 | pypi_internal_vpn_gateway: 172.16.57.17 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/postgresql/postgresql.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | postgresql: 4 | source: 172.16.57.0/24 5 | port: 5432 6 | 7 | 8 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/pypi-deploys/pypi.sls: -------------------------------------------------------------------------------- 1 | 2 | pypi-deploy-pypi: 3 | name: pypi 4 | user: pypi 5 | group: pypi 6 | user_uid: 7000 7 | group_gid: 7000 8 | 9 | source_uri: https://github.com/pypa/pypi-legacy.git 10 | source_rev: production 11 | 12 | fastly_syslog_name: counterpypicdn 13 | path: /opt/pypi 14 | site_packages: False 15 | data_mount: /data/pypi 16 | data_device: 17 | type: local 18 | uri: None 19 | 20 | docs_bucket: pypi-docs 21 | files_bucket: pypi-files 22 | 23 | server_names: 24 | - pypi.python.org 25 | - legacy.pypi.org 26 | - pypi.a.ssl.fastly.net 27 | tls_port: 9000 28 | docs_port: 9010 29 | internal_pypi_port: 40713 30 | internal_docs_port: 40715 31 | url: https://pypi.python.org 32 | 33 | statuspage_id: 2p66nmmycsj3 34 | 35 | cdn_log_archiver: 36 | pypi_log_bucket: pypi-cdn-logs 37 | s3_host: objects-us-west-1.dream.io 38 | debug: false 39 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/pypi-deploys/testpypi.sls: -------------------------------------------------------------------------------- 1 | 2 | pypi-deploy-testpypi: 3 | name: testpypi 4 | user: testpypi 5 | group: testpypi 6 | user_uid: 6666 7 | group_gid: 6666 8 | 9 | source_uri: https://github.com/pypa/pypi-legacy.git 10 | source_rev: master 11 | 12 | fastly_syslog_name: testpypicdn 13 | path: /opt/testpypi 14 | site_packages: False 15 | data_mount: /data/testpypi 16 | data_device: 17 | type: local 18 | uri: None 19 | 20 | docs_bucket: pypi-docs-staging 21 | files_bucket: pypi-files-staging 22 | 23 | server_names: 24 | - testpypi.python.org 25 | - testpypi.a.ssl.fastly.net 26 | tls_port: 9001 27 | docs_port: 9011 28 | internal_pypi_port: 40714 29 | internal_docs_port: 40716 30 | url: https://testpypi.python.org 31 | 32 | statuspage_id: 928bjjg42vzc 33 | 34 | rate_limit: 35 | enable: True 36 | zone_size: 10m 37 | max_rate: 3r/s 38 | burst: 6 39 | nodelay: False 40 | 41 | cdn_log_archiver: 42 | pypi_log_bucket: testpypi-cdn-logs 43 | s3_host: objects-us-west-1.dream.io 44 | debug: true 45 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/pypi-mirror/init.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_tags: 3 | - service:pypi 4 | - role:mirror 5 | 6 | firewall: 7 | http: 8 | port: 80 9 | https: 10 | port: 443 11 | pypi-mirror: 12 | port: 9000 13 | testpypi-mirror: 14 | port: 9001 15 | 16 | bandersnatch: 17 | pypi: 18 | mirror: 19 | directory: /data/pypi-mirror 20 | master: https://pypi.python.org 21 | timeout: 10 22 | workers: 10 23 | stop-on-error: false 24 | delete-packages: true 25 | statistics: 26 | access-log-pattern: /var/log/nginx/pypi-mirror/access*.log 27 | server_names: 28 | - mirror0.pypi.io 29 | - pypi.python.org 30 | tls_port: 9000 31 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/pypi/log.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_tags: 3 | - service:pypi 4 | - role:counter 5 | 6 | firewall: 7 | syslog_tcp_72: 8 | source: 199.27.72.0/24 9 | port: 514 10 | syslog_tcp_77: 11 | source: 199.27.77.0/24 12 | port: 514 13 | syslog_udp_72: 14 | source: 199.27.72.0/24 15 | port: 514 16 | protocol: udp 17 | syslog_udp_77: 18 | source: 199.27.77.0/24 19 | port: 514 20 | protocol: udp 21 | redis: 22 | source: 172.16.57.0/24 23 | port: 6379 24 | elasticsearch: 25 | source: 172.16.57.0/24 26 | port: 9200 27 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/pypi/web.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_tags: 3 | - service:pypi 4 | - role:web 5 | 6 | datadog_dogstreams: 7 | - /var/log/nginx/pypi/access.log:/usr/share/datadog/datadog-nginx-status-codes.py:parse 8 | 9 | firewall: 10 | http: 11 | port: 80 12 | https: 13 | port: 443 14 | pypi-port: 15 | port: 9000 16 | testpypi-port: 17 | port: 9001 18 | pythonhosted-port: 19 | port: 9010 20 | test-pythonhosted-port: 21 | port: 9011 22 | internal-pypi-port: 23 | port: 40713 24 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 25 | internal-testpypi-port: 26 | port: 40714 27 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 28 | internal-docs-port: 29 | port: 40715 30 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 31 | internal-testdocs-port: 32 | port: 40716 33 | src: {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }} 34 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/salt-master/init.sls: -------------------------------------------------------------------------------- 1 | 2 | datadog_tags: 3 | - service:pypi 4 | - role:salt-master 5 | 6 | firewall: 7 | salt_master_pypi_internal: 8 | port: 4505:4506 9 | source: 172.16.57.0/24 10 | salt_master_remote_backup: 11 | port: 4505:4506 12 | source: 166.78.184.219 13 | salt_master_ord_mirror: 14 | port: 4505:4506 15 | source: 23.253.174.176 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/sudoers/init.sls: -------------------------------------------------------------------------------- 1 | sudoer_groups: 2 | psf-admin: 3 | commands: 4 | - "ALL=(ALL) NOPASSWD: ALL" 5 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/top.sls: -------------------------------------------------------------------------------- 1 | base: 2 | 3 | '*': 4 | - networking 5 | - users 6 | - sudoers 7 | - monitoring.client 8 | - secrets.monitoring.datadog 9 | 10 | 'roles:salt-master': 11 | - match: grain 12 | - salt-master 13 | 14 | 'roles:pypi-mirror': 15 | - match: grain 16 | - pypi-mirror 17 | - monitoring.client.dfw 18 | 19 | 'G@roles:pypi not G@roles:develop': 20 | - match: compound 21 | - pypi.web 22 | - pypi-deploys.testpypi 23 | - secrets.testpypi 24 | - pypi-deploys.pypi 25 | - secrets.pypi 26 | 27 | 'G@roles:pypi_log not G@roles:develop': 28 | - match: compound 29 | - pypi.log 30 | - pypi-deploys.testpypi 31 | - secrets.testpypi 32 | - pypi-deploys.pypi 33 | - secrets.pypi 34 | 35 | 'G@roles:warehouse not G@roles:develop': 36 | - match: compound 37 | - secrets.testpypi 38 | - warehouse-deploys.testpypi 39 | - secrets.pypi 40 | - warehouse-deploys.pypi 41 | - warehouse.web 42 | 43 | 44 | 'G@roles:pypi and G@roles:develop': 45 | - match: compound 46 | - pypi.web 47 | - pypi-deploys.pypi-dev 48 | - secrets.pypi-dev 49 | 50 | 'G@roles:pypi_log and G@roles:develop': 51 | - match: compound 52 | - pypi.log 53 | - pypi-deploys.pypi-dev 54 | - secrets.pypi-dev 55 | 56 | 'roles:monitoring_server': 57 | - match: grain 58 | - monitoring.server 59 | 60 | 'roles:backup_server': 61 | - match: grain 62 | - backup.server 63 | - monitoring.client.dfw 64 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/users/admin.sls: -------------------------------------------------------------------------------- 1 | users-admin: 2 | ernestd: 3 | add_group: 4 | - psf-admin 5 | ssh_keys: 6 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZOuPNmBeKubAvLrnZlVGzp49Duw/qn8cq4IsOmxKsiVtiTDdLLKr6YBA67CPu/QrVEZYMU/N7RpNOKRfqueYdw8aNB+KgGOy8B/OIiX2obi854q0B81NaYTxdjUmuo67q9gNcdrEv6GmJJzBFqx//d4Nl4F4pcQLFTmxfZg3MDB1zyo2qP8ZpW5jttNty7LpvFlReSHEB/87o7wlNrhnrwSg68NykC2x/DzPv5ZOY5ccW4YiatjS5R2P7MDWv+aabbzZD4G0r3ElaXyW4fRV4KWuyh0ow/5djG0ZIWQSSuqqsnxcmRiV67IV28X6c2uw2zoFO5LLYly8UKMC6OLuFJlrmUPuzXvEeB/AcudgPPFxaip0aSzhowih7Ij0i+rVgma535q6pPqSkPCAcOLpmSi0yk23V9Mr38zg2d33poqV+bFbtSOE3a2gA52rDj4+YMcHQZqxZrgUv0KrSuKwcEm5WyBSOYI1VUoGCxiF27HY+9NaTqfMSHF0AJIVGHsv31uAWa4wqynQVrbZfH4wxlC6yC447HKjizp2LIoZx16LqSHMFfCGQBI4trZPGwZb+OBq2rhFY0GxgQaowrIK0E+VgTiDjcq6VtraglEHJsysK7jj4NiEWiIFVEv0WNW5e2DKVQKZ9BrN7sILcVp+g9VUSRirAZTXzheHLb6+8Hw== ewd3yubikey2018 7 | dstufft: 8 | add_group: 9 | - psf-admin 10 | ssh_keys: 11 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC5md5DAPib7J+uGHanzgQOJ9GwGYAi7RLbG+rB/0NRk8UbUnwzn0JwkqNTXzeomUapO5Z3cOxQ98jzb0k03hGOzhzjIQpaKI1uKxPPquDevf/PwM5ZQaJzlx/8ah76GzJtEQpIIIbw/fofzywv9pZKTBCDL3wBHB94oByQjr0BG9CfjbMZq6FXcBKfo89L56nLQ8cdvxg2tjNJQElva5gL/xnqjpowtQYjA9MPKFmDwJPcrRF2AstBg5Zpkg+8K4JhJltucXTPEva97alK9prshGFY6XLtVD0mtgbwpHFXjFm7cIQYr8XG3pJdtWki0fLg0o3W1YBukQ+reDblT8SnFaDscgF1gStTra5zXfVF5OJaaRFE8zaYuwC01DQT9sN9G4fV4eK8HRbgpObCJcxnCyTs/SYGVhO1PpOiQYmyswGUlV4vU8G3gl3u0D+gkcpHRkdko0HlFbNUt1wKIZWcGLJcKkNWMKGSf1TaciU+6+2A4QVDxtdab8HdjnbuugX9/FckqjZypaUOwl8U4fYc4JbdUQ78PvcQrSQvhRPZB+1KSvm8rwRuBnFWiNlYmThLhGmKDBXegNF8eFRe4dApzv2DyshasHs4tNv46YIox6FdFEw0voRNPqhbTCF3XIdOQvxkHeZRyGGWv4WCTImM5+3GhXrbOQUtB9NjfM7rhQ== donald@stufft.io 12 | benjaminws: 13 | add_group: 14 | - psf-admin 15 | ssh_keys: 16 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1BJUpRtzq1fCntjuNm4YeIDcefBFbkjzFCvN7Zot4UVWpExWqOLJynRYmaAUAFnJNQd5QuXsBIEmC9ySPV0gs+ueX9yg+RLieXcPoym2fMQ7UgmkaJloYgLnWJM3apG0UnGEDRO6Bz4cm+PC5NPfuZlOdYeOmNVKZoOe3via2RABec+hsWRdr2mD7OVL4PUR3AL3IPa9r8WlLhIBG53MkiVU2su8RVnEEyHmc61YQL8sFnI2zt6aSNiFuHvo6sHL3cMsP9XNArOtONZCc3NPvzN9Lh9jCk+JEe47ox/17CxMCOVhn3B9nRh2oGXydYf6LWH2wkhQ5y07dIjULKi9T benjaminws@macbuntu 17 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcY/+b32IynLZhF/fjBfGjUjGlS1XCaMYKNxPZNekBv0hWteBh185k3A1yAZWRWAgLsvpHpe5Srs3Wxoz+NF51UWHMYVtpPzXEmpcjsqOe96rKrixFSlrYt89iHklW4FdAV3oJbbQpvXb8c6eFD6dantzmHj8FFRg+f5Bb+lsGhLIzxDcjcKJbySGLHHS+SgQvaXMFd1XE+Gs/SXgQxpbWV347BdOETJplA96jVB74bxoIP+GuCImO34VCu4eG+klnhMeY2MscYgmBa3ePjD86qef0StBu9zzruR5s+y4cYQK8h5Xm2+sC6RdZbZaSeQL+yfYXhPhfvEv4v5WT/QDb bsmith@bsmith-laptop 18 | coderanger: 19 | add_group: 20 | - psf-admin 21 | ssh_keys: 22 | - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAvV0vQo3OpzqDFDBHW5o5abdbNJNNg9YkiawTpSnOusB+E9Hp2Pae1jT3r+7ZUqIIutXuNOPOefIy6oR0YuZhL6d9uhRIl9LMTB0XXzj5aW5ZbbGO6nsaERQUU3ALYzrwxgX8kLvCcAQmrVhhRH88Zqo4lkRkxXpN1LdPANh9qa0= coderanger@alfred 23 | richard: 24 | add_group: 25 | - psf-admin 26 | ssh_keys: 27 | - ssh-dss AAAAB3NzaC1kc3MAAACBAOBHkGxDowSpIqTioMKmYP8Vu0AXvcLZTsDETOWM4ak6LM8mxjgLWaZ3JoOES+tBJL26ygvugRzZNVfbM5xeN3s8OoYLnsXwJPn3FX9viQ5MSHX2fD9nu6MCNuxWjvhezMHHTWF/Hdop2uFxwizlnectGIwtGwj0s6wue7TZcsxFAAAAFQClEEy3n0dYb0JSHtWH1LA2PgY1LQAAAIAiG1xyHajn9/fRXRvflSltwG0RmIwf1e1Hmw9Fno3SUdWmPvLRWAc7uUSxXiifCRDlvfzhERMw07ra1hTye1GjPMxG144ooF1peCEEIsLBvv9LbsP+9+byR/yHg7TOeNjYpbEZLdaY/EG4T94cLp/y939MPI/+JZV5s1p69+UKPwAAAIEAhbOSvfz4ge03In/asws0YkS1VMranFGCAUKXqckVcEPbCMYxHAA3aEWLvNSXXWgSHl1zcvntfE1spZyHYpfC0Iif+0HE+zOF4TTsybbVHMqsyCd21U7H86CGWo0B5CdG9/zYcgfRx+++fltfr8jSRwGHz8WE1e36K2rEK+MD7ZI= richard@localhost 28 | - ssh-dss AAAAB3NzaC1kc3MAAACBAO/XIk/r+XOXy3w9+kDPtcoPa+pgtWtKAWVfQtTxBGzrGe9HEHZgvTaoBV0EPkT6WxhycS2SZoOWAG/yYtn7z9BMUrXVITSKXxjkZhQ5KIPLDhJ7GzRKHpRUVQOinIDJLo6AhcVd9LH3fsp3j7pDgTMylmqcTfaCjKTTCZVwo14XAAAAFQCBOydEbfYGOJXWIR8k0SbRJbL57QAAAIEAr3Mez9ACqpM/xraoEofQPZy+gSKq27fj3UuqKd3q9Xp5B21+8j8TU7u0GqpRIVdTGXEV2TJ+KRgAlxyTZKCheSfDRklOa7V3Iu+GZYFRxU2J5sVJ2hmLZX6JIZodBHJsCIZkZtV4/Qv+XBIkvYeFXUktTOlJLGn8dLrbp5/OYesAAACBAMxwv3wmJwabwnIvaFwR0trgSJ36lBOZjRqtxKGK2d2rp7JgGkUsXBbyefvn1H7uvJhT/gsQHLPGbPPt2HkJeQvTgXMhoEZgxDJSeMJeNEKoDVriX0/RO2ttWqXAnsZqD72tP2yIJnPYBB6b7vCKI0GUDO2Y2ADVdEMxNnRytXWc richard@ximinez.python.org 29 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIhQoc4+cQys7cZdhdzxCYjMN4sm3FKZNOwibX/4nN5ld0xWPVZ0TNByBzZAD8IEHWhyqKCLXNMhzsSoTXuRAD67ayEOEL4YjpNzOarcbdDQcCMN4RwvegK4ToHMCUaPjM/KT2Kg+GfhUdf5AtniqI2NeGOzUYiL3f0AnxCA6E9lcqpG4Vp2Tswtov474w4+Yq4oZmn1vWFq5i91OFr37HeuVwIuSY4FTL0XXGfAoL1n+Ge5vTp5vNv/nqINrykT1raWimhboFGOVg0nn26lAnFZFsCzUv7p/K9oORIb8aTGwDJzSOuEJ+PB4Z9/nC+2zUeRJqpxRKLWgIM5eH33bv rjones@mhs 30 | benjamin: 31 | add_group: 32 | - psf-admin 33 | ssh_keys: 34 | - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubCE0Iou9q1duI1FEDPCRMPBrT8+9X5iFOz5L/acwKa1BjzmCF2N91z4sox+H8VxN9kVkYcaizEm/4x91UDWo4oTjo+UOTHuoInrTAMYgq2EskSt1YA64PaPQ8xx0WuK+5LWvyPi0RTTXEtinriy0XCoGtXl/fDaYNjtB8V7JIqTqqyO0rAZExYBRbQed6ZpLeO8vGwc4cN/9xMmkbAoB0G4NekuGBCvvioBO5d2/MO93YdjRh/PzYNQSqycC7X6o7C4PoYy39aFc4fbDubl937MlG44R/8BAI9dKXsQu8hLqAm7XZcTdwR5kVAndXuwmzERznSZKldZmtyw9AvDWw== Benjamin Peterson 35 | apollo13: 36 | add_group: 37 | - psf-admin 38 | ssh_keys: 39 | - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCrtvKWmfzYa6o6dsk6iFDGeMaMnSDKNn9Bj9PedVTSzjhrg1j8wAnFdbgwFRo9atyiqmZcTbAlDyiyJjfRV3ajmfsCQbWljK49TfMzLH2MVl7/1Uze5z4cuJNaeIUTR18Uu49S4D6ChEW3Oqt2UyDA5iZgtd/MDrtUSLXyZ4N4h3C1wwKR03YkJO8eBPKIV149FLUL9yYakuOKX4pAiflIsz7sm1Qyij4hFY5vNo3qlMzgErhNPQwV/Hl6vP/TcXWfkhatusw2AUEIzqXGMQqtGu92kGPbRwd1vVtWoiqe+BFEDCDuGPohSWIOwAN6FoAwD9zylElQ9yKiNsoF2NMx7OhpOFA4qDKVGkES6QoakKJrwI0NyD97xefZYP5D8y3CWtobDrVaeYpj0gtqzJTHVkvWKtFlm0AZ44ELecW1a18jHCIT/fkHsbkivfn5pPSsHyKzR49IskvgJKQApy/JUV4qr9smzUHrpEThZyYh41/yzZnz/a5fqFhoE7ioPfzVeiPzYguiMz9WVV0FC9ByVJMI3P3UTpaRgXhIv2hxsmjs3opuYKrjrG7UC+Jh7XStgms8ZLWkGcqXE2chtCnqzDT1YbJHJgfZyfY9xMN9askT5pYLSPRYqdz9oMErBHCM/u61/1GUqtFBWVnqtV44+TPmgGUoQoXrCLDRcqegcQ== florian@apollo13 40 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/users/init.sls: -------------------------------------------------------------------------------- 1 | include: 2 | - users.admin 3 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/warehouse-deploys/pypi.sls: -------------------------------------------------------------------------------- 1 | 2 | warehouse-deploy-pypi: 3 | name: warehouse-pypi 4 | user: warehouse-pypi 5 | group: warehouse-pypi 6 | user_uid: 7000 7 | group_gid: 7000 8 | path: /opt/warehouse-pypi 9 | 10 | port: 9001 11 | 12 | secrets_key: secrets-warehouse-pypi 13 | 14 | source_uri: https://github.com/pypa/warehouse.git 15 | source_rev: werkzeug 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/warehouse-deploys/testpypi.sls: -------------------------------------------------------------------------------- 1 | 2 | warehouse-deploy-testpypi: 3 | name: warehouse-testpypi 4 | user: warehouse-testpypi 5 | group: warehouse-testpypi 6 | user_uid: 6666 7 | group_gid: 6666 8 | path: /opt/warehouse-testpypi 9 | 10 | port: 9000 11 | 12 | secrets_key: secrets-warehouse-testpypi 13 | 14 | source_uri: https://github.com/pypa/warehouse.git 15 | source_rev: werkzeug 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/prod-pillar/warehouse/web.sls: -------------------------------------------------------------------------------- 1 | 2 | firewall: 3 | http_test: 4 | port: 9000 5 | http_pypi: 6 | port: 9001 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/_grains/xenstore_datadog_tags.py: -------------------------------------------------------------------------------- 1 | import salt.utils 2 | import salt.modules.cmdmod 3 | 4 | __salt__ = { 5 | 'cmd.run_all': salt.modules.cmdmod._run_all_quiet, 6 | } 7 | 8 | KEYS = { 9 | 'provider_hostname': 'vm-data/hostname', 10 | 'provider': 'vm-data/provider_data/provider', 11 | 'region': 'vm-data/provider_data/region', 12 | } 13 | 14 | def xenstore(): 15 | data = dict() 16 | cmd = salt.utils.which('xenstore-read') 17 | if not cmd: 18 | return data 19 | try: 20 | command = '{0} domid'.format(cmd) 21 | ret = __salt__['cmd.run_all'](command) 22 | domid = ret['stdout'].rstrip() 23 | except Exception as e: 24 | return data 25 | 26 | found = {} 27 | for key, xkey in KEYS.iteritems(): 28 | try: 29 | command = '{0} /local/domain/{1}/{2}'.format(cmd, domid, xkey) 30 | ret = __salt__['cmd.run_all'](command) 31 | if ret and ret['retcode'] == 0: 32 | found[key] = ret['stdout'] 33 | except Exception: 34 | pass 35 | data['datadog_tags_from_metadata'] = ['%s:%s' % (k, v) for k, v in found.items()] 36 | return data 37 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/_modules/ip_picker.py: -------------------------------------------------------------------------------- 1 | import salt 2 | from salt.utils.network import in_subnet 3 | import socket 4 | import struct 5 | 6 | 7 | def __virtual__(): 8 | ''' 9 | Only work on POSIX-like systems 10 | ''' 11 | # Disable on Windows, a specific file module exists: 12 | if salt.utils.is_windows(): 13 | return False 14 | 15 | return True 16 | 17 | def ip_addrs(interface=None, include_loopback=False, cidr=None): 18 | ''' 19 | Returns a list of IPv4 addresses assigned to the host. 127.0.0.1 is 20 | ignored, unless 'include_loopback=True' is indicated. If 'interface' is 21 | provided, then only IP addresses from that interface will be returned. 22 | Providing a CIDR via 'cidr="10.0.0.0/8"' will return only the addresses 23 | which are within that subnet. 24 | 25 | CLI Example: 26 | 27 | .. code-block:: bash 28 | 29 | salt '*' network.ip_addrs 30 | ''' 31 | addrs = salt.utils.network.ip_addrs(interface=interface, 32 | include_loopback=include_loopback) 33 | if cidr: 34 | return sorted([i for i in addrs if in_subnet(cidr, [i])], 35 | key=lambda item: socket.inet_aton(item)) 36 | else: 37 | return addrs 38 | 39 | def interfaces_for_cidr(cidr='0.0.0.0/0'): 40 | ''' 41 | Return a dictionary of information about all the interfaces on the minion 42 | which have an address in the givin CIDR. 43 | 44 | CLI Example: 45 | 46 | .. code-block:: bash 47 | 48 | salt '*' network.interfaces_for_cidr cidr="192.168.1.1/24" 49 | ''' 50 | interfaces = salt.utils.network.interfaces() 51 | matched = [] 52 | for interface, data in interfaces.iteritems(): 53 | for net in data.get('inet'): 54 | if salt.utils.network.in_subnet(cidr, [net['address']]): 55 | matched.append(interface) 56 | return list(set(matched)) 57 | 58 | 59 | def subnet_mask_for_cidr(cidr="0.0.0.0/0"): 60 | """ 61 | Returns the ip address and subnet mask for a given CIDR. 62 | """ 63 | ip, cidr_mask = cidr.split("/") 64 | subnet_mask = socket.inet_ntoa( 65 | struct.pack(">I", (0xffffffff << (32 - int(cidr_mask))) & 0xffffffff) 66 | ) 67 | return {"address": ip, "subnet": subnet_mask} 68 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/auto-security/init.sls: -------------------------------------------------------------------------------- 1 | 2 | security_yum_extensions: 3 | pkg.installed: 4 | - pkgs: 5 | - yum-plugin-security 6 | - yum-cron 7 | 8 | yum_cron_security_only: 9 | file.replace: 10 | - name: /etc/sysconfig/yum-cron 11 | - path: /etc/sysconfig/yum-cron 12 | - pattern: '^YUM_PARAMETER=$' 13 | - repl: 'YUM_PARAMETER="--security"' 14 | - require: 15 | - pkg: security_yum_extensions 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/base.sls: -------------------------------------------------------------------------------- 1 | 2 | rdiff-backup: 3 | pkg.installed 4 | 5 | rsync: 6 | pkg.installed 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/client/README.md: -------------------------------------------------------------------------------- 1 | 2 | Format for pillar data: 3 | 4 | # Root Key, enables the state 5 | backup: 6 | # Dictionary of directories to backup 7 | directories: 8 | # A backup configuration 9 | postgres-archives: 10 | # Frequency of backup, currently {hourly, daily} are supported 11 | frequency: hourly 12 | # User to run backup as 13 | user: devpypi 14 | # Source Directory to backup 15 | source_directory: /var/lib/pgsql/9.3/backup/archives 16 | # Target backup server 17 | target_host: 172.16.57.201 18 | # Target directory on backup server 19 | target_directory: /backup/postgres/archives 20 | # Target user on backup server 21 | target_user: devpypi 22 | # SSH Private Key for backup server access as target_user 23 | ssh_key: | 24 | -----BEGIN RSA PRIVATE KEY----- 25 | MIIEowIBAAKCAQEAsagAYbuOiROc0+vLjcKQXZqiP3mH3qXwPT1PAMWxkferuaz3 26 | ... 27 | pqMI3F6ButHsMcwjZotivkf3baM1FmnjIJ4oeTFPyagaLZdRCuiT 28 | -----END RSA PRIVATE KEY----- 29 | # Backup example with pre/post/cleanup scripts 30 | postgres-base: 31 | frequency: daily 32 | user: postgres 33 | source_directory: /var/lib/pgsql/9.3/backups/base 34 | target_host: 172.16.57.201 35 | target_directory: /backup/postgres/base 36 | target_user: postgres 37 | # Script to run before rdiff-backup command 38 | pre_script: 'pg_basebackup -D /var/lib/pgsql/9.3/backups/base/$(date --iso-8601=seconds)' 39 | # Script to run after rdiff-backup command 40 | post_script: '/usr/local/backup/postgres-archives/scripts/backup.bash' 41 | # Cleanup script to remove old backups 42 | cleanup_script: 'find /var/lib/pgsql/9.3/backups/base -maxdepth 1 -type d -mtime +7 -execdir rm -rf {} \;' 43 | ssh_key: | 44 | -----BEGIN RSA PRIVATE KEY----- 45 | MIIEowIBAAKCAQEAsagAYbuOiROc0+vLjcKQXZqiP3mH3qXwPT1PAMWxkferuaz3 46 | ... 47 | pqMI3F6ButHsMcwjZotivkf3baM1FmnjIJ4oeTFPyagaLZdRCuiT 48 | -----END RSA PRIVATE KEY----- 49 | 50 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/client/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - backup.base 4 | 5 | /etc/backup: 6 | file.directory: 7 | - user: root 8 | - group: root 9 | - mode: 755 10 | 11 | /etc/backup/.ssh: 12 | file.directory: 13 | - user: root 14 | - group: root 15 | - mode: 755 16 | 17 | {% for backup, config in salt['pillar.get']('backup:directories', {}).iteritems() %} 18 | 19 | {{ backup }}-ssh-key: 20 | file.managed: 21 | - name: /etc/backup/.ssh/id_rsa_{{ backup }} 22 | - contents_pillar: backup:directories:{{ backup }}:ssh_key 23 | - user: {{ config['user'] }} 24 | - mode: 600 25 | 26 | {{ backup }}-script-dir: 27 | file.directory: 28 | - name: /usr/local/backup/{{ backup }}/scripts 29 | - makedirs: True 30 | 31 | {{ backup }}-script: 32 | file.managed: 33 | - name: /usr/local/backup/{{ backup }}/scripts/backup.bash 34 | - user: {{ config['user'] }} 35 | - mode: 500 36 | - source: salt://backup/client/templates/backup.bash.jinja 37 | - template: jinja 38 | - context: 39 | pre_script: '{{ config.get('pre_script', ":") }}' 40 | remote_command: '/usr/bin/rdiff-backup --no-eas --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup --server" {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}' 41 | post_script: '{{ config.get('post_script', ":") }}' 42 | cleanup_script: '{{ config.get('cleanup_script', ":") }}' 43 | 44 | {{ backup }}-cron: 45 | file.managed: 46 | - name: /etc/cron.d/{{ backup }}-backup 47 | - template: jinja 48 | - source: salt://backup/client/templates/cron.jinja 49 | - context: 50 | job_frequency: {{ config['frequency'] }} 51 | job_user: {{ config['user'] }} 52 | job_command: /usr/local/backup/{{ backup }}/scripts/backup.bash 53 | 54 | {% endfor %} 55 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/client/templates/backup.bash.jinja: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec 200<$0 4 | flock -n 200 || exit 1 5 | 6 | {{ pre_script }} 7 | {{ remote_command }} 8 | {{ post_script }} 9 | 10 | {{ cleanup_script }} 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/client/templates/cron.jinja: -------------------------------------------------------------------------------- 1 | {% if job_frequency == 'hourly' %} 2 | {% set cron = '0 * * * *' %} 3 | {% elif job_frequency == 'daily' %} 4 | {% set cron = '0 0 * * *' %} 5 | {% else %} 6 | {% set cron = '0 * * * *' %} 7 | {% endif %} 8 | {{ cron }} {{ job_user }} {{ job_command }} 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/server/README.md: -------------------------------------------------------------------------------- 1 | Format for pillar data: 2 | 3 | # Root Key, enables the state 4 | backup-server: 5 | # Volumes to format and mount 6 | volumes: 7 | # device mount point 8 | /dev/sdb: /backup 9 | # Dictionary of directories for backup clients 10 | directories: 11 | # Directory for backup client 12 | /backup/postgres/archives: 13 | # Retention Period for backup increments (see rdiff-backup --remove-older-than) 14 | increment_retention: 10d 15 | # User the client can access the backup server as 16 | user: postgres 17 | # Authorized Key for client access 18 | authorized_key: ssh-rsa AAAAB3NzaC1y.. ...CXVxa6LHKJB6RDT3eYyMQSugFSCrHxQ8j/F 19 | 20 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/server/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - backup.base 4 | 5 | {% for volume, mount in salt['pillar.get']('backup-server:volumes').iteritems() %} 6 | {{ volume }}-mkfs: 7 | cmd.run: 8 | - name: 'yes y | mkfs.ext4 {{ volume }}' 9 | - unless: 'file -s {{ volume }} | grep ext4' 10 | 11 | {{ mount }}: 12 | mount.mounted: 13 | - device: {{ volume }} 14 | - fstype: ext4 15 | - mkmnt: True 16 | - opts: 17 | - defaults 18 | {% endfor %} 19 | 20 | {% for backup, config in salt['pillar.get']('backup-server:backups').iteritems() %} 21 | 22 | {{ backup }}-user: 23 | user.present: 24 | - name: {{ config['user'] }} 25 | 26 | {{ backup }}-ssh: 27 | ssh_auth: 28 | - present 29 | - user: {{ config['user'] }} 30 | - names: 31 | - {{ config['authorized_key'] }} 32 | - options: 33 | - command="rdiff-backup --server" 34 | - no-pty 35 | - no-port-forwarding 36 | - no-agent-forwarding 37 | - no-X11-forwarding 38 | - require: 39 | - user: {{ config['user'] }} 40 | 41 | {{ backup }}: 42 | file.directory: 43 | - name: {{ config['directory'] }} 44 | - user: {{ config['user'] }} 45 | - makedirs: True 46 | - require: 47 | - user: {{ config['user'] }} 48 | 49 | {{ backup }}-increment-cleanup: 50 | file.managed: 51 | - name: /etc/cron.d/{{ backup }}-backup-cleanup 52 | - user: root 53 | - group: root 54 | - template: jinja 55 | - source: salt://backup/server/templates/cron.jinja 56 | - context: 57 | cron: '0 3 * * *' 58 | job_user: root 59 | job_command: 'rdiff-backup --force --remove-older-than {{ config['increment_retention'] }} {{ config['directory'] }}' 60 | 61 | {% endfor %} 62 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/backup/server/templates/cron.jinja: -------------------------------------------------------------------------------- 1 | {{ cron }} {{ job_user }} {{ job_command }} 2 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/base/auto-highstate.sls: -------------------------------------------------------------------------------- 1 | 2 | 15m-interval-highstate: 3 | cron.present: 4 | - name: salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 5 | - minute: '*/15' 6 | 7 | /etc/logrotate.d/salt: 8 | file.managed: 9 | - source: salt://base/config/salt-logrotate.conf 10 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/base/config/salt-logrotate.conf: -------------------------------------------------------------------------------- 1 | /var/log/salt/* { 2 | daily 3 | rotate 7 4 | missingok 5 | notifempty 6 | compress 7 | delaycompress 8 | } 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/base/sanity.sls: -------------------------------------------------------------------------------- 1 | 2 | niceties: 3 | pkg.installed: 4 | - pkgs: 5 | - htop 6 | - tmux 7 | - tree 8 | - bash-completion 9 | - vim-enhanced 10 | 11 | time-sync: 12 | pkg.installed: 13 | - pkgs: 14 | - ntp 15 | - ntpdate 16 | 17 | ntpd: 18 | service: 19 | - running 20 | - enable: True 21 | 22 | /etc/profile.d/yum.sh: 23 | file.absent 24 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/datadog/config/RPM-GPG-KEY-DATADOG: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Version: GnuPG v1 3 | 4 | mQGiBFXdFPYRBAD5s5lh5teo4g84NEFWhb7HuVpuaZVRhIiYkrxM2vh8yEXJPP5f 5 | Ay9W2wYaO4eJpOiS1zm9lQ9iEwsSufJtkzhgeL9FWVt2pNa3Ev/8h7I25TVOOww5 6 | K+Jb/n276cAly2dEOafifwow1MrBWQ2hkCmQOGVxSMth95+hTPdcsInEKwCg+tFp 7 | diYlo5ROP0qTE1WfLILkwpsD/0WWrN5PjO2wk00Ze7DrEhwpUcJOk3UeVeNZft1O 8 | cUXGwHyKLe8rSHWjyUjL70msSYdBVFIhT0D/edZ8+Stusu1Wg+eOz76cRD+3L7bk 9 | SkkRKhjvykRiK1p5mg4O5/KGhMRS6oVTyoRJCxJx/rEnEKoV/lTgiNBRXSOJZYJo 10 | QnEYBACjTp0gVzoppS4HAzo/LydDZVHS8bQLAlgMvvc8gvHvbvp2LLGVzgkOQhc9 11 | zrE4IpdonMJcuT+Y494qgO4R9Erxo+rMIblEyk5TXlNSrKnJZ7hGULJXmUjtIGon 12 | dt5GWYDyw7wEf3U3GyxrO+fT2yLY441oJaAT+AIQf85XH/v7C7QoRGF0YWRvZyBQ 13 | YWNrYWdlcyA8cGFja2FnZUBkYXRhZG9naHEuY29tPohiBBMRAgAiBQJV3RT2AhsD 14 | BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAGm1b1QXKiMJemAKDc8pVqUQZi 15 | bJ42iSumpysXd4RhfQCdHnf39vCylNbLT8wcLL3rFUOJtVu5AQ0EVd0U9hAEAJbK 16 | uVM2j4i+in3azrkjeE0JbHImWkj0i9nGt2U+ivkTKYvUS8Z7NvOLvkfp41HzeC1c 17 | yQ+6MuhJQsvazGwBTNcq08XKpnVc7UwsZ0G3e+faGCKmhnFIMiYFtzcOkdim2UPu 18 | 1JztJaz+BMVY0jRAaT5icLtAK1HGLmTUsq1VHEnLAAMFA/9BmhW80bFjm0u/F+xE 19 | knXZZ4AG4ajzPBcodzbRRnKnDh2e97XvE5euiQUz0kLolRk5gQHWCybRQo5BR1Hv 20 | teerNmsopW9k8s0dFA3FUidiwsePb+UUwEeTzmqNjWFJ72ktXkaHBmDR+rhrrSmv 21 | bIsP1/Y0wO6jhtNPUzS7gAmKdIhJBBgRAgAJBQJV3RT2AhsMAAoJEAabVvVBcqIw 22 | jV4AoLxp7gWBrU+FARqEuajGuz4ZDRwcAKDANItJowyw3QXAizU2m7jiyJgkzw== 23 | =rswC 24 | -----END PGP PUBLIC KEY BLOCK----- 25 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/datadog/files/datadog-nginx-status-codes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Lifted from https://gist.github.com/technovangelist/6090e4a66bc60b99c566 3 | """ 4 | 5 | from datetime import datetime 6 | import time 7 | import re 8 | 9 | METRIC_TYPES = { 10 | '5': 'nginx.net.5xx_status', 11 | '4': 'nginx.net.4xx_status', 12 | '3': 'nginx.net.3xx_status', 13 | '2': 'nginx.net.2xx_status', 14 | '1': 'nginx.net.1xx_status' 15 | } 16 | 17 | def parse(log, line): 18 | if len(line) == 0: 19 | log.info("Skipping empty line") 20 | return None 21 | timestamp = getTimestamp(line) 22 | status = line.split()[8] 23 | objToReturn = [] 24 | try: 25 | objToReturn.append((METRIC_TYPES[status[0]], timestamp, 1, {'metric_type': 'counter'})) 26 | except KeyError: 27 | pass 28 | return objToReturn 29 | 30 | def getTimestamp(line): 31 | line_parts = line.split() 32 | dt = line_parts[3] 33 | date = datetime.strptime(dt, "[%d/%b/%Y:%H:%M:%S") 34 | date = time.mktime(date.timetuple()) 35 | return date 36 | 37 | if __name__ == "__main__": 38 | import sys 39 | import pprint 40 | import logging 41 | logging.basicConfig() 42 | log = logging.getLogger() 43 | lines = open(sys.argv[1]).readlines() 44 | pprint.pprint([parse(log, line) for line in lines]) 45 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/datadog/init.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/pki/rpm-gpg/RPM-GPG-KEY-DATADOG: 3 | file.managed: 4 | - source: salt://datadog/config/RPM-GPG-KEY-DATADOG 5 | - user: root 6 | - group: root 7 | - mode: 444 8 | 9 | datadog_repo: 10 | pkgrepo.managed: 11 | - humanname: Datadog, Inc. 12 | - baseurl: https://yum.datadoghq.com/rpm/$basearch/ 13 | - gpgcheck: 1 14 | - gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-DATADOG 15 | - require: 16 | - file: /etc/pki/rpm-gpg/RPM-GPG-KEY-DATADOG 17 | 18 | {% set in_datadog_tags = pillar.get('datadog_tags', []) + grains.get('datadog_tags', []) + grains.get('datadog_tags_from_metadata', []) %} 19 | {% set datadog_tags = [] %} 20 | {% for tag in in_datadog_tags if tag not in datadog_tags %} 21 | {% do datadog_tags.append(tag) %} 22 | {% endfor %} 23 | 24 | {% set dogstreams = pillar.get('datadog_dogstreams') %} 25 | 26 | /usr/share/datadog: 27 | file.recurse: 28 | - source: salt://datadog/files 29 | 30 | {% if 'datadog_api_key' in pillar %} 31 | datadog-agent: 32 | pkg: 33 | - installed 34 | - require: 35 | - pkgrepo: datadog_repo 36 | service: 37 | - running 38 | - enable: True 39 | - require: 40 | - ini: /etc/dd-agent/datadog.conf 41 | - pkg: datadog-agent 42 | - watch: 43 | - ini: /etc/dd-agent/datadog.conf 44 | 45 | /etc/dd-agent/datadog.conf: 46 | file.managed: 47 | - user: root 48 | - group: root 49 | - mode: 0644 50 | - replace: False 51 | ini.options_present: 52 | - sections: 53 | Main: 54 | dd_url: https://app.datadoghq.com 55 | api_key: {{ pillar.get('datadog_api_key') }} 56 | tags: {{ datadog_tags|join(", ") }} 57 | use_dogstatsd: yes 58 | dogstatsd_port: 18125 59 | dogstreams: "{{ dogstreams|join(", ") if dogstreams else ''}}" 60 | - require: 61 | - file: /etc/dd-agent/datadog.conf 62 | 63 | {% endif %} 64 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/elasticsearch/init.sls: -------------------------------------------------------------------------------- 1 | java-1.7.0-openjdk: 2 | pkg.installed 3 | 4 | elasticsearch: 5 | pkg.installed: 6 | - sources: 7 | - elasticsearch: https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.5.2.noarch.rpm 8 | - require: 9 | - pkg: java-1.7.0-openjdk 10 | service.running: 11 | - require: 12 | - pkg: elasticsearch 13 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/firewall/config/iptables.jinja: -------------------------------------------------------------------------------- 1 | # Firewall configuration written by salt 2 | # Manual customization of this file is not cool. 3 | 4 | {% set rules = salt['pillar.get']('firewall', {}) %} 5 | 6 | *filter 7 | :INPUT ACCEPT [0:0] 8 | :FORWARD ACCEPT [0:0] 9 | :OUTPUT ACCEPT [0:0] 10 | -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT 11 | -A INPUT -p icmp -j ACCEPT 12 | -A INPUT -i lo -j ACCEPT 13 | -A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT 14 | {% for rule, config in rules.iteritems() -%} 15 | {% set port = config.get('port', 22) -%} 16 | {% set proto = config.get('protocol', 'tcp') -%} 17 | {% set source = config.get('source', '0.0.0.0/0') -%} 18 | # {{ rule }} 19 | -A INPUT -m state --state NEW -m {{ proto }} -p {{ proto }} -s {{ source }} --dport {{ port }} -j ACCEPT 20 | {% endfor -%} 21 | -A INPUT -j REJECT --reject-with icmp-host-prohibited 22 | -A FORWARD -j REJECT --reject-with icmp-host-prohibited 23 | COMMIT 24 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/firewall/init.sls: -------------------------------------------------------------------------------- 1 | /etc/sysconfig/iptables: 2 | file.managed: 3 | - source: salt://firewall/config/iptables.jinja 4 | - user: root 5 | - group: root 6 | - mode: 600 7 | - template: jinja 8 | 9 | iptables: 10 | service: 11 | - running 12 | - watch: 13 | - file: /etc/sysconfig/iptables 14 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/base.sls: -------------------------------------------------------------------------------- 1 | 2 | {% set client_config = pillar.get('monitoring_client', {}) %} 3 | {% set bucky_config = pillar.get('bucky', {}) %} 4 | 5 | include: 6 | - supervisor 7 | 8 | /etc/bucky/bucky.conf: 9 | file.managed: 10 | - source: salt://monitoring/client/config/bucky.conf.jinja 11 | - template: jinja 12 | - context: 13 | graphite_ip: {{ bucky_config.get('graphite_ip', '127.0.0.1') }} 14 | graphite_port: {{ bucky_config.get('graphite_port', 2003) }} 15 | graphite_max_reconnects: {{ bucky_config.get('graphite_max_reconnects', 3) }} 16 | graphite_reconnect_delay: {{ bucky_config.get('graphite_reconnect_delay', 5) }} 17 | 18 | python-bucky: 19 | pkg.installed 20 | 21 | /etc/supervisord.d/bucky.ini: 22 | file.managed: 23 | - source: salt://monitoring/client/config/bucky.ini 24 | 25 | bucky-supervisor: 26 | cmd.wait: 27 | - name: supervisorctl reread && supervisorctl update 28 | - require: 29 | - file: /etc/supervisord.d/bucky.ini 30 | - file: /etc/bucky/bucky.conf 31 | - pkg: python-bucky 32 | - watch: 33 | - file: /etc/supervisord.d/bucky.ini 34 | 35 | /etc/collectd.conf: 36 | file.managed: 37 | - source: salt://monitoring/client/config/collectd.conf.jinja 38 | - template: jinja 39 | - context: 40 | hostname: {{ grains['fqdn'] }} 41 | collectd_host: {{ client_config.get('collectd_host', '127.0.0.1') }} 42 | collectd_port: {{ client_config.get('collectd_port', 25826) }} 43 | 44 | /usr/local/lib/collectd/plugins: 45 | file.recurse: 46 | - source: salt://monitoring/client/plugins 47 | 48 | /etc/collectd.d/base.conf: 49 | file.managed: 50 | - source: salt://monitoring/client/config/collectd.d/base.conf 51 | 52 | collectd: 53 | pkg: 54 | - installed 55 | service: 56 | - running 57 | - enable: True 58 | - restart: True 59 | - watch: 60 | - file: /etc/collectd.conf 61 | - file: /etc/collectd.d/* 62 | - require: 63 | - file: /etc/collectd.conf 64 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/bucky.conf.jinja: -------------------------------------------------------------------------------- 1 | debug = False 2 | log_level = "INFO" 3 | full_trace = False 4 | 5 | graphite_ip = "{{ graphite_ip }}" 6 | graphite_port = {{ graphite_port }} 7 | graphite_max_reconnects = {{ graphite_max_reconnects }} 8 | graphite_reconnect_delay = {{ graphite_reconnect_delay }} 9 | 10 | name_prefix = None 11 | name_postfix = None 12 | name_replace_char = '_' 13 | name_strip_duplicates = True 14 | name_host_trim = [] 15 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/bucky.ini: -------------------------------------------------------------------------------- 1 | [program:bucky] 2 | command = /usr/bin/bucky /etc/bucky/bucky.conf 3 | autostart = true 4 | autorestart = true 5 | user = bucky 6 | log_stdout = true 7 | log_stderr = true 8 | logfile = /var/log/bucky/bucky.log 9 | environment = HOME=/,USER=bucky 10 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.conf.jinja: -------------------------------------------------------------------------------- 1 | Hostname {{ hostname }} 2 | LoadPlugin syslog 3 | LoadPlugin network 4 | 5 | Server "{{ collectd_host }} {% if collectd_port %}{{ collectd_port }}{% endif %}" 6 | 7 | Include "/etc/collectd.d/*.conf" 8 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/base.conf: -------------------------------------------------------------------------------- 1 | LoadPlugin cpu 2 | LoadPlugin disk 3 | LoadPlugin df 4 | LoadPlugin interface 5 | LoadPlugin load 6 | LoadPlugin memory 7 | LoadPlugin swap 8 | 9 | Disk "/^[hs]?d[a-z][a-z]??$/" 10 | Disk "/^xvd[a-z][a-z]??$/" 11 | IgnoreSelected false 12 | 13 | 14 | ReportReserved true 15 | ReportInodes true 16 | 17 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/nginx.conf: -------------------------------------------------------------------------------- 1 | LoadPlugin nginx 2 | 3 | 4 | URL "http://localhost/nginx_status?auto" 5 | 6 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/postgresql.conf.jinja: -------------------------------------------------------------------------------- 1 | LoadPlugin postgresql 2 | 3 | 4 | 5 | 6 | Statement "SELECT pg_xlog_location_diff(pg_current_xlog_location(), '0/0') AS offset;" 7 | 8 | Type gauge 9 | InstancePrefix "master_xlog_postition" 10 | ValuesFrom offset 11 | 12 | 13 | 14 | 15 | Statement "SELECT pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0') AS receive;" 16 | 17 | Type gauge 18 | InstancePrefix "standby_xlog_rec" 19 | ValuesFrom receive 20 | 21 | 22 | 23 | 24 | Statement "SELECT pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0') AS replay;" 25 | 26 | Type gauge 27 | InstancePrefix "standby_xlog_rep" 28 | ValuesFrom replay 29 | 30 | 31 | 32 | 33 | Statement "SELECT EXTRACT(EPOCH FROM (SELECT NOW() - pg_last_xact_replay_timestamp())) AS replication_delay;" 34 | 35 | Type gauge 36 | InstancePrefix "standby_approx_lag" 37 | ValuesFrom replication_delay 38 | 39 | 40 | 41 | 42 | Statement "SELECT COUNT(*) as connection_count FROM pg_stat_activity;" 43 | 44 | Type gauge 45 | InstancePrefix "connection_count" 46 | ValuesFrom connection_count 47 | 48 | 49 | 50 | 51 | Host "{{ monitor_host }}" 52 | Port "{{ monitor_port }}" 53 | User "{{ monitor_user }}" 54 | Password "{{ monitor_password }}" 55 | {% if 'primary' in grains['roles'] %} 56 | Query master_xlog_postition 57 | {% endif %} 58 | {% if 'standby' in grains['roles'] %} 59 | Query standby_xlog_rec 60 | Query standby_xlog_rep 61 | Query standby_approx_lag 62 | {% endif %} 63 | Query connections 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/pypi-backend.conf: -------------------------------------------------------------------------------- 1 | 2 | Globals true 3 | 4 | 5 | 6 | ModulePath "/usr/local/lib/collectd/plugins/python" 7 | Import "pypi_backend" 8 | 9 | 10 | Host "127.0.0.1" 11 | Port "9000" 12 | 13 | 14 | 15 | 16 | ModulePath "/usr/local/lib/collectd/plugins/python" 17 | Import "uwsgi_stats" 18 | 19 | 20 | SocketPath "/var/run/pypi/pypi-stats.sock" 21 | 22 | 23 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/pypi-mirror.conf: -------------------------------------------------------------------------------- 1 | 2 | Globals true 3 | 4 | 5 | 6 | ModulePath "/usr/local/lib/collectd/plugins/python" 7 | Import "pypi_mirror" 8 | 9 | 10 | MirrorDir "/data/pypi-mirror" 11 | 12 | 13 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/config/collectd.d/redis.conf: -------------------------------------------------------------------------------- 1 | 2 | Globals true 3 | 4 | 5 | 6 | ModulePath "/usr/local/lib/collectd/plugins/python" 7 | Import "redis_info" 8 | 9 | 10 | Host "localhost" 11 | Port 6379 12 | Verbose false 13 | 14 | 15 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - monitoring.client.base 4 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/nginx.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/collectd.d/nginx.conf: 3 | file.managed: 4 | - source: salt://monitoring/client/config/collectd.d/nginx.conf 5 | 6 | collectd-nginx: 7 | pkg: 8 | - installed 9 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/plugins/python/pypi_backend.py: -------------------------------------------------------------------------------- 1 | import os 2 | import calendar 3 | from datetime import datetime 4 | import urllib2 5 | 6 | import collectd 7 | 8 | # Mirror Directory to read. Override in config by specifying 'MirrorDir'. 9 | HOST = 'localhost' 10 | PORT = 9000 11 | 12 | def configure_callback(conf): 13 | global HOST, PORT 14 | for node in conf.children: 15 | if node.key == 'Host': 16 | HOST = node.values[0] 17 | if node.key == 'Port': 18 | PORT = int(node.values[0]) 19 | else: 20 | collectd.warning('pypi_backend plugin: Unknown config key: %s.' 21 | % node.key) 22 | 23 | def read_callback(): 24 | SERIAL_URL = "https://%s:%s/serial" % (HOST, PORT) 25 | TIME_URL = "https://%s:%s/daytime" % (HOST, PORT) 26 | 27 | handler = urllib2.urlopen(SERIAL_URL) 28 | current_serial = handler.read().rstrip('\n') 29 | 30 | handler = urllib2.urlopen(TIME_URL) 31 | last_modified_timestamp = handler.read().rstrip('\n') 32 | 33 | cur_serial = collectd.Values(type='gauge') 34 | cur_serial.plugin='pypi_backend.current_serial' 35 | cur_serial.dispatch(values=[int(current_serial)]) 36 | 37 | last_modified_unix = calendar.timegm(datetime.strptime(last_modified_timestamp, "%Y%m%dT%H:%M:%S").utctimetuple()) 38 | last_mod = collectd.Values(type='gauge') 39 | last_mod.plugin='pypi_backend.last_modified' 40 | last_mod.dispatch(values=[int(last_modified_unix)]) 41 | 42 | # register callbacks 43 | collectd.register_config(configure_callback) 44 | collectd.register_read(read_callback) 45 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/plugins/python/pypi_mirror.py: -------------------------------------------------------------------------------- 1 | import os 2 | import calendar 3 | from datetime import datetime 4 | 5 | import collectd 6 | 7 | # Mirror Directory to read. Override in config by specifying 'MirrorDir'. 8 | MIRROR_DIR = '/data/pypi-mirror' 9 | 10 | def configure_callback(conf): 11 | global MIRROR_DIR 12 | for node in conf.children: 13 | if node.key == 'MirrorDir': 14 | MIRROR_DIR = node.values[0] 15 | else: 16 | collectd.warning('pypi_mirror plugin: Unknown config key: %s.' 17 | % node.key) 18 | 19 | def read_callback(): 20 | with open(os.path.join(MIRROR_DIR, 'status'), 'r') as f: 21 | current_serial = f.read() 22 | 23 | with open(os.path.join(MIRROR_DIR, 'web', 'last-modified'), 'r') as f: 24 | last_modified_timestamp = f.read().rstrip('\n') 25 | 26 | cur_serial = collectd.Values(type='gauge') 27 | cur_serial.plugin='pypi_mirror.current_serial' 28 | cur_serial.dispatch(values=[int(current_serial)]) 29 | 30 | last_modified_unix = calendar.timegm(datetime.strptime(last_modified_timestamp, "%Y%m%dT%H:%M:%S").utctimetuple()) 31 | last_mod = collectd.Values(type='gauge') 32 | last_mod.plugin='pypi_mirror.last_modified' 33 | last_mod.dispatch(values=[int(last_modified_unix)]) 34 | 35 | # register callbacks 36 | collectd.register_config(configure_callback) 37 | collectd.register_read(read_callback) 38 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/plugins/python/redis_info.py: -------------------------------------------------------------------------------- 1 | # redis-collectd-plugin - redis_info.py 2 | # 3 | # This program is free software; you can redistribute it and/or modify it 4 | # under the terms of the GNU General Public License as published by the 5 | # Free Software Foundation; only version 2 of the License is applicable. 6 | # 7 | # This program is distributed in the hope that it will be useful, but 8 | # WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | # General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License along 13 | # with this program; if not, write to the Free Software Foundation, Inc., 14 | # 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 15 | # 16 | # Authors: 17 | # Garret Heaton 18 | # 19 | # About this plugin: 20 | # This plugin uses collectd's Python plugin to record Redis information. 21 | # 22 | # collectd: 23 | # http://collectd.org 24 | # Redis: 25 | # http://redis.googlecode.com 26 | # collectd-python: 27 | # http://collectd.org/documentation/manpages/collectd-python.5.shtml 28 | 29 | import collectd 30 | import socket 31 | 32 | 33 | # Host to connect to. Override in config by specifying 'Host'. 34 | REDIS_HOST = 'localhost' 35 | 36 | # Port to connect on. Override in config by specifying 'Port'. 37 | REDIS_PORT = 6379 38 | 39 | # Verbose logging on/off. Override in config by specifying 'Verbose'. 40 | VERBOSE_LOGGING = False 41 | 42 | 43 | def fetch_info(): 44 | """Connect to Redis server and request info""" 45 | try: 46 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 47 | s.connect((REDIS_HOST, REDIS_PORT)) 48 | log_verbose('Connected to Redis at %s:%s' % (REDIS_HOST, REDIS_PORT)) 49 | except socket.error, e: 50 | collectd.error('redis_info plugin: Error connecting to %s:%d - %r' 51 | % (REDIS_HOST, REDIS_PORT, e)) 52 | return None 53 | fp = s.makefile('r') 54 | log_verbose('Sending info command') 55 | s.sendall('info\r\n') 56 | 57 | status_line = fp.readline() 58 | content_length = int(status_line[1:-1]) # status_line looks like: $ 59 | data = fp.read(content_length) 60 | log_verbose('Received data: %s' % data) 61 | s.close() 62 | 63 | linesep = '\r\n' if '\r\n' in data else '\n' 64 | return parse_info(data.split(linesep)) 65 | 66 | 67 | def parse_info(info_lines): 68 | """Parse info response from Redis""" 69 | info = {} 70 | for line in info_lines: 71 | if "" == line or line.startswith('#'): 72 | continue 73 | 74 | if ':' not in line: 75 | collectd.warning('redis_info plugin: Bad format for info line: %s' 76 | % line) 77 | continue 78 | 79 | key, val = line.split(':') 80 | 81 | # Handle multi-value keys (for dbs). 82 | # db lines look like "db0:keys=10,expire=0" 83 | if ',' in val: 84 | split_val = val.split(',') 85 | val = {} 86 | for sub_val in split_val: 87 | k, _, v = sub_val.rpartition('=') 88 | val[k] = v 89 | 90 | info[key] = val 91 | info["changes_since_last_save"] = info.get("changes_since_last_save", info.get("rdb_changes_since_last_save")) 92 | return info 93 | 94 | 95 | def configure_callback(conf): 96 | """Receive configuration block""" 97 | global REDIS_HOST, REDIS_PORT, VERBOSE_LOGGING 98 | for node in conf.children: 99 | if node.key == 'Host': 100 | REDIS_HOST = node.values[0] 101 | elif node.key == 'Port': 102 | REDIS_PORT = int(node.values[0]) 103 | elif node.key == 'Verbose': 104 | VERBOSE_LOGGING = bool(node.values[0]) 105 | else: 106 | collectd.warning('redis_info plugin: Unknown config key: %s.' 107 | % node.key) 108 | log_verbose('Configured with host=%s, port=%s' % (REDIS_HOST, REDIS_PORT)) 109 | 110 | 111 | def dispatch_value(info, key, type, type_instance=None): 112 | """Read a key from info response data and dispatch a value""" 113 | if key not in info: 114 | collectd.warning('redis_info plugin: Info key not found: %s' % key) 115 | return 116 | 117 | if not type_instance: 118 | type_instance = key 119 | 120 | value = int(info[key]) 121 | log_verbose('Sending value: %s=%s' % (type_instance, value)) 122 | 123 | val = collectd.Values(plugin='redis_info') 124 | val.type = type 125 | val.type_instance = type_instance 126 | val.values = [value] 127 | val.dispatch() 128 | 129 | 130 | def read_callback(): 131 | log_verbose('Read callback called') 132 | info = fetch_info() 133 | 134 | if not info: 135 | collectd.error('redis plugin: No info received') 136 | return 137 | 138 | # send high-level values 139 | dispatch_value(info, 'uptime_in_seconds','gauge') 140 | dispatch_value(info, 'connected_clients', 'gauge') 141 | dispatch_value(info, 'connected_slaves', 'gauge') 142 | dispatch_value(info, 'blocked_clients', 'gauge') 143 | dispatch_value(info, 'evicted_keys', 'gauge') 144 | dispatch_value(info, 'used_memory', 'bytes') 145 | dispatch_value(info, 'used_memory_peak', 'bytes') 146 | dispatch_value(info, 'changes_since_last_save', 'gauge') 147 | dispatch_value(info, 'total_connections_received', 'counter', 148 | 'connections_received') 149 | dispatch_value(info, 'total_commands_processed', 'counter', 150 | 'commands_processed') 151 | 152 | # database and vm stats 153 | for key in info: 154 | if key.startswith('vm_stats_'): 155 | dispatch_value(info, key, 'gauge') 156 | if key.startswith('db'): 157 | dispatch_value(info[key], 'keys', 'gauge', '%s-keys' % key) 158 | 159 | 160 | def log_verbose(msg): 161 | if not VERBOSE_LOGGING: 162 | return 163 | collectd.info('redis plugin [verbose]: %s' % msg) 164 | 165 | 166 | # register callbacks 167 | collectd.register_config(configure_callback) 168 | collectd.register_read(read_callback) 169 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/plugins/python/uwsgi_stats.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import socket 4 | 5 | import collectd 6 | 7 | # uWSGI stat sock to read. Override in config by specifying 'SocketPath'. 8 | SOCK = '/var/run/pypi/pypi-stats.sock' 9 | 10 | def configure_callback(conf): 11 | global SOCK 12 | for node in conf.children: 13 | if node.key == 'SocketPath': 14 | SOCK = node.values[0] 15 | else: 16 | collectd.warning('uwsgi_stats plugin: Unknown config key: %s.' 17 | % node.key) 18 | 19 | def read_callback(): 20 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 21 | s.connect(SOCK) 22 | 23 | js = '' 24 | while True: 25 | data = s.recv(4096) 26 | if len(data) < 1: 27 | break 28 | js += data.decode('utf8') 29 | 30 | s.close() 31 | 32 | dd = json.loads(js) 33 | 34 | worker_statuses = [w['status'] for w in dd['workers']] 35 | counter = {'total': len(worker_statuses), 'idle': 0, 'busy': 0} 36 | for status in worker_statuses: 37 | counter[status] += 1 38 | 39 | idle = collectd.Values(type='gauge') 40 | idle.plugin='pypi_backend.uwsgi.idle' 41 | idle.dispatch(values=[int(counter['idle'])]) 42 | 43 | busy = collectd.Values(type='gauge') 44 | busy.plugin='pypi_backend.uwsgi.busy' 45 | busy.dispatch(values=[int(counter['busy'])]) 46 | 47 | total = collectd.Values(type='gauge') 48 | total.plugin='pypi_backend.uwsgi.total' 49 | total.dispatch(values=[int(counter['total'])]) 50 | 51 | # register callbacks 52 | collectd.register_config(configure_callback) 53 | collectd.register_read(read_callback) 54 | 55 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/postgresql.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/collectd.d/postgresql.conf: 3 | file.managed: 4 | - source: salt://monitoring/client/config/collectd.d/postgresql.conf.jinja 5 | - template: jinja 6 | - context: 7 | monitor_host: {{ salt['pillar.get']('monitoring-client-postgresql:host', '127.0.0.1') }} 8 | monitor_port: {{ salt['pillar.get']('monitoring-client-postgresql:port', '5432') }} 9 | monitor_user: {{ salt['pillar.get']('monitoring-client-postgresql:user', 'monitoring') }} 10 | monitor_password: {{ salt['pillar.get']('monitoring-client-postgresql:password', "") }} 11 | 12 | collectd-postgresql: 13 | pkg: 14 | - installed 15 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/pypi-backend.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/collectd.d/pypi-backend.conf: 3 | file.managed: 4 | - source: salt://monitoring/client/config/collectd.d/pypi-backend.conf 5 | - require: 6 | - file: /usr/local/lib/collectd/plugins 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/pypi-mirror.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/collectd.d/pypi-mirror.conf: 3 | file.managed: 4 | - source: salt://monitoring/client/config/collectd.d/pypi-mirror.conf 5 | - require: 6 | - file: /usr/local/lib/collectd/plugins 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/client/redis.sls: -------------------------------------------------------------------------------- 1 | 2 | /etc/collectd.d/redis.conf: 3 | file.managed: 4 | - source: salt://monitoring/client/config/collectd.d/redis.conf 5 | - require: 6 | - file: /usr/local/lib/collectd/plugins 7 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/carbon.sls: -------------------------------------------------------------------------------- 1 | 2 | {% set carbon_config = pillar.get('carbon', {}) %} 3 | {% set graphite_config = pillar.get('graphite', {}) %} 4 | 5 | graphite-pkgs: 6 | pkg.installed: 7 | - pkgs: 8 | - graphite-web 9 | - python-carbon 10 | - httpd 11 | 12 | /etc/carbon/carbon.conf: 13 | file.managed: 14 | - source: salt://monitoring/server/config/carbon.conf.jinja 15 | - template: jinja 16 | - context: 17 | line_receiver_interface: {{ carbon_config.get('line_receiver_interface', '0.0.0.0') }} 18 | line_reciver_port: {{ carbon_config.get('line_reciver_port', 2003) }} 19 | pickle_receiver_interface: {{ carbon_config.get('pickle_receiver_interface', '0.0.0.0') }} 20 | pickle_receiver_port: {{ carbon_config.get('pickle_receiver_port', 2004) }} 21 | cache_query_interface: {{ carbon_config.get('cache_query_interface', '0.0.0.0') }} 22 | cache_query_port: {{ carbon_config.get('cache_query_port', 7002) }} 23 | 24 | /etc/carbon/storage-schemas.conf: 25 | file.managed: 26 | - source: salt://monitoring/server/config/storage-schemas.conf.jinja 27 | - template: jinja 28 | 29 | carbon-cache: 30 | service: 31 | - running 32 | - enable: True 33 | - restart: True 34 | - watch: 35 | - file: /etc/carbon/carbon.conf 36 | - file: /etc/carbon/storage-schemas.conf 37 | - require: 38 | - file: /etc/carbon/carbon.conf 39 | - file: /etc/carbon/storage-schemas.conf 40 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/carbon.conf.jinja: -------------------------------------------------------------------------------- 1 | [cache] 2 | STORAGE_DIR = /var/lib/carbon/ 3 | LOCAL_DATA_DIR = /var/lib/carbon/whisper/ 4 | WHITELISTS_DIR = /var/lib/carbon/lists/ 5 | CONF_DIR = /etc/carbon/ 6 | LOG_DIR = /var/log/carbon/ 7 | PID_DIR = /var/run/ 8 | 9 | ENABLE_LOGROTATION = True 10 | USER = carbon 11 | MAX_CACHE_SIZE = inf 12 | MAX_UPDATES_PER_SECOND = 500 13 | MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 14 | MAX_CREATES_PER_MINUTE = 50 15 | 16 | LINE_RECEIVER_INTERFACE = {{ line_receiver_interface }} 17 | LINE_RECEIVER_PORT = {{ line_reciver_port }} 18 | 19 | PICKLE_RECEIVER_INTERFACE = {{ pickle_receiver_interface }} 20 | PICKLE_RECEIVER_PORT = {{ pickle_receiver_port }} 21 | 22 | LOG_LISTENER_CONNECTIONS = True 23 | USE_INSECURE_UNPICKLER = False 24 | 25 | CACHE_QUERY_INTERFACE = {{ cache_query_interface }} 26 | CACHE_QUERY_PORT = {{ cache_query_port }} 27 | 28 | USE_FLOW_CONTROL = True 29 | 30 | LOG_UPDATES = False 31 | LOG_CACHE_HITS = False 32 | LOG_CACHE_QUEUE_SORTS = True 33 | 34 | CACHE_WRITE_STRATEGY = sorted 35 | 36 | WHISPER_AUTOFLUSH = False 37 | 38 | # WHISPER_SPARSE_CREATE = False 39 | WHISPER_FALLOCATE_CREATE = True 40 | # WHISPER_LOCK_WRITES = False 41 | # USE_WHITELIST = False 42 | # CARBON_METRIC_PREFIX = carbon 43 | # CARBON_METRIC_INTERVAL = 60 44 | 45 | [relay] 46 | USER = carbon 47 | LINE_RECEIVER_INTERFACE = 0.0.0.0 48 | LINE_RECEIVER_PORT = 2013 49 | PICKLE_RECEIVER_INTERFACE = 0.0.0.0 50 | PICKLE_RECEIVER_PORT = 2014 51 | 52 | LOG_LISTENER_CONNECTIONS = True 53 | 54 | #RELAY_METHOD = rules 55 | #RELAY_METHOD = consistent-hashing 56 | #RELAY_METHOD = aggregated-consistent-hashing 57 | RELAY_METHOD = rules 58 | 59 | REPLICATION_FACTOR = 1 60 | 61 | DESTINATIONS = 127.0.0.1:2004 62 | 63 | MAX_DATAPOINTS_PER_MESSAGE = 500 64 | MAX_QUEUE_SIZE = 10000 65 | 66 | USE_FLOW_CONTROL = True 67 | 68 | # CARBON_METRIC_PREFIX = carbon 69 | # CARBON_METRIC_INTERVAL = 60 70 | 71 | [aggregator] 72 | USER = carbon 73 | LINE_RECEIVER_INTERFACE = 0.0.0.0 74 | LINE_RECEIVER_PORT = 2023 75 | 76 | PICKLE_RECEIVER_INTERFACE = 0.0.0.0 77 | PICKLE_RECEIVER_PORT = 2024 78 | 79 | LOG_LISTENER_CONNECTIONS = True 80 | 81 | FORWARD_ALL = True 82 | 83 | DESTINATIONS = 127.0.0.1:2004 84 | 85 | REPLICATION_FACTOR = 1 86 | 87 | MAX_QUEUE_SIZE = 10000 88 | 89 | USE_FLOW_CONTROL = True 90 | 91 | MAX_DATAPOINTS_PER_MESSAGE = 500 92 | 93 | MAX_AGGREGATION_INTERVALS = 5 94 | 95 | # WRITE_BACK_FREQUENCY = 0 96 | 97 | # USE_WHITELIST = False 98 | 99 | # CARBON_METRIC_PREFIX = carbon 100 | # CARBON_METRIC_INTERVAL = 60 101 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/graphite-web-app.conf.jinja: -------------------------------------------------------------------------------- 1 | client_max_body_size 64M; 2 | keepalive_timeout 5; 3 | 4 | access_log /var/log/nginx/graphite-web/access.log; 5 | error_log /var/log/nginx/graphite-web/error.log; 6 | 7 | root /usr/share/graphite/webapp; 8 | 9 | location /media/ { 10 | root /usr/lib/python2.6/site-packages/django/contrib/admin/media; 11 | } 12 | 13 | try_files $uri/index.html $uri.html $uri @app; 14 | 15 | location @app { 16 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 17 | proxy_set_header Host $http_host; 18 | proxy_redirect off; 19 | proxy_pass http://graphite_web_server; 20 | } 21 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/graphite-web-nginx.conf.jinja: -------------------------------------------------------------------------------- 1 | upstream graphite_web_server { 2 | server unix:/var/run/graphite-web/gunicorn.sock fail_timeout=0; 3 | } 4 | 5 | server { 6 | listen 80; 7 | server_name {{ server_names }}; 8 | 9 | {% if https_only %} 10 | location / { 11 | return 301 https://$server_name$request_uri; 12 | } 13 | location /.well-known/acme-challenge/ { 14 | alias /etc/lego/.well-known/acme-challenge/; 15 | try_files $uri =404; 16 | } 17 | {% else %} 18 | include conf.d/graphite-web/app.conf; 19 | {% endif %} 20 | } 21 | 22 | server { 23 | listen 443; 24 | server_name {{ server_names }}; 25 | include nginx.ssl.conf; 26 | add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; 27 | include conf.d/graphite-web/app.conf; 28 | } 29 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/graphite-web.ini.jinja: -------------------------------------------------------------------------------- 1 | [program:{{ program_name }}] 2 | command = gunicorn_django -w 8 --bind unix:/var/run/graphite-web/gunicorn.sock --pythonpath=/usr/lib/python2.6/site-packages/graphite --settings=settings settings.py 3 | autostart = true 4 | autorestart = true 5 | stopwaitsecs = 15 6 | stopsignal = TERM 7 | stdout_logfile = /var/log/{{ program_name }}/gunicorn-stdout.log 8 | stderr_logfile = /var/log/{{ program_name }}/gunicorn-stderr.log 9 | user = {{ user }} 10 | directory = /usr/lib/python2.6/site-packages/graphite 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/local_settings.py.jinja: -------------------------------------------------------------------------------- 1 | SECRET_KEY = '{{ secret_key }}' 2 | ALLOWED_HOSTS = [ '{{ allowed_hosts }}' ] 3 | TIME_ZONE = 'Etc/UTC' 4 | 5 | LOG_RENDERING_PERFORMANCE = True 6 | LOG_CACHE_PERFORMANCE = True 7 | LOG_METRIC_ACCESS = True 8 | 9 | DEBUG = False 10 | 11 | GRAPHITE_ROOT = '/usr/share/graphite' 12 | CONF_DIR = '/etc/graphite-web' 13 | STORAGE_DIR = '/var/lib/graphite-web' 14 | CONTENT_DIR = '/usr/share/graphite/webapp/content' 15 | #DASHBOARD_CONF = '/etc/graphite-web/dashboard.conf' 16 | #GRAPHTEMPLATES_CONF = '/etc/graphite-web/graphTemplates.conf' 17 | 18 | WHISPER_DIR = '/var/lib/carbon/whisper/' 19 | RRD_DIR = '/var/lib/carbon/rrd' 20 | DATA_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables 21 | LOG_DIR = '/var/log/graphite-web/' 22 | INDEX_FILE = '/var/lib/graphite-web/index' # Search index file 23 | 24 | #USE_REMOTE_USER_AUTHENTICATION = True 25 | #LOGIN_URL = '/account/login' 26 | 27 | DATABASES = { 28 | 'default': { 29 | 'NAME': '{{ sqlite3_path }}', 30 | 'ENGINE': 'django.db.backends.sqlite3', 31 | 'USER': '', 32 | 'PASSWORD': '', 33 | 'HOST': '', 34 | 'PORT': '' 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/riemann.config.jinja: -------------------------------------------------------------------------------- 1 | ; -*- mode: clojure; -*- 2 | ; vim: filetype=clojure 3 | 4 | (logging/init :file "/var/log/riemann/riemann.log") 5 | 6 | ; Listen on the local interface over TCP (5555), UDP (5555), and websockets 7 | ; (5556) 8 | (let [host "{{ host }}"] 9 | (tcp-server :host host) 10 | (udp-server :host host) 11 | (ws-server :host host)) 12 | 13 | (graphite-server :host "{{ graphite_host }}" :port {{ graphite_port }}) 14 | 15 | (def graph (graphite {:host "{{ graphite_downstream_host }}" :port {{ graphite_downstream_port }}})) 16 | 17 | ; Expire old events from the index every 5 seconds. 18 | (periodically-expire 5) 19 | 20 | ; Keep events in the index for 5 minutes by default. 21 | (let [index (default :ttl 300 (update-index (index)))] 22 | 23 | ; Inbound events will be passed to these streams: 24 | (streams 25 | 26 | ; Index all events immediately. 27 | index 28 | ; Forward to graphite immediately. 29 | graph 30 | 31 | ; Calculate an overall rate of events. 32 | (with {:metric 1 :host nil :state "ok" :service "events/sec"} 33 | (rate 5 index)) 34 | 35 | ; Log expired events. 36 | (expired 37 | (fn [event] (info "expired" event))) 38 | )) 39 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/config/storage-schemas.conf.jinja: -------------------------------------------------------------------------------- 1 | [carbon] 2 | pattern = ^carbon\. 3 | retentions = 60:90d 4 | 5 | [downloads] 6 | pattern = ^stats\.downloads 7 | retentions = 10s:1h,1m:1d,1h:7d,1d:28d,28d:5y 8 | 9 | [downloads_counts] 10 | pattern = ^stats_counts\.downloads 11 | retentions = 10s:1h,1m:1d,1h:7d,1d:28d,28d:5y 12 | 13 | [default_10s_for_90d] 14 | pattern = .* 15 | retentions = 10s:90d 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/graphite-web.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - nginx 4 | - supervisor 5 | 6 | graphite: 7 | user: 8 | - present 9 | 10 | {% set graphite_config = pillar.get('graphite', {}) %} 11 | {% set secrets = pillar.get('monitoring_secrets', {}) %} 12 | 13 | {% for dir in ['log', 'lib', 'run'] %} 14 | /var/{{ dir }}/graphite-web: 15 | file.directory: 16 | - user: graphite 17 | - group: graphite 18 | - mode: 755 19 | - require: 20 | - pkg: graphite-pkgs 21 | - user: graphite 22 | {% endfor %} 23 | 24 | graphite-web-gunicorn: 25 | pkg.installed: 26 | - pkgs: 27 | - python-gunicorn 28 | - python-setuptools 29 | 30 | /etc/graphite-web/local_settings.py: 31 | file.managed: 32 | - source: salt://monitoring/server/config/local_settings.py.jinja 33 | - template: jinja 34 | - context: 35 | secret_key: '{{ secrets.get('secret_key') }}' 36 | allowed_hosts: '{{ graphite_config.get('allowed_hosts', '*') }}' 37 | sqlite3_path: {{ graphite_config.get('sqlite3_path', '/var/lib/graphite-web/graphite.db') }} 38 | 39 | graphite-web-init-db: 40 | cmd.run: 41 | - name: '/usr/lib/python2.6/site-packages/graphite/manage.py syncdb --noinput' 42 | - user: graphite 43 | - cwd: /usr/lib/python2.6/site-packages/graphite 44 | 45 | /etc/supervisord.d/graphite-web.ini: 46 | file.managed: 47 | - source: salt://monitoring/server/config/graphite-web.ini.jinja 48 | - template: jinja 49 | - context: 50 | program_name: graphite-web 51 | user: graphite 52 | - require: 53 | - service: supervisord 54 | 55 | graphite-web-supervisor: 56 | cmd.wait: 57 | - name: supervisorctl reread && supervisorctl update 58 | - require: 59 | - file: /etc/supervisord.d/graphite-web.ini 60 | - cmd: graphite-web-init-db 61 | - watch: 62 | - file: /etc/supervisord.d/graphite-web.ini 63 | 64 | graphite-web-reload: 65 | cmd.wait: 66 | - name: 'supervisorctl restart graphite-web' 67 | - require: 68 | - cmd: graphite-web-supervisor 69 | - watch: 70 | - file: /etc/graphite-web/local_settings.py 71 | 72 | /var/log/nginx/graphite-web: 73 | file.directory 74 | 75 | /etc/nginx/conf.d/graphite-web: 76 | file.directory 77 | 78 | /etc/nginx/conf.d/graphite-web/app.conf: 79 | file.managed: 80 | - source: salt://monitoring/server/config/graphite-web-app.conf.jinja 81 | - template: jinja 82 | 83 | /etc/nginx/conf.d/graphite-web.conf: 84 | file.managed: 85 | - source: salt://monitoring/server/config/graphite-web-nginx.conf.jinja 86 | - template: jinja 87 | - context: 88 | server_names: {{ ", ".join(secrets.get('server_names')) }} 89 | https_only: {{ graphite_config.get('https_only', False) }} 90 | - require: 91 | - file: /var/log/nginx/graphite-web 92 | - file: /etc/nginx/conf.d/graphite-web/app.conf 93 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - monitoring.server.carbon 4 | - monitoring.server.riemann 5 | - monitoring.server.graphite-web 6 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/monitoring/server/riemann.sls: -------------------------------------------------------------------------------- 1 | 2 | {% set riemann_config = pillar.get('riemann', {}) %} 3 | 4 | riemann-pkg: 5 | pkg.installed: 6 | - sources: 7 | - riemann: http://aphyr.com/riemann/riemann-0.2.4-1.noarch.rpm 8 | 9 | riemann: 10 | service: 11 | - running 12 | - enable: True 13 | - reload: True 14 | - watch: 15 | - file: /etc/riemann/riemann.config 16 | - require: 17 | - pkg: riemann-pkg 18 | 19 | /etc/riemann/riemann.config: 20 | file.managed: 21 | - source: salt://monitoring/server/config/riemann.config.jinja 22 | - template: jinja 23 | - context: 24 | host: {{ riemann_config.get('host', '0.0.0.0') }} 25 | graphite_host: {{ riemann_config.get('graphite_host', '0.0.0.0') }} 26 | graphite_port: {{ riemann_config.get('graphite_port', 2002) }} 27 | graphite_downstream_host: {{ riemann_config.get('graphite_downstream_host', '127.0.0.1') }} 28 | graphite_downstream_port: {{ riemann_config.get('graphite_downstream_port', 2003) }} 29 | - user: 30 | - group: riemann 31 | - require: 32 | - pkg: riemann-pkg 33 | 34 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/nginx/config/RPM-GPG-KEY-NGINX: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Version: GnuPG v1.4.11 (FreeBSD) 3 | 4 | mQENBE5OMmIBCAD+FPYKGriGGf7NqwKfWC83cBV01gabgVWQmZbMcFzeW+hMsgxH 5 | W6iimD0RsfZ9oEbfJCPG0CRSZ7ppq5pKamYs2+EJ8Q2ysOFHHwpGrA2C8zyNAs4I 6 | QxnZZIbETgcSwFtDun0XiqPwPZgyuXVm9PAbLZRbfBzm8wR/3SWygqZBBLdQk5TE 7 | fDR+Eny/M1RVR4xClECONF9UBB2ejFdI1LD45APbP2hsN/piFByU1t7yK2gpFyRt 8 | 97WzGHn9MV5/TL7AmRPM4pcr3JacmtCnxXeCZ8nLqedoSuHFuhwyDnlAbu8I16O5 9 | XRrfzhrHRJFM1JnIiGmzZi6zBvH0ItfyX6ttABEBAAG0KW5naW54IHNpZ25pbmcg 10 | a2V5IDxzaWduaW5nLWtleUBuZ2lueC5jb20+iQE+BBMBAgAoBQJOTjJiAhsDBQkJ 11 | ZgGABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCr9b2Ce9m/YpvjB/98uV4t 12 | 94d0oEh5XlqEZzVMrcTgPQ3BZt05N5xVuYaglv7OQtdlErMXmRWaFZEqDaMHdniC 13 | sF63jWMd29vC4xpzIfmsLK3ce9oYo4t9o4WWqBUdf0Ff1LMz1dfLG2HDtKPfYg3C 14 | 8NESud09zuP5NohaE8Qzj/4p6rWDiRpuZ++4fnL3Dt3N6jXILwr/TM/Ma7jvaXGP 15 | DO3kzm4dNKp5b5bn2nT2QWLPnEKxvOg5Zoej8l9+KFsUnXoWoYCkMQ2QTpZQFNwF 16 | xwJGoAz8K3PwVPUrIL6b1lsiNovDgcgP0eDgzvwLynWKBPkRRjtgmWLoeaS9FAZV 17 | ccXJMmANXJFuCf26iQEcBBABAgAGBQJOTkelAAoJEKZP1bF62zmo79oH/1XDb29S 18 | YtWp+MTJTPFEwlWRiyRuDXy3wBd/BpwBRIWfWzMs1gnCjNjk0EVBVGa2grvy9Jtx 19 | JKMd6l/PWXVucSt+U/+GO8rBkw14SdhqxaS2l14v6gyMeUrSbY3XfToGfwHC4sa/ 20 | Thn8X4jFaQ2XN5dAIzJGU1s5JA0tjEzUwCnmrKmyMlXZaoQVrmORGjCuH0I0aAFk 21 | RS0UtnB9HPpxhGVbs24xXZQnZDNbUQeulFxS4uP3OLDBAeCHl+v4t/uotIad8v6J 22 | SO93vc1evIje6lguE81HHmJn9noxPItvOvSMb2yPsE8mH4cJHRTFNSEhPW6ghmlf 23 | Wa9ZwiVX5igxcvaIRgQQEQIABgUCTk5b0gAKCRDs8OkLLBcgg1G+AKCnacLb/+W6 24 | cflirUIExgZdUJqoogCeNPVwXiHEIVqithAM1pdY/gcaQZmIRgQQEQIABgUCTk5f 25 | YQAKCRCpN2E5pSTFPnNWAJ9gUozyiS+9jf2rJvqmJSeWuCgVRwCcCUFhXRCpQO2Y 26 | Va3l3WuB+rgKjsQ= 27 | =A015 28 | -----END PGP PUBLIC KEY BLOCK----- 29 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/nginx/config/nginx.conf.jinja: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes 4; 3 | 4 | error_log /var/log/nginx/error.log; 5 | pid /var/run/nginx.pid; 6 | 7 | events { 8 | worker_connections 1024; 9 | } 10 | 11 | http { 12 | include /etc/nginx/mime.types; 13 | default_type application/octet-stream; 14 | 15 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 16 | '$status $body_bytes_sent "$http_referer" ' 17 | '"$http_user_agent" "$http_x_forwarded_for"'; 18 | 19 | access_log /var/log/nginx/access.log main; 20 | sendfile on; 21 | keepalive_timeout 65; 22 | 23 | include /etc/nginx/conf.d/*.conf; 24 | 25 | server { 26 | listen 80 default; 27 | location /nginx_status { 28 | stub_status on; 29 | access_log off; 30 | allow 127.0.0.1; 31 | allow {{ salt['pillar.get']('pypi_internal_network', '127.0.0.0/24') }}; 32 | deny all; 33 | } 34 | location /.well-known/acme-challenge/ { 35 | alias /etc/lego/.well-known/acme-challenge/; 36 | try_files $uri =404; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/nginx/config/nginx.logrotate: -------------------------------------------------------------------------------- 1 | /var/log/nginx/*.log { 2 | daily 3 | rotate 10 4 | missingok 5 | notifempty 6 | compress 7 | sharedscripts 8 | postrotate 9 | /bin/kill -USR1 $(cat /var/run/nginx.pid 2>/dev/null) 2>/dev/null || : 10 | endscript 11 | } 12 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/nginx/config/nginx.ssl.conf.jinja: -------------------------------------------------------------------------------- 1 | ssl on; 2 | ssl_certificate /etc/lego/certificates/{{ grains['fqdn'] }}.crt; 3 | ssl_certificate_key /etc/lego/certificates/{{ grains['fqdn'] }}.key; 4 | 5 | 6 | ssl_protocols TLSv1.2; 7 | ssl_dhparam /etc/pki/tls/private/dhparams.pem; 8 | ssl_ciphers EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA512:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:ECDH+AESG; 9 | ssl_prefer_server_ciphers on; 10 | 11 | ssl_session_timeout 60m; 12 | ssl_session_cache shared:SSL:32m; 13 | ssl_buffer_size 8k; 14 | 15 | ssl_stapling on; 16 | ssl_stapling_verify on; 17 | resolver 8.8.8.8 8.8.4.4 valid=300s; 18 | resolver_timeout 1s; 19 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/nginx/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - tls 4 | 5 | /etc/pki/rpm-gpg/RPM-GPG-KEY-NGINX: 6 | file.managed: 7 | - source: salt://nginx/config/RPM-GPG-KEY-NGINX 8 | - user: root 9 | - group: root 10 | - mode: 444 11 | 12 | nginx-release: 13 | pkgrepo.managed: 14 | - humanname: nginx CentOS YUM repository 15 | - baseurl: http://nginx.org/packages/centos/$releasever/$basearch/ 16 | - gpgcheck: 1 17 | - gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-NGINX 18 | - require: 19 | - file: /etc/pki/rpm-gpg/RPM-GPG-KEY-NGINX 20 | 21 | nginx: 22 | user.present: 23 | - system: True 24 | - shell: /sbin/nologin 25 | - groups: 26 | - nginx 27 | - require: 28 | - group: nginx 29 | group.present: 30 | - system: True 31 | pkg: 32 | - installed 33 | - require: 34 | - pkgrepo: nginx-release 35 | service: 36 | - running 37 | - enable: True 38 | - reload: True 39 | - watch: 40 | - file: /etc/nginx/nginx.conf 41 | - file: /etc/nginx/nginx.ssl.conf 42 | - file: /etc/nginx/conf.d/* 43 | - require: 44 | - file: /etc/nginx/nginx.conf 45 | - pkg: nginx 46 | - user: nginx 47 | 48 | /etc/nginx/nginx.conf: 49 | file.managed: 50 | - source: salt://nginx/config/nginx.conf.jinja 51 | - template: jinja 52 | - user: root 53 | - group: root 54 | - mode: 644 55 | - require: 56 | - file: /var/log/nginx 57 | 58 | /etc/nginx/nginx.ssl.conf: 59 | file.managed: 60 | - source: salt://nginx/config/nginx.ssl.conf.jinja 61 | - template: jinja 62 | - user: root 63 | - group: root 64 | - mode: 644 65 | - require: 66 | - sls: tls 67 | 68 | /etc/logrotate.d/nginx: 69 | file.managed: 70 | - source: salt://nginx/config/nginx.logrotate 71 | - user: root 72 | - group: root 73 | - mode: 644 74 | 75 | /var/log/nginx: 76 | file.directory: 77 | - user: nginx 78 | - group: root 79 | - mode: 0755 80 | 81 | /etc/nginx/conf.d/default.conf: 82 | file.absent 83 | 84 | /etc/nginx/conf.d/virtual.conf: 85 | file.absent 86 | 87 | /etc/nginx/conf.d/ssl.conf: 88 | file.absent 89 | 90 | {% if 'datadog_api_key' in pillar %} 91 | /etc/dd-agent/conf.d/nginx.yaml: 92 | file.managed: 93 | - user: root 94 | - group: root 95 | - mode: '0644' 96 | - contents: | 97 | init_config: 98 | instances: 99 | - nginx_status_url: http://127.0.0.1/nginx_status/ 100 | {% endif %} 101 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/openvpn/routing.sls: -------------------------------------------------------------------------------- 1 | {% set vpn0_internal = salt["pillar.get"]("vpn0_internal_network") %} 2 | {% set vpn1_internal = salt["pillar.get"]("vpn1_internal_network") %} 3 | 4 | {% set vpn0_internal = salt["ip_picker.subnet_mask_for_cidr"](cidr=vpn0_internal) %} 5 | {% set vpn1_internal = salt["ip_picker.subnet_mask_for_cidr"](cidr=vpn1_internal) %} 6 | 7 | {% set psf_internal = salt["pillar.get"]("psf_internal_network") %} 8 | {% set pypi_internal = salt["pillar.get"]("pypi_internal_network") %} 9 | 10 | {% set interfaces = salt["ip_picker.interfaces_for_cidr"](cidr=psf_internal) %} 11 | {% set gateway = salt["pillar.get"]("psf_internal_vpn_gateway") %} 12 | 13 | {% if not interfaces %} 14 | {% set interfaces = salt["ip_picker.interfaces_for_cidr"](cidr=pypi_internal) %} 15 | {% set gateway = salt["pillar.get"]("pypi_internal_vpn_gateway") %} 16 | {% endif %} 17 | 18 | 19 | {% for interface in interfaces %} 20 | {{ interface }}: 21 | network.routes: 22 | - routes: 23 | - name: vpn 24 | ipaddr: {{ vpn0_internal.address }} 25 | netmask: {{ vpn0_internal.subnet }} 26 | gateway: {{ gateway }} 27 | - name: vpn-https 28 | ipaddr: {{ vpn1_internal.address }} 29 | netmask: {{ vpn1_internal.subnet }} 30 | gateway: {{ gateway }} 31 | {% endfor %} 32 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pkg/git.sls: -------------------------------------------------------------------------------- 1 | git: 2 | pkg.installed 3 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/postgresql/93/RPM-GPG-KEY-PGDG-93: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Version: GnuPG v1.4.7 (GNU/Linux) 3 | 4 | mQGiBEeD8koRBACC1VBRsUwGr9gxFFRho9kZpdRUjBJoPhkeOTvp9LzkdAQMFngr 5 | BFi6N0ov1kCX7LLwBmDG+JPR7N+XcH9YR1coSHpLVg+JNy2kFDd4zAyWxJafjZ3a 6 | 9zFg9Yx+0va1BJ2t4zVcmKS4aOfbgQ5KwIOWUujalQW5Y+Fw39Gn86qjbwCg5dIo 7 | tkM0l19h2sx50D027pV5aPsD/2c9pfcFTbMhB0CcKS836GH1qY+NCAdUwPs646ee 8 | Ex/k9Uy4qMwhl3HuCGGGa+N6Plyon7V0TzZuRGp/1742dE8IO+I/KLy2L1d1Fxrn 9 | XOTBZd8qe6nBwh12OMcKrsPBVBxn+iSkaG3ULsgOtx+HHLfa1/p22L5+GzGdxizr 10 | peBuA/90cCp+lYcEwdYaRoFVR501yDOTmmzBc1DrsyWP79QMEGzMqa393G0VnqXt 11 | L4pGmunq66Agw2EhPcIt3pDYiCmEt/obdVtSJH6BtmSDB/zYhbE8u3vLP3jfFDa9 12 | KXxgtYj0NvuUVoRmxSKm8jtfmj1L7zoKNz3jl+Ba3L0WxIv4+bRBUG9zdGdyZVNR 13 | TCBSUE0gQnVpbGRpbmcgUHJvamVjdCA8cGdzcWxycG1zLWhhY2tlcnNAcGdmb3Vu 14 | ZHJ5Lm9yZz6IYAQTEQIAIAUCR4PySgIbIwYLCQgHAwIEFQIIAwQWAgMBAh4BAheA 15 | AAoJEB8W0uFELfD4jnkAoMqd6ZwwsgYHZ3hP9vt+DJt1uDW7AKDbRwP8ESKFhwdJ 16 | 8m91RPBeJW/tMLkCDQRHg/JKEAgA64+ZXgcERPYfZYo4p+yMTJAAa9aqnE3U4Ni6 17 | ZMB57GPuEy8NfbNya+HiftO8hoozmJdcI6XFyRBCDUVCdZ8SE+PJdOx2FFqZVIu6 18 | dKnr8ykhgLpNNEFDG3boK9UfLj/5lYQ3Y550Iym1QKOgyrJYeAp6sZ+Nx2PavsP3 19 | nMFCSD67BqAbcLCVQN7a2dAUXfEbfXJjPHXTbo1/kxtzE+KCRTLdXEbSEe3nHO04 20 | K/EgTBjeBUOxnciH5RylJ2oGy/v4xr9ed7R1jJtshsDKMdWApwoLlCBJ63jg/4T/ 21 | z/OtXmu4AvmWaJxaTl7fPf2GqSqqb6jLCrQAH7AIhXr9V0zPZwADBQgAlpptNQHl 22 | u7euIdIujFwwcxyQGfee6BG+3zaNSEHMVQMuc6bxuvYmgM9r7aki/b0YMfjJBk8v 23 | OJ3Eh1vDH/woJi2iJ13vQ21ot+1JP3fMd6NPR8/qEeDnmVXu7QAtlkmSKI9Rdnjz 24 | FFSUJrQPHnKsH4V4uvAM+njwYD+VFiwlBPTKNeL8cdBb4tPN2cdVJzoAp57wkZAN 25 | VA2tKxNsTJKBi8wukaLWX8+yPHiWCNWItvyB4WCEp/rZKG4A868NM5sZQMAabpLd 26 | l4fTiGu68OYgK9qUPZvhEAL2C1jPDVHPkLm+ZsD+90Pe66w9vB00cxXuHLzm8Pad 27 | GaCXCY8h3xi6VIhJBBgRAgAJBQJHg/JKAhsMAAoJEB8W0uFELfD4K4cAoJ4yug8y 28 | 1U0cZEiF5W25HDzMTtaDAKCaM1m3Cbd+AZ0NGWNg/VvIX9MsPA== 29 | =au6K 30 | -----END PGP PUBLIC KEY BLOCK----- 31 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/postgresql/93/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - postgresql.93.repo 4 | 5 | postgresql93-server: 6 | pkg: 7 | - installed 8 | - require: 9 | - pkgrepo: pgdg-93-centos 10 | service: 11 | - name: postgresql-9.3 12 | - running 13 | - enable: True 14 | - reload: True 15 | - watch: 16 | - file: /var/lib/pgsql/9.3/data/pg_hba.conf 17 | - require: 18 | - pkg: postgresql93-server 19 | cmd.wait: 20 | - name: service postgresql-9.3 initdb 21 | - watch: 22 | - pkg: postgresql93-server 23 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/postgresql/93/repo.sls: -------------------------------------------------------------------------------- 1 | /etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG-93: 2 | file.managed: 3 | - source: salt://postgresql/93/RPM-GPG-KEY-PGDG-93 4 | - user: root 5 | - group: root 6 | - mode: 444 7 | 8 | pgdg-93-centos: 9 | pkgrepo.managed: 10 | - humanname: PostgreSQL 9.3 $releasever - $basearch 11 | - baseurl: http://yum.postgresql.org/9.3/redhat/rhel-$releasever-$basearch 12 | - gpgcheck: 1 13 | - gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG-93 14 | - require: 15 | - file: /etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG-93 16 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/config/bandersnatch.conf.jinja: -------------------------------------------------------------------------------- 1 | 2 | [mirror] 3 | directory = {{ directory }} 4 | json = true 5 | master = {{ master }} 6 | timeout = {{ timeout }} 7 | workers = {{ workers }} 8 | stop-on-error = {{ stop_on_error }} 9 | delete-packages = {{ delete_packages }} 10 | 11 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/config/bandersnatch.logrotate.conf: -------------------------------------------------------------------------------- 1 | /var/log/nginx/{{ mirror }}-mirror/*.log { 2 | daily 3 | rotate 14 4 | missingok 5 | notifempty 6 | compress 7 | delaycompress 8 | sharedscripts 9 | postrotate 10 | /bin/kill -USR1 $(cat /var/run/nginx.pid 2>/dev/null) 2>/dev/null || : 11 | endscript 12 | } 13 | 14 | /data/{{ mirror }}-mirror/*.log { 15 | daily 16 | rotate 7 17 | missingok 18 | notifempty 19 | compress 20 | delaycompress 21 | } 22 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/config/bandersnatch.nginx.app.conf: -------------------------------------------------------------------------------- 1 | autoindex on; 2 | charset utf-8; 3 | 4 | add_header Surrogate-Control max-age=60; 5 | 6 | error_page 503 /index.html; 7 | 8 | location = /last-modified { 9 | access_log off; 10 | } 11 | 12 | location = /index.html { 13 | root /usr/share/www/mirror/{{ mirror }}; 14 | } 15 | 16 | location = /daytime { 17 | return 501; 18 | } 19 | 20 | root /data/{{ mirror }}-mirror/web; 21 | 22 | location /local-stats {} 23 | location /packages {} 24 | location /simple {} 25 | location /serversig {} 26 | 27 | location / { 28 | return 503; 29 | } 30 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/config/bandersnatch.nginx.conf: -------------------------------------------------------------------------------- 1 | 2 | log_format {{ mirror }}_fastly_backend '$remote_addr $http_x_forwarded_for $remote_user [$time_local] ' 3 | '"$request" $status $body_bytes_sent ' 4 | '"$http_referer" "$http_user_agent"'; 5 | 6 | server { 7 | listen 80; 8 | server_name {{ server_names }}; 9 | 10 | access_log /var/log/nginx/{{ mirror }}-mirror/access.log {{ mirror }}_fastly_backend; 11 | error_log /var/log/nginx/{{ mirror }}-mirror/error.log; 12 | 13 | include conf.d/{{ mirror }}-mirror/app.conf; 14 | } 15 | 16 | server { 17 | listen 443; 18 | server_name {{ server_names }}; 19 | include nginx.ssl.conf; 20 | 21 | access_log /var/log/nginx/{{ mirror }}-mirror/access.log {{ mirror }}_fastly_backend; 22 | error_log /var/log/nginx/{{ mirror }}-mirror/error.log; 23 | 24 | add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; 25 | include conf.d/{{ mirror }}-mirror/app.conf; 26 | } 27 | 28 | 29 | server { 30 | listen {{ tls_port }}; 31 | include nginx.ssl.conf; 32 | 33 | access_log /var/log/nginx/{{ mirror }}-mirror/access.log {{ mirror }}_fastly_backend; 34 | error_log /var/log/nginx/{{ mirror }}-mirror/error.log; 35 | 36 | port_in_redirect off; 37 | 38 | add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; 39 | include conf.d/{{ mirror }}-mirror/app.conf; 40 | } 41 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/config/crontab.jinja: -------------------------------------------------------------------------------- 1 | {% if 'develop' in grains['roles'] %} 2 | #*/1 * * * * pypi-mirror /opt/bandersnatch/bin/bandersnatch -c /etc/bandersnatch/{{ mirror }}-mirror.conf mirror |& logger -t bandersnatch[{{ mirror }}-mirror] 3 | {% else %} 4 | */1 * * * * pypi-mirror /opt/bandersnatch/bin/bandersnatch -c /etc/bandersnatch/{{ mirror }}-mirror.conf mirror |& logger -t bandersnatch[{{ mirror }}-mirror] 5 | {% endif %} 6 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi-mirror/init.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - nginx 4 | - monitoring.client.pypi-mirror 5 | - python.35 6 | 7 | pypi-mirror: 8 | user.present: 9 | - home: /home/pypi-mirror 10 | 11 | /etc/bandersnatch: 12 | file.directory 13 | 14 | /opt/bandersnatch: 15 | file.directory: 16 | - mode: 755 17 | - makedirs: True 18 | virtualenv.managed: 19 | - venv_bin: /usr/bin/pyvenv-3.5 20 | - system_site_packages: False 21 | - require: 22 | - file: /opt/bandersnatch 23 | 24 | bandersnatch: 25 | pip.installed: 26 | - name: bandersnatch==2.1.3 27 | - bin_env: /opt/bandersnatch 28 | - require: 29 | - virtualenv: /opt/bandersnatch 30 | 31 | {% for mirror, config in salt['pillar.get']('bandersnatch').iteritems() %} 32 | 33 | /usr/share/www/mirror/{{ mirror }}/index.html: 34 | file.managed: 35 | - makedirs: True 36 | - source: salt://pypi-mirror/config/index.html.jinja 37 | - template: jinja 38 | - context: 39 | source_url: {{ config.get('mirror', {}).get('master', 'https://pypi.python.org') }} 40 | 41 | /data/{{ mirror }}-mirror: 42 | file.directory: 43 | - user: pypi-mirror 44 | - group: pypi-mirror 45 | - mode: 755 46 | - makedirs: True 47 | - require: 48 | - user: pypi-mirror 49 | 50 | /var/log/nginx/{{ mirror }}-mirror: 51 | file.directory: 52 | - user: root 53 | - group: root 54 | - makedirs: True 55 | 56 | /etc/nginx/conf.d/{{ mirror }}-mirror: 57 | file.directory 58 | 59 | /etc/nginx/conf.d/{{ mirror }}-mirror/app.conf: 60 | file.managed: 61 | - source: salt://pypi-mirror/config/bandersnatch.nginx.app.conf 62 | - template: jinja 63 | - context: 64 | mirror: {{ mirror }} 65 | - user: root 66 | - group: root 67 | - mode: 640 68 | - makedirs: True 69 | - require: 70 | - file: /etc/nginx/conf.d/{{ mirror }}-mirror 71 | 72 | /etc/nginx/conf.d/{{ mirror }}-mirror.conf: 73 | file.managed: 74 | - source: salt://pypi-mirror/config/bandersnatch.nginx.conf 75 | - template: jinja 76 | - context: 77 | mirror: {{ mirror }} 78 | server_names: {{ " ".join(config.get('server_names')) }} 79 | tls_port: {{ config.get('tls_port', '8989') }} 80 | - user: root 81 | - group: root 82 | - mode: 640 83 | - makedirs: True 84 | - require: 85 | - file: /etc/nginx/conf.d/{{ mirror }}-mirror/app.conf 86 | - file: /var/log/nginx/{{ mirror }}-mirror 87 | 88 | /etc/logrotate.d/{{ mirror }}-mirror: 89 | file.managed: 90 | - source: salt://pypi-mirror/config/bandersnatch.logrotate.conf 91 | - template: jinja 92 | - context: 93 | mirror: {{ mirror }} 94 | - user: root 95 | - group: root 96 | - mode: 644 97 | - makedirs: True 98 | 99 | /etc/bandersnatch/{{ mirror }}-mirror.conf: 100 | file.managed: 101 | - source: salt://pypi-mirror/config/bandersnatch.conf.jinja 102 | - template: jinja 103 | - context: 104 | directory: {{ config.get('mirror', {}).get('directory', '/data/pypi') }} 105 | master: {{ config.get('mirror', {}).get('master', 'https://pypi.python.org') }} 106 | timeout: {{ config.get('mirror', {}).get('timeout', '10') }} 107 | workers: {{ config.get('mirror', {}).get('workers', '3') }} 108 | stop_on_error: {{ config.get('mirror', {}).get('stop-on-error', 'false') }} 109 | delete_packages: {{ config.get('mirror', {}).get('delete-packages', 'true') }} 110 | access_log_pattern: {{ config.get('statistics', {}).get('access-log-pattern', '/var/log/nginx/pypi-mirror/access*.log') }} 111 | - require: 112 | - file: /etc/bandersnatch 113 | 114 | {% if 'develop' in grains['roles'] %} 115 | {% if not salt['file.directory_exists']('/data/' + mirror + '-mirror/web') %} 116 | bandersnatch_short_sync: 117 | cmd.run: 118 | - name: '( /opt/bandersnatch/bin/bandersnatch -c /etc/bandersnatch/{{ mirror }}-mirror.conf mirror ) & sleep 30; kill $!' 119 | - cwd: /opt/bandersnatch 120 | - user: pypi-mirror 121 | - require: 122 | - file: /etc/bandersnatch/{{ mirror }}-mirror.conf 123 | - virtualenv: /opt/bandersnatch 124 | {% endif %} 125 | {% endif %} 126 | 127 | /etc/cron.d/{{ mirror }}-mirror: 128 | file.managed: 129 | - source: salt://pypi-mirror/config/crontab.jinja 130 | - template: jinja 131 | - context: 132 | mirror: {{ mirror }} 133 | - require: 134 | - file: /etc/bandersnatch/{{ mirror }}-mirror.conf 135 | - virtualenv: /opt/bandersnatch 136 | 137 | {% endfor %} 138 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/base.sls: -------------------------------------------------------------------------------- 1 | 2 | include: 3 | - python.27.virtualenv 4 | - pkg.git 5 | 6 | pypi-system-deps: 7 | pkg.installed: 8 | - pkgs: 9 | - python27-devel 10 | - postgresql-devel 11 | - gcc 12 | - openssl-devel 13 | - libffi-devel 14 | - require: 15 | - pkgrepo: ewdurbin-pythons-el6 16 | 17 | 18 | # Fix an error with Elastcisearch's IPv6 going sideways 19 | net.ipv6.conf.all.disable_ipv6: 20 | sysctl.present: 21 | - value: 1 22 | 23 | {% set deploys = {} %} 24 | {% for k,v in pillar.items() %} 25 | {% if k.startswith('pypi-deploy-') %} 26 | {% do deploys.update({k: v}) %} 27 | {% endif %} 28 | {% endfor %} 29 | 30 | {% for key, config in deploys.items() %} 31 | 32 | {{ config['name'] }}-user: 33 | group.present: 34 | - name: {{ config['group'] }} 35 | - gid: {{ config['group_gid'] }} 36 | user.present: 37 | - name: {{ config['user'] }} 38 | - uid: {{ config['user_uid'] }} 39 | - gid: {{ config['group_gid'] }} 40 | - home: {{ config['path'] }} 41 | - createhome: True 42 | - require: 43 | - group: {{ config['group'] }} 44 | 45 | {{ config['path'] }}: 46 | file.directory: 47 | - user: {{ config['user'] }} 48 | - group: {{ config['group'] }} 49 | - mode: 755 50 | 51 | /var/log/{{ config['name'] }}: 52 | file.directory: 53 | - user: {{ config['user'] }} 54 | - group: {{ config['group'] }} 55 | - mode: 750 56 | 57 | /var/run/{{ config['name'] }}: 58 | file.directory: 59 | - user: {{ config['user'] }} 60 | - group: {{ config['group'] }} 61 | - mode: 755 62 | 63 | {{ config['name'] }}-source: 64 | git.latest: 65 | - name: {{ config.get('source_uri', "https://github.com/pypa/pypi-legacy.git") }} 66 | - rev: {{ config.get('source_rev', "master") }} 67 | - target: {{ config['path'] }}/src 68 | - user: {{ config['name'] }} 69 | - require: 70 | - user: {{ config['name'] }} 71 | - file: {{ config['path'] }} 72 | 73 | /opt/{{ config['name'] }}/env: 74 | virtualenv.managed: 75 | - venv_bin: /usr/bin/virtualenv 76 | - python: /usr/bin/python2.7 77 | - system_site_packages: {{ config.get('site_packages', True) }} 78 | - user: {{ config['name'] }} 79 | - cwd: {{ config['path'] }}/src 80 | - requirements: {{ config['path'] }}/src/requirements.txt 81 | - require: 82 | - file: {{ config['path'] }} 83 | - git: {{ config['name'] }}-source 84 | - user: {{ config['user'] }} 85 | - pip: virtualenv-2.7 86 | - pkg: pypi-system-deps 87 | 88 | /opt/{{ config['name'] }}/uwsgi/plugins: 89 | file.directory: 90 | - user: {{ config['user'] }} 91 | - group: {{ config['group'] }} 92 | - mode: 755 93 | - makedirs: True 94 | 95 | {{ config['name'] }}_initial_build_dogstatsd_uwsgi: 96 | cmd.run: 97 | - creates: /opt/{{ config['name'] }}/uwsgi/plugins/dogstatsd_plugin.so 98 | - name: "/opt/{{ config['name'] }}/env/bin/uwsgi --build-plugin https://github.com/Datadog/uwsgi-dogstatsd" 99 | - user: {{ config['user'] }} 100 | - cwd: /opt/{{ config['name'] }}/uwsgi/plugins 101 | 102 | {{ config['name'] }}_build_dogstatsd_uwsgi: 103 | cmd.run: 104 | - onchanges: 105 | - virtualenv: /opt/{{ config['name'] }}/env 106 | - name: "/opt/{{ config['name'] }}/env/bin/uwsgi --build-plugin https://github.com/Datadog/uwsgi-dogstatsd" 107 | - user: {{ config['user'] }} 108 | - cwd: /opt/{{ config['name'] }}/uwsgi/plugins 109 | 110 | {{ config['path'] }}/src/config.ini: 111 | file.managed: 112 | - source: salt://pypi/config/pypi.ini.jinja 113 | - user: {{ config['name'] }} 114 | - group: {{ config['name'] }} 115 | - mode: 640 116 | - template: jinja 117 | - context: 118 | app_key: {{ key }} 119 | - require: 120 | - file: /var/log/{{ config['name'] }} 121 | - file: /var/run/{{ config['name'] }} 122 | - virtualenv: {{ config['path'] }}/env 123 | 124 | {% endfor %} 125 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi-cdn-log-archiver-wrapper.sh.jinja: -------------------------------------------------------------------------------- 1 | {% set secrets = salt['pillar.get']("secrets-"+app_key) -%} 2 | #!/bin/bash 3 | export ACCESS_KEY={{ secrets['cdn_log_archiver']['access_key'] }} 4 | export SECRET_KEY={{ secrets['cdn_log_archiver']['secret_key'] }} 5 | export PYPI_LOG_BUCKET={{ pypi_log_bucket }} 6 | export S3_HOST={{ s3_host }} 7 | {% if debug -%} 8 | export DEBUG=10 9 | {%- endif %} 10 | 11 | /opt/pypi-cdn-log-archiver/env/bin/pypi-cdn-log-archiver 12 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi-docs-proxy.ini.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | 3 | [program:{{ config['name'] }}-docs-proxy] 4 | command = /opt/pypi-docs-proxy/env/bin/gunicorn -k eventlet -b 0.0.0.0:{{ config['docs_port'] }} --access-logfile - --error-logfile - docs_proxy:application 5 | autostart = true 6 | autorestart = true 7 | stopwaitsecs = 15 8 | stopsignal = TERM 9 | stdout_logfile = /var/log/{{ config['name'] }}/docs-proxy.stdout.log 10 | stderr_logfile = /var/log/{{ config['name'] }}/docs-proxy.stderr.log 11 | user = {{ config['user'] }} 12 | environment = DOCS_PROXY_BUCKET={{ config['docs_bucket'] }} 13 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi-syslog.logrotate.conf: -------------------------------------------------------------------------------- 1 | /var/log/cdn/{{ syslog_name }}/*.log { 2 | rotate 24 3 | missingok 4 | notifempty 5 | compress 6 | sharedscripts 7 | dateext 8 | dateformat -%Y-%m-%d-%s 9 | postrotate 10 | /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true 11 | endscript 12 | } 13 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi-worker.ini.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | 3 | [program:{{ config['name'] }}-worker] 4 | command = {{ config['path'] }}/env/bin/python {{ config['path'] }}/src/tools/worker.py 5 | autostart = true 6 | autorestart = true 7 | stopwaitsecs = 15 8 | stopsignal = TERM 9 | stdout_logfile = /var/log/{{ config['name'] }}/worker.stdout.log 10 | stderr_logfile = /var/log/{{ config['name'] }}/worker.stderr.log 11 | user = {{ config['user'] }} 12 | directory = {{ config['path'] }}/src 13 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.ini.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | {% set secrets = salt['pillar.get']("secrets-"+app_key) %} 3 | 4 | [database] 5 | 6 | ; Postgres Database 7 | host = {{ secrets['postgresql']['host'] }} 8 | port = {{ secrets['postgresql'].get('port', '5432') }} 9 | name = {{ secrets['postgresql']['database'] }} 10 | user = {{ secrets['postgresql']['user'] }} 11 | password={{ secrets['postgresql']['password'] }} 12 | 13 | ; Redis 14 | {% for key, value in secrets['redis'].iteritems() %} 15 | {{ key }} = {{ value }} 16 | {% endfor %} 17 | 18 | ; Storage Directories 19 | files_dir = {{ config['data_mount'] }}/packages 20 | docs_dir = {{ config['data_mount'] }}/packagedocs 21 | 22 | ; S3 Buckets 23 | files_bucket = {{ config['files_bucket'] }} 24 | docs_bucket = {{ config['docs_bucket'] }} 25 | 26 | ; AWS Key 27 | aws_access_key_id = {{ secrets['aws']['aws_access_key_id'] }} 28 | aws_secret_access_key = {{ secrets['aws']['aws_secret_access_key'] }} 29 | 30 | ; ElasticSearch 31 | {% if secrets.get('releases_index_url', False) and secrets.get('releases_index_name', False) %} 32 | releases_index_url = {{ secrets['releases_index_url'] }} 33 | releases_index_name = {{ secrets['releases_index_name'] }} 34 | {% endif %} 35 | 36 | ; Third-Party 37 | pubsubhubbub = http://pubsubhubbub.appspot.com/ 38 | 39 | [webui] 40 | 41 | ; PyPI config 42 | debug_mode = no 43 | rss_file = /tmp/{{ app_key }}-rss.xml 44 | packages_rss_file = /tmp/{{ app_key }}-packages_rss.xml 45 | 46 | ; Email 47 | adminemail = PyPI Admin 48 | replyto = admin@mail.pypi.python.org 49 | 50 | ; Secrets 51 | sshkeys_update = {{ config['path'] }}/src/sshkeys_update 52 | key_dir = {{ config['path'] }}/secret 53 | cheesecake_password = {{ secrets['webui']['cheescake_password'] }} 54 | reset_secret = {{ secrets['webui']['reset_secret'] }} 55 | 56 | ; URI Paths 57 | simple_script = /simple 58 | raw_package_prefix = /raw-packages 59 | simple_sign_script = /serversig 60 | 61 | ; URLs 62 | url = {{ config['url'] }}/pypi 63 | files_url = {{ config['url'] }}/packages/ 64 | pydotorg = https://www.python.org/ 65 | package_docs_url = http://pythonhosted.org/ 66 | 67 | ; StatusPage.io 68 | statuspage_id = {{ config['statuspage_id'] }} 69 | 70 | [smtp] 71 | hostname = {{ secrets['smtp']['hostname'] }} 72 | starttls = {{ secrets['smtp'].get('starttls', 'off') }} 73 | auth = {{ secrets['smtp'].get('auth', 'off') }} 74 | login = {{ secrets['smtp'].get('login', '') }} 75 | password = {{ secrets['smtp'].get('password', '') }} 76 | 77 | [passlib] 78 | schemes = argon2, bcrypt_sha256, bcrypt, django_bcrypt, unix_disabled 79 | argon2__time_cost=6 80 | argon2__memory_cost=1024 81 | argon2__parallelism=6 82 | 83 | [logging] 84 | file = /var/log/{{ config['name'] }}/pypi.log 85 | mail_logger = {{ secrets['logging'].get('mail_logger', 'off') }} 86 | fromaddr = {{ secrets['logging']['fromaddr'] }} 87 | toaddrs = {{ secrets['logging']['toaddrs'] }} 88 | 89 | [sentry] 90 | dsn = {{ secrets['sentry']['dsn'] }} 91 | 92 | [uwsgi] 93 | uid={{ config['user'] }} 94 | gid={{ config['group'] }} 95 | wsgi-file = {{ config['path'] }}/src/pypi.wsgi 96 | socket = /var/run/{{ config['name'] }}/pypi.sock 97 | stats = /var/run/{{ config['name'] }}/pypi-stats.sock 98 | pidfile = /var/run/{{ config['name'] }}/pypi.pid 99 | daemonize = 127.0.0.1:8224 100 | processes = {{ secrets['uwsgi'].get('processes', 2) }} 101 | harakiri = 60 102 | reload-on-as = 400 103 | reload-on-rss = 128 104 | max-requests = 5000 105 | master = 1 106 | post-buffering = 8192 107 | chmod-socket = 666 108 | disable-logging = true 109 | log-5xx = true 110 | logto2 = /var/log/{{ config['name'] }}/uwsgi.log 111 | listen = 1024 112 | buffer-size = 16384 113 | ignore-sigpipe = true 114 | ignore-write-errors = true 115 | disable-write-exception = true 116 | plugins-dir = /opt/{{ config['name'] }}/uwsgi/plugins 117 | 118 | {% if 'datadog_api_key' in pillar %} 119 | enable-metrics = true 120 | plugin = dogstatsd 121 | stats-push = dogstatsd:127.0.0.1:18125,{{ config['name'] }} 122 | {% endif %} 123 | 124 | ; CDN API 125 | [fastly] 126 | api_domain = https://api.fastly.com/ 127 | api_key = {{ secrets['fastly']['api_key'] }} 128 | service_id = {{ secrets['fastly']['service_id'] }} 129 | 130 | [blocking] 131 | blocked_timeout = {{ secrets.get('blocking', {}).get('blocked_timeout', 600) }} 132 | blocked_attempts = {{ secrets.get('blocking', {}).get('blocked_attempts', 10) }} 133 | blocked_attempts_ip = {{ secrets.get('blocking', {}).get('blocked_attempts_ip', 20) }} 134 | blocked_attempts_user = {{ secrets.get('blocking', {}).get('blocked_attempts_user', 1000) }} 135 | 136 | [xmlrpc] 137 | max_concurrent = {{ secrets.get('xmlrpc', {}).get('max_concurrent', 9999) }} 138 | enforce = {{ secrets.get('xmlrpc', {}).get('enforce', 'false') }} 139 | request_log_file = /var/log/{{ config['name'] }}/xmlrpc.log 140 | 141 | [authomatic] 142 | secure = {{ secrets.get('authomatic', {}).get('secure', 'true') }} 143 | secret = {{ secrets.get('authomatic', {}).get('secret', 'randodata') }} 144 | 145 | [google] 146 | client_id = {{ secrets.get('google', {}).get('client_id', '') }} 147 | client_secret = {{ secrets.get('google', {}).get('client_secret', '') }} 148 | 149 | [datadog] 150 | dogstatsd_port = 18125 151 | tags = deployment:{{ config['name'] }} 152 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.initd.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | #!/bin/sh 3 | 4 | ### BEGIN INIT INFO 5 | # Provides: {{ config['name'] }} 6 | # Required-Start: $local_fs $remote_fs $network 7 | # Required-Stop: $local_fs $remote_fs $network 8 | # Default-Start: 2 3 4 5 9 | # Default-Stop: 0 1 6 10 | # Short-Description: starts the {{ config['name'] }} uwsgi server 11 | # Description: starts the {{ config['name'] }} uwsgi server 12 | ### END INIT INFO 13 | 14 | . /etc/rc.d/init.d/functions 15 | 16 | daemon={{ config['path'] }}/env/bin/uwsgi 17 | pidfile=/var/run/{{ config['name'] }}/pypi.pid 18 | lockfile=/var/lock/subsys/{{ config['name'] }} 19 | 20 | prog={{ config['name'] }} 21 | 22 | daemon_opts="{{ config['path'] }}/src/config.ini" 23 | 24 | start() { 25 | [ -x $daemon ] || exit 5 26 | echo -n $"Starting $prog: " 27 | daemon $daemon $daemon_opts 28 | retval=$? 29 | echo 30 | [ $retval -eq 0 ] && touch $lockfile 31 | return $retval 32 | } 33 | 34 | stop() { 35 | echo -n $"Stopping $prog: " 36 | killproc -p $pidfile $prog 37 | retval=$? 38 | echo 39 | [ $retval -eq 0 ] && rm -f $lockfile 40 | return $retval 41 | } 42 | 43 | check_status() { 44 | status -p $pidfile 45 | } 46 | 47 | check_status_q() { 48 | status -p $pidfile > /dev/null 2>&1 49 | } 50 | 51 | reload() { 52 | echo -n $"Reloading $prog: " 53 | killproc -p $pidfile $prog -HUP 54 | echo 55 | } 56 | 57 | case "$1" in 58 | start) 59 | check_status_q && exit 0 60 | start 61 | ;; 62 | stop) 63 | check_status_q || exit 0 64 | stop 65 | ;; 66 | status) 67 | check_status 68 | ;; 69 | reload) 70 | reload 71 | ;; 72 | force-reload) 73 | reload 74 | ;; 75 | restart) 76 | stop 77 | start 78 | ;; 79 | *) 80 | echo "Usage: $proc {start|stop|status|restart|reload|force-reload}" >&2 81 | exit 1 82 | ;; 83 | esac 84 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.logrotate.conf: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | 3 | /var/log/nginx/{{ config['name'] }}/*.log { 4 | daily 5 | rotate 14 6 | missingok 7 | notifempty 8 | compress 9 | delaycompress 10 | sharedscripts 11 | postrotate 12 | /bin/kill -USR1 $(cat /var/run/nginx.pid 2>/dev/null) 2>/dev/null || : 13 | endscript 14 | } 15 | 16 | /var/log/{{ config['name'] }}/*.log { 17 | daily 18 | rotate 14 19 | missingok 20 | notifempty 21 | compress 22 | delaycompress 23 | sharedscripts 24 | copytruncate 25 | } 26 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.nginx.app.conf.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | 3 | access_log /var/log/nginx/{{ config['name'] }}/access.log; 4 | error_log /var/log/nginx/{{ config['name'] }}/error.log; 5 | 6 | add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; 7 | 8 | set_real_ip_from 0.0.0.0/0; 9 | real_ip_header X-Forwarded-For; 10 | 11 | server_name_in_redirect off; 12 | port_in_redirect off; 13 | 14 | rewrite ^/$ /pypi redirect; 15 | 16 | client_max_body_size 100M; 17 | 18 | location ~* ^/(daytime|serial).* { 19 | access_log off; 20 | include uwsgi_params; 21 | uwsgi_pass unix:/var/run/{{ config['name'] }}/pypi.sock; 22 | uwsgi_param SCRIPT_NAME /$1; 23 | # the following magic stands for "UWSGI_MODIFIER_MANAGE_PATH_INFO" 24 | # .. when setting the SCRIPT_NAME remove that bit from the PATH_INFO 25 | uwsgi_modifier1 30; 26 | } 27 | 28 | location ~* ^/pypi/.*/json$ { 29 | include uwsgi_params; 30 | {% if rate_limit is defined %} 31 | limit_req zone={{ zone_name }} burst={{ burst }} {% if nodelay -%}nodelay{%- endif -%}; 32 | limit_req_status 429; 33 | {% endif %} 34 | uwsgi_pass unix:/var/run/{{ config['name'] }}/pypi.sock; 35 | uwsgi_param SCRIPT_NAME /$1; 36 | # the following magic stands for "UWSGI_MODIFIER_MANAGE_PATH_INFO" 37 | # .. when setting the SCRIPT_NAME remove that bit from the PATH_INFO 38 | uwsgi_modifier1 30; 39 | } 40 | 41 | location ~* ^/(pypi|mirrors|oauth|security|tos|google_login|openid_login|openid_claim).* { 42 | error_page 500 502 503 504 /static/error/500.html; 43 | error_page 404 /static/error/400.html; 44 | error_page 429 /static/error/429.html; 45 | uwsgi_intercept_errors on; 46 | include uwsgi_params; 47 | {% if rate_limit is defined %} 48 | limit_req zone={{ zone_name }} burst={{ burst }} {% if nodelay -%}nodelay{%- endif -%}; 49 | limit_req_status 429; 50 | {% endif %} 51 | uwsgi_pass unix:/var/run/{{ config['name'] }}/pypi.sock; 52 | uwsgi_param SCRIPT_NAME /$1; 53 | # the following magic stands for "UWSGI_MODIFIER_MANAGE_PATH_INFO" 54 | # .. when setting the SCRIPT_NAME remove that bit from the PATH_INFO 55 | uwsgi_modifier1 30; 56 | } 57 | 58 | location ~* ^/(simple|serversig|packages).* { 59 | include uwsgi_params; 60 | uwsgi_pass unix:/var/run/{{ config['name'] }}/pypi.sock; 61 | uwsgi_param SCRIPT_NAME /$1; 62 | # the following magic stands for "UWSGI_MODIFIER_MANAGE_PATH_INFO" 63 | # .. when setting the SCRIPT_NAME remove that bit from the PATH_INFO 64 | uwsgi_modifier1 30; 65 | 66 | gzip on; 67 | gzip_comp_level 9; 68 | gzip_vary on; 69 | } 70 | 71 | location /raw-packages { 72 | alias {{ config['data_mount'] }}/packages; 73 | 74 | add_header X-PYPI-LAST-SERIAL $upstream_http_x_pypi_last_serial; 75 | add_header Surrogate-Key $upstream_http_surrogate_key; 76 | 77 | internal; 78 | 79 | autoindex on; 80 | 81 | add_header Strict-Transport-Security "max-age=31536000"; 82 | } 83 | 84 | location ~* ^/(stats|local-stats).* { 85 | autoindex on; 86 | } 87 | 88 | location /static { 89 | alias {{ config['path'] }}/src/static; 90 | 91 | add_header Cache-Control public; 92 | expires 1d; 93 | } 94 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.nginx.conf.jinja: -------------------------------------------------------------------------------- 1 | {% set config = salt['pillar.get'](app_key) %} 2 | 3 | server { 4 | listen 443; 5 | server_name {{ " ".join(config['server_names']) }}; 6 | include nginx.ssl.conf; 7 | include conf.d/{{ config['name'] }}/app.conf; 8 | } 9 | 10 | server { 11 | listen {{ config['tls_port'] }}; 12 | include nginx.ssl.conf; 13 | include conf.d/{{ config['name'] }}/app.conf; 14 | } 15 | 16 | {% if serve_docs %} 17 | server { 18 | listen {{ config['docs_port'] }}; 19 | server_name_in_redirect off; 20 | port_in_redirect off; 21 | root {{ config['data_mount'] }}/packagedocs; 22 | rewrite ^([^.]*[^/])$ https://$http_host$request_uri/ permanent; 23 | } 24 | {% endif %} 25 | 26 | server { 27 | listen {{ config['internal_pypi_port']}}; 28 | include conf.d/{{ config['name'] }}/app.conf; 29 | } 30 | 31 | server { 32 | listen {{ config['internal_docs_port']}}; 33 | server_name_in_redirect off; 34 | port_in_redirect off; 35 | root {{ config['data_mount'] }}/packagedocs; 36 | rewrite ^([^.]*[^/])$ https://$http_host$request_uri/ permanent; 37 | } 38 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.nginx.ratelimit.conf.jinja: -------------------------------------------------------------------------------- 1 | limit_req_zone $binary_remote_addr zone={{ zone_name }}:{{ zone_size }} rate={{ max_rate }}; 2 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/pypi.rsyslog.conf.jinja: -------------------------------------------------------------------------------- 1 | # Provides UDP syslog reception 2 | $ModLoad imudp 3 | $UDPServerRun 514 4 | 5 | # Provides TCP syslog reception 6 | $ModLoad imtcp 7 | $InputTCPServerRun 514 8 | 9 | $ModLoad imtcp 10 | $InputTCPServerRun 514 11 | 12 | # Don't rate-limit 13 | $SystemLogRateLimitInterval 0 14 | 15 | # Don't filter duplicate lines 16 | $RepeatedMsgReduction off 17 | 18 | $DirCreateMode 0755 19 | 20 | # Log everything from the CDN to a file 21 | :app-name, isequal, "{{ syslog_name }}" /var/log/cdn/{{ syslog_name }}/access.log 22 | # Discard logs 23 | :app-name, isequal, "{{ syslog_name }}" ~ 24 | 25 | $DirCreateMode 0700 26 | 27 | # Turn back on filtering duplicate lines 28 | $RepeatedMsgReduction on 29 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/config/test-pg_hba.conf: -------------------------------------------------------------------------------- 1 | local all all ident 2 | host all all 127.0.0.1/32 md5 3 | host all all ::1/128 md5 4 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/dev-db.sls: -------------------------------------------------------------------------------- 1 | {% set deploys = {} %} 2 | {% for k,v in pillar.items() %} 3 | {% if k.startswith('pypi-deploy-') %} 4 | {% do deploys.update({k: v}) %} 5 | {% endif %} 6 | {% endfor %} 7 | 8 | postgresql: 9 | pkg.installed 10 | 11 | pypi_postgres_contrib: 12 | pkg.installed: 13 | - name: postgresql93-contrib 14 | 15 | pypi_postgres_pg_hba: 16 | file.managed: 17 | - name: /var/lib/pgsql/9.3/data/pg_hba.conf 18 | - source: salt://pypi/config/test-pg_hba.conf 19 | - user: postgres 20 | - group: postgres 21 | - mode: 600 22 | - require: 23 | - pkg: postgresql93-server 24 | - cmd: postgresql93-server 25 | 26 | {% for key, config in deploys.items() %} 27 | {% set secrets = salt['pillar.get']("secrets-"+key) %} 28 | 29 | {{ config['name'] }}_dev_postgres_user: 30 | postgres_user.present: 31 | - name: {{ secrets['postgresql']['user'] }} 32 | - password: {{ secrets['postgresql']['password'] }} 33 | - user: postgres 34 | - require: 35 | - service: postgresql-9.3 36 | - pkg: postgresql 37 | 38 | {{ config['name'] }}_postgres_database: 39 | postgres_database.present: 40 | - name: {{ secrets['postgresql']['database'] }} 41 | - owner: {{ secrets['postgresql']['user'] }} 42 | - require: 43 | - postgres_user: {{ config['name'] }}_dev_postgres_user 44 | 45 | {{ config['name'] }}_postgres_citext: 46 | cmd.wait: 47 | - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' 48 | - user: postgres 49 | - require: 50 | - postgres_database: {{ config['name'] }}_postgres_database 51 | - postgres_user: {{ config['name'] }}_dev_postgres_user 52 | - pkg: pypi_postgres_contrib 53 | - watch: 54 | - postgres_database: {{ config['name'] }}_postgres_database 55 | - postgres_user: {{ config['name'] }}_dev_postgres_user 56 | - pkg: pypi_postgres_contrib 57 | 58 | {{ config['name'] }}_postgres_plpgsql: 59 | cmd.wait: 60 | - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' 61 | - user: postgres 62 | - require: 63 | - postgres_database: {{ config['name'] }}_postgres_database 64 | - postgres_user: {{ config['name'] }}_dev_postgres_user 65 | - watch: 66 | - postgres_database: {{ config['name'] }}_postgres_database 67 | - postgres_user: {{ config['name'] }}_dev_postgres_user 68 | 69 | {{ config['name'] }}_postgres_uuid-ossp: 70 | cmd.wait: 71 | - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\" WITH SCHEMA public;"' 72 | - user: postgres 73 | - require: 74 | - postgres_database: {{ config['name'] }}_postgres_database 75 | - postgres_user: {{ config['name'] }}_dev_postgres_user 76 | - watch: 77 | - postgres_database: {{ config['name'] }}_postgres_database 78 | - postgres_user: {{ config['name'] }}_dev_postgres_user 79 | 80 | {{ config['path'] }}/.pgpass: 81 | file.managed: 82 | - contents: "{{ secrets['postgresql']['host'] }}:*:{{ secrets['postgresql']['database'] }}:{{ secrets['postgresql']['user'] }}:{{ secrets['postgresql']['password'] }}" 83 | - user: {{ config['user'] }} 84 | - group: {{ config['group'] }} 85 | - mode: 600 86 | - require: 87 | - user: {{ config['user'] }} 88 | - group: {{ config['group'] }} 89 | 90 | {{ config['name'] }}_schema_load: 91 | cmd.wait: 92 | - name: 'psql {{ secrets['postgresql']['database'] }} -U {{ secrets['postgresql']['user'] }} -h {{ secrets['postgresql']['host'] }} -f {{ config['path'] }}/src/pkgbase_schema.sql' 93 | - user: {{ config['user'] }} 94 | - cwd: {{ config['path'] }} 95 | - env: 96 | PGPASSWORD: {{ secrets['postgresql']['password'] }} 97 | - require: 98 | - postgres_database: {{ config['name'] }}_postgres_database 99 | - postgres_user: {{ config['name'] }}_dev_postgres_user 100 | - cmd: {{ config['name'] }}_postgres_uuid-ossp 101 | - cmd: {{ config['name'] }}_postgres_plpgsql 102 | - cmd: {{ config['name'] }}_postgres_citext 103 | - git: {{ config['name'] }}-source 104 | - file: {{ config['path'] }}/.pgpass 105 | - watch: 106 | - postgres_database: {{ config['name'] }}_postgres_database 107 | 108 | {{ config['name'] }}_sample_data_load: 109 | cmd.wait: 110 | - name: '{{ config['path'] }}/env/bin/python {{ config['path'] }}/src/tools/demodata.py' 111 | - user: {{ config['user'] }} 112 | - cwd: {{ config['path'] }}/src 113 | - require: 114 | - cmd: {{ config['name'] }}_schema_load 115 | - virtualenv: {{ config['path'] }}/env 116 | - file: {{ config['path'] }}/src/config.ini 117 | - file: {{ config['data_mount'] }} 118 | - watch: 119 | - postgres_database: {{ config['name'] }}_postgres_database 120 | 121 | {% endfor %} 122 | -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/files/dogstatsd_plugin.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/salt/907bbf1c4c9b831c03b4eba755d982a32e70589e/provisioning/salt/roots/salt/pypi/files/dogstatsd_plugin.so -------------------------------------------------------------------------------- /provisioning/salt/roots/salt/pypi/files/error.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Mirror of https://pypi.python.org 6 | 39 | 40 | 41 |
42 |

Something is wrong with PyPI

43 |

44 | PyPI is down for maintenance or is having an outage.
45 | Packages and the simple index are still being served via our CDN.
46 |
47 | Check http://status.python.org or 48 | follow @PythonStatus for updates. 49 |

50 |