├── .dockerignore ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Jenkinsfile ├── README.md ├── README.rst ├── bootstrap.sh ├── bootstrap_plugins ├── centos72.plugin.sh ├── centos73.plugin.sh ├── centos74.plugin.sh ├── centos75.plugin.sh ├── centos76.plugin.sh ├── centos77.plugin.sh ├── centos78.plugin.sh ├── centos79.plugin.sh ├── os-router.plugin.sh ├── plugin-defaults.plugin.sh ├── ubuntu14.plugin.sh └── us_en.i18n.plugin.sh ├── contrib ├── scripts │ ├── fwd_settings.sh │ └── instant_docker_registry.sh └── sslproxycert │ ├── README.md │ ├── emc_ssl.pem │ ├── emc_sslv2.pem │ └── install-cert.sh ├── docs ├── .gitkeep ├── Makefile ├── design │ ├── bootstrapper.mmd │ ├── deploy.yml.mmd │ ├── deploy.yml.png │ ├── ecsdeploy.mmd │ ├── ecsmap.mmd │ ├── paths.mmd │ └── reference.deploy.yml ├── legacy │ └── changelog.md └── source │ ├── README.txt │ ├── building │ ├── building.md │ ├── building.rst │ ├── development-build.md │ └── development-build.rst │ ├── changelog.rst │ ├── conf.py │ ├── index.rst │ ├── installation │ ├── ECS-Installation.md │ ├── ECS-Installation.rst │ ├── Island_Installation.md │ ├── Island_Installation.rst │ ├── OVA_Installation.md │ ├── OVA_Installation.rst │ ├── Standard_Installation.md │ ├── Standard_Installation.rst │ ├── deploy.yml.md │ ├── deploy.yml.rst │ ├── public-key-authentication.md │ └── public-key-authentication.rst │ ├── media │ ├── Create_replication_group.PNG │ ├── Dell_EMC_logo.svg.png │ ├── create_namespace.PNG │ ├── create_object_user.png │ ├── create_object_user_keys.png │ ├── create_storage_vpool.PNG │ ├── create_virtual_data_center.PNG │ ├── ecc-host-network-example.png │ ├── ecs-disk-install-step1.PNG │ ├── ecs-disk-install-step2.PNG │ ├── ecs-disk-install-step3.PNG │ ├── ecs-disk-install-step4.PNG │ ├── ecs-waiting-for-webserver.PNG │ └── input_license.PNG │ ├── troubleshooting │ ├── ECS-Troubleshooting.md │ ├── ECS-Troubleshooting.rst │ ├── FAQ.md │ └── FAQ.rst │ ├── update-docs.sh │ ├── use │ ├── ECS-UI-Web-Interface.md │ ├── ECS-UI-Web-Interface.rst │ ├── Migration.md │ └── Migration.rst │ └── utilities │ ├── utilities.md │ └── utilities.rst ├── examples ├── local-lab-1-node │ ├── README.md │ ├── bootstrap-local.sh │ ├── deploy.yml │ └── registry.crt ├── local-lab-4-node │ ├── README.md │ ├── bootstrap-local.sh │ └── deploy.yml └── local-lab-all-in-one │ ├── README.md │ ├── bootstrap-local.sh │ └── deploy.yml ├── license.txt ├── patches ├── 2.x │ ├── 2.2.0.0 │ │ ├── Dockerfile │ │ ├── storageos-blobclient.jar │ │ └── storageserver-partition-config.sh │ ├── 2.2.0.1 │ │ ├── Dockerfile │ │ ├── Dockerfile.dare │ │ ├── storageos-dataservice │ │ ├── storageos-ssm.jar │ │ └── storageserver-partition-config.sh │ ├── 2.2.0.3 │ │ ├── Dockerfile │ │ ├── coordinator.properties │ │ ├── storageos-ssm.jar │ │ ├── storageserver-partition-config.sh │ │ ├── transformsvc │ │ └── vnest.object.properties │ ├── 2.2.1.0 │ │ ├── Dockerfile │ │ ├── coordinator.properties │ │ ├── storageserver-partition-config.sh │ │ ├── systool │ │ ├── transformsvc │ │ └── vnest.object.properties │ └── 2.2.1.1 │ │ ├── Dockerfile │ │ ├── coordinator.properties │ │ ├── storageserver-partition-config.sh │ │ ├── systool │ │ ├── transformsvc │ │ └── vnest.object.properties ├── 3.0.0.1 │ ├── Dockerfile │ ├── storageserver-partition-config.sh │ ├── transformsvc │ └── vnest.object.properties ├── 3.0.0.2 │ ├── Dockerfile │ └── vnest-common-conf-template.xml ├── 3.1.0.0 │ └── Dockerfile ├── 3.1.0.1 │ └── Dockerfile ├── 3.1.0.2 │ └── Dockerfile ├── 3.1.0.3 │ ├── Dockerfile │ └── build.sh ├── 3.2.0.0 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.2.0.1 │ ├── .dockerignore │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.3.0.0 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.3.0.2 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.4.0.0 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.5.0.0 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.5.1.2 │ ├── Dockerfile │ ├── build.sh │ └── image.conf ├── 3.8.0.1 │ ├── Dockerfile │ ├── app.d4e76e5fa6bc1dc79cf5.js │ ├── build.sh │ ├── image.conf │ ├── storageserver-partition-config.sh │ └── tune_sr ├── 3.9.0.0 │ ├── Dockerfile │ ├── app.6fb9f5b2bd922a3cb087.js │ ├── app.acbc922074b0cc5197f2.js │ ├── build.sh │ ├── eventsvc │ ├── image.conf │ ├── rm │ ├── shared-threadpool-conf.xml │ ├── storageos-dataservice │ └── storageserver-partition-config.sh ├── build-ecs-software.sh └── libexec │ ├── gen_dockerfile.sh │ └── patch_script.sh ├── release.conf ├── tests ├── ansible │ ├── install_node.yml │ ├── install_node_reboot.yml │ ├── install_node_setup.yml │ ├── roles │ │ ├── install_node │ │ │ └── tasks │ │ │ │ └── main.yml │ │ ├── install_node_reboot │ │ │ └── tasks │ │ │ │ └── main.yml │ │ └── install_node_setup │ │ │ └── tasks │ │ │ └── main.yml │ └── templates │ │ └── deploy.yml.j2 ├── lab.tf ├── output.tf ├── tf_to_hosts ├── tf_to_hosts.py ├── tf_to_ssh.py └── vars.tf ├── tools ├── bootstrap-conf.sh ├── clear-installer-image.sh ├── update-deploy.sh ├── update-ecs-image.sh ├── update-installer-image.sh └── zerofill.sh └── ui ├── __init__.py ├── ansible ├── __init__.py ├── callback_plugins │ ├── __init__.py │ ├── ansible_callback.py │ └── profile_tasks.py ├── catfacts.sh ├── clicmd_access.yml ├── clicmd_access_host.yml ├── clicmd_bootstrap.yml ├── clicmd_cache.yml ├── clicmd_deploy.yml ├── clicmd_disable_cache.yml ├── clicmd_enable_cache.yml ├── clicmd_preflight.yml ├── clicmd_purge.yml ├── clicmd_reboot.yml ├── clicmd_start.yml ├── clicmd_stop.yml ├── group_vars │ ├── .gitkeep │ ├── install_node.yml │ ├── os_CentOS_7.yml │ └── os_CentOS_7_configure_access.yml ├── host_vars │ └── .gitkeep ├── library │ └── __init__.py ├── roles │ ├── CentOS_7_baseline_install │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── emcecs_ssl_proxy.pem.j2 │ │ │ ├── firewalld.trusted.zone.j2 │ │ │ ├── ntp.conf.j2 │ │ │ └── resolv.conf.j2 │ ├── CentOS_7_configure_ssh │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── CentOS_7_purge │ │ └── tasks │ │ │ └── main.yml │ ├── CentOS_7_reboot │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── CentOS_7_service_policies │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── ecs.service.j2 │ ├── CentOS_7_sync_caches_prep │ │ └── tasks │ │ │ └── main.yml │ ├── common_access_test │ │ └── tasks │ │ │ └── main.yml │ ├── common_baseline_check │ │ ├── files │ │ │ └── listening_ports.sh │ │ └── tasks │ │ │ └── main.yml │ ├── common_baseline_install │ │ ├── files │ │ │ └── additional_prep.sh │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── agent.json.j2 │ │ │ ├── id-old.json.j2 │ │ │ ├── id.json.j2 │ │ │ ├── object-main_network.json.j2 │ │ │ ├── rev0-network.json.j2 │ │ │ ├── rev1-network.json.j2 │ │ │ └── seeds.j2 │ │ └── vars │ │ │ └── main.yml │ ├── common_collect_facts │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── data_node.fact.j2 │ ├── common_deploy │ │ ├── files │ │ │ ├── cron_patch_template │ │ │ ├── postinstall_patch │ │ │ └── setup_python_requests │ │ ├── tasks │ │ │ └── main.yml │ │ ├── templates │ │ │ ├── common.object.properties.j2 │ │ │ └── ssm.object.properties.j2 │ │ └── vars │ │ │ └── main.yml │ ├── common_disable_cache │ │ └── tasks │ │ │ └── main.yml │ ├── common_enable_cache │ │ └── tasks │ │ │ └── main.yml │ ├── common_etc_hosts │ │ └── tasks │ │ │ └── main.yml │ ├── common_export_ssl_pem │ │ └── task │ │ │ └── main.yml │ ├── common_purge_cleanup │ │ └── tasks │ │ │ └── main.yml │ ├── common_purge_containers │ │ └── tasks │ │ │ └── main.yml │ ├── common_run_cf_client │ │ └── tasks │ │ │ └── main.yml │ ├── common_set_hostname │ │ └── tasks │ │ │ └── main.yml │ ├── common_start_enable_service │ │ └── tasks │ │ │ └── main.yml │ ├── common_stop_disable_service │ │ └── tasks │ │ │ └── main.yml │ ├── common_sync_caches_by_copy │ │ └── tasks │ │ │ └── main.yml │ ├── common_sync_caches_by_torrent │ │ └── tasks │ │ │ └── main.yml │ ├── firewalld_configure_access │ │ ├── handlers │ │ │ └── main.yml │ │ └── tasks │ │ │ └── main.yml │ ├── installer_build_cache │ │ └── tasks │ │ │ └── main.yml │ ├── installer_generate_ssh_keys │ │ ├── tasks │ │ │ └── main.yml │ │ └── vars │ │ │ └── main.yml │ └── testing │ │ ├── files │ │ └── .gitkeep │ │ ├── tasks │ │ ├── .gitkeep │ │ └── main.yml │ │ ├── templates │ │ └── .gitkeep │ │ └── vars │ │ └── .gitkeep ├── templates │ ├── .gitkeep │ ├── all.j2 │ ├── autonames.yml.j2 │ ├── caches.yml.j2 │ ├── dumpall.yml.j2 │ ├── dumpvars.yml.j2 │ └── hosts.ini.j2 ├── testing.yml └── vars │ ├── .gitkeep │ ├── password_auth.yml │ ├── ports.yml │ └── pubkey_auth.yml ├── build_image.sh ├── ecsconfig.py ├── ecsdeploy.py ├── ecsremove.py ├── etc ├── autonames.yml ├── caches.yml ├── config.yml ├── generic_help.j2.yml ├── install_script.yml ├── release.conf └── schema.yml ├── inventory.py ├── libexec ├── console.sh ├── dockerutils.sh ├── includes.sh ├── osutils.sh ├── shipit.lib.sh └── versioning.sh ├── pull_image.sh ├── requirements.txt ├── resources ├── __init__.py ├── docker │ ├── Rockerfile │ ├── ansible.cfg │ ├── ecs-install-requirements.txt │ ├── entrypoint.sh │ └── torrent.sh └── futures │ ├── __init__.py │ ├── caches.yml │ ├── mt_urwid │ ├── .gitignore │ ├── Readme │ ├── __init__.py │ ├── scrollview.py │ ├── splitview.py │ └── test.py │ ├── pyservmonitor │ ├── .gitignore │ ├── LICENSE.md │ ├── README.md │ ├── __init__.py │ ├── cfg_reader.py │ ├── config.ini │ ├── examples │ │ └── pyservmonitor.png │ ├── libdocker.py │ ├── plugin.py │ ├── pyservmonitor.py │ └── ui.py │ ├── sqllog │ ├── log.sql │ └── log.sqlite │ └── weave │ └── weave.yml ├── run.sh ├── setup.py ├── tui ├── __init__.py ├── ansibler.py ├── constants.py ├── director.py ├── ecsapi.py ├── ecsconf.py ├── ecsdiag.py ├── schema_functions.py ├── script.py ├── state.py ├── tabview.py ├── tools.py ├── tui.py └── widgets.py ├── ui.py └── update_image.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | /install.log 2 | /local_serial 3 | /ui/resources/futures 4 | /.idea 5 | /.projectKnowledge 6 | /patches 7 | /legacy/ecs-multi-node 8 | /examples 9 | /legacy/ecs-single-node 10 | /bootstrap_plugins 11 | /legacy/Documentation 12 | /README.md 13 | /bootstrap.sh 14 | /.gitignore 15 | /docs 16 | /ECS-CommunityEdition.iml 17 | /.gitattributes 18 | /.dockerignore 19 | ### Created by .ignore support plugin (hsz.mobi) 20 | ### Linux template 21 | *~ 22 | 23 | ### KDE directory preferences 24 | .directory 25 | 26 | ### Linux trash folder which might appear on any partition or disk 27 | .Trash-* 28 | ### Emacs template 29 | ### -*- mode: gitignore; -*- 30 | \###*\### 31 | /.emacs.desktop 32 | /.emacs.desktop.lock 33 | *.elc 34 | auto-save-list 35 | tramp 36 | .\###* 37 | 38 | ### Org-mode 39 | /.joblist 40 | /.drone.yml 41 | .org-id-locations 42 | *_archive 43 | 44 | ### flymake-mode 45 | *_flymake.* 46 | 47 | ### eshell files 48 | /eshell/history 49 | /eshell/lastdir 50 | 51 | ### elpa packages 52 | /elpa/ 53 | 54 | ### reftex files 55 | *.rel 56 | 57 | ### AUCTeX auto folder 58 | /auto/ 59 | 60 | ### cask packages 61 | .cask/ 62 | ### Vagrant template 63 | .vagrant/ 64 | ### SublimeText template 65 | ### cache files for sublime text 66 | *.tmlanguage.cache 67 | *.tmPreferences.cache 68 | *.stTheme.cache 69 | 70 | ### workspace files are user-specific 71 | *.sublime-workspace 72 | 73 | /.idea/scratch 74 | ### project files should be checked into the repository, unless a significant 75 | ### proportion of contributors will probably not be using SublimeText 76 | ### *.sublime-project 77 | 78 | ### sftp configuration file 79 | sftp-config.json 80 | ### Example user template template 81 | ### Example user template 82 | 83 | ### IntelliJ project files 84 | .idea 85 | *.iml 86 | out 87 | gen### Tags template 88 | ### Ignore tags created by etags, ctags, gtags (GNU global) and cscope 89 | TAGS 90 | !TAGS/ 91 | tags 92 | !tags/ 93 | gtags.files 94 | GTAGS 95 | GRTAGS 96 | GPATH 97 | cscope.files 98 | cscope.out 99 | cscope.in.out 100 | cscope.po.out 101 | 102 | ### Windows template 103 | ### Windows image file caches 104 | Thumbs.db 105 | ehthumbs.db 106 | 107 | ### Folder config file 108 | Desktop.ini 109 | 110 | ### Recycle Bin used on file shares 111 | $RECYCLE.BIN/ 112 | 113 | ### Windows Installer files 114 | *.cab 115 | *.msi 116 | *.msm 117 | *.msp 118 | 119 | ### Windows shortcuts 120 | *.lnk 121 | ### Vim template 122 | [._]*.s[a-w][a-z] 123 | [._]s[a-w][a-z] 124 | *.un~ 125 | Session.vim 126 | .netrwhist 127 | 128 | ### NotepadPP template 129 | *.bak 130 | /virtualenv/ 131 | /.idea/ 132 | /.projectKnowledge/ 133 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, in case people don't have core.autocrlf set. 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Expected Behavior 2 | 3 | 4 | ### Actual Behavior 5 | (Please put additional output and logs in the section for that below) 6 | 7 | ### Steps to Reproduce Behavior 8 | 9 | 1. 10 | 2. 11 | 3. 12 | 13 | ### Relevant Output and Logs 14 | ``` 15 | # Output and Logs go here 16 | ``` 17 | 18 | --- 19 | Notifies: @nikhil-vr 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### This pull request addresses the following referenced issue(s): 2 | 3 | 1. 4 | 2. 5 | 3. 6 | 7 | ### Overview of changes: 8 | 9 | 1. 10 | 2. 11 | 3. 12 | 13 | --- 14 | Notifies: @nikhil-vr 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.projectKnowledge 2 | /.compliance 3 | # Created by .ignore support plugin (hsz.mobi) 4 | ### Python template 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | /.joblist 22 | /.drone.yml 23 | .eggs/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *,cover 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # Sphinx documentation 60 | docs/_build/ 61 | 62 | # PyBuilder 63 | target/ 64 | 65 | 66 | ### JetBrains template 67 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion 68 | 69 | *.iml 70 | 71 | ## Directory-based project format: 72 | .idea/ 73 | # if you remove the above rule, at least ignore the following: 74 | 75 | # User-specific stuff: 76 | # .idea/workspace.xml 77 | # .idea/tasks.xml 78 | # .idea/dictionaries 79 | 80 | # Sensitive or high-churn files: 81 | # .idea/dataSources.ids 82 | # .idea/dataSources.xml 83 | # .idea/sqlDataSources.xml 84 | # .idea/dynamic.xml 85 | # .idea/uiDesigner.xml 86 | 87 | # Gradle: 88 | # .idea/gradle.xml 89 | # .idea/libraries 90 | 91 | # Mongo Explorer plugin: 92 | # .idea/mongoSettings.xml 93 | 94 | ### Eclipse template 95 | # Covers Eclipse 96 | 97 | # Eclipse Core 98 | .project 99 | 100 | ## File-based project format: 101 | *.ipr 102 | *.iws 103 | 104 | ## Plugin-specific files: 105 | 106 | # IntelliJ 107 | /out/ 108 | 109 | # mpeltonen/sbt-idea plugin 110 | .idea_modules/ 111 | 112 | # JIRA plugin 113 | atlassian-ide-plugin.xml 114 | 115 | # Crashlytics plugin (for Android Studio and IntelliJ) 116 | com_crashlytics_export_strings.xml 117 | crashlytics.properties 118 | crashlytics-build.properties 119 | /virtualenv/ 120 | 121 | # Other 122 | 123 | 124 | /.projectKnowledge/ 125 | -------------------------------------------------------------------------------- /bootstrap_plugins/os-router.plugin.sh: -------------------------------------------------------------------------------- 1 | #@IgnoreInspection BashAddShebang 2 | 3 | # Copyright (c) 2015 EMC Corporation 4 | # All Rights Reserved 5 | # 6 | # This software contains the intellectual property of EMC Corporation 7 | # or is licensed to EMC Corporation from third parties. Use of this 8 | # software and the intellectual property contained therein is expressly 9 | # limited to the terms and conditions of the License Agreement under which 10 | # it is provided by or on behalf of EMC. 11 | 12 | # OS detection and routing 13 | 14 | ### Basic OS detection 15 | os='' 16 | detect_os() { 17 | 18 | # detect Ubuntu and anything using lsb_release 19 | if [ -f /etc/lsb-release ]; then 20 | os=$(lsb_release -d -s) 21 | 22 | # detect Debian version 23 | elif [ -f /etc/debian_version ]; then 24 | os="debian $(cat /etc/debian_version)" 25 | 26 | # detect RHEL, CentOS, and variants 27 | elif [ -f /etc/redhat-release ]; then 28 | os=`cat /etc/redhat-release` 29 | # TODO: SuSE, Arch, etc... 30 | 31 | # fallback for unknowns, perhaps they can be added to the list. 32 | else 33 | os="$(uname -s) $(uname -r) $(cat /etc/*_ver* /etc/*-rel*)" 34 | fi 35 | 36 | # string to lowercase 37 | os="${os,,}" 38 | } 39 | 40 | ### Route by detected OS 41 | route_os() { 42 | 43 | case "${os}" in 44 | 45 | centos\ linux\ release\ 7.4*) 46 | source ${plugins}/centos74.plugin.sh 47 | ;; 48 | centos\ linux\ release\ 7.5*) 49 | source ${plugins}/centos75.plugin.sh 50 | ;; 51 | centos\ linux\ release\ 7.6*) 52 | source ${plugins}/centos76.plugin.sh 53 | ;; 54 | centos\ linux\ release\ 7.7*) 55 | source ${plugins}/centos77.plugin.sh 56 | ;; 57 | centos\ linux\ release\ 7.8*) 58 | source ${plugins}/centos78.plugin.sh 59 | ;; 60 | centos\ linux\ release\ 7.9*) 61 | source ${plugins}/centos79.plugin.sh 62 | ;; 63 | # Die on unknowns 64 | *) 65 | die "OS unknown and not supported: $os" 66 | ;; 67 | esac 68 | } 69 | -------------------------------------------------------------------------------- /bootstrap_plugins/plugin-defaults.plugin.sh: -------------------------------------------------------------------------------- 1 | #@IgnoreInspection BashAddShebang 2 | 3 | # Copyright (c) 2015 EMC Corporation 4 | # All Rights Reserved 5 | # 6 | # This software contains the intellectual property of EMC Corporation 7 | # or is licensed to EMC Corporation from third parties. Use of this 8 | # software and the intellectual property contained therein is expressly 9 | # limited to the terms and conditions of the License Agreement under which 10 | # it is provided by or on behalf of EMC. 11 | 12 | # Plugin defaults, functions can be overridden if 'unset' first: 13 | 14 | os_supported=false 15 | 16 | # Caller symlinks 17 | unset symlink_scripts 18 | symlink_scripts() { 19 | symlinks="ecsdeploy ecsconfig ecsremove catfacts enter pingnodes update_deploy" 20 | symlinks="${symlinks} update_image rebuild_image inventory testbook videploy" 21 | symlinks="${symlinks} step1 step2" 22 | symlinks="${symlinks} island-step1 island-step2 island-step3" 23 | symlinks="${symlinks} ova-step1 ova-step2" 24 | mkdir -p "${HOME}/bin" 25 | for l in $symlinks; do 26 | ln -s "${root}/ui/run.sh" "${HOME}/bin/$l" 2>/dev/null 27 | done 28 | } 29 | 30 | unset update_path_in_bashrc 31 | update_path_in_bashrc() { 32 | log "sed error is OK here if the proxy config file does not yet exist." 33 | sudo sed -i -e '/PATH/d' $HOME/.bashrc 34 | echo 'export PATH=$PATH:$HOME/.local/bin:$HOME/bin' >> $HOME/.bashrc 35 | } 36 | 37 | create_install_tree() { 38 | for pathname in "${ui_host_ssl_dir}" "${ui_host_ssh_dir}"; do 39 | sudo mkdir -p "${pathname}" 40 | done 41 | } 42 | -------------------------------------------------------------------------------- /bootstrap_plugins/ubuntu14.plugin.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015 EMC Corporation 2 | # All Rights Reserved 3 | # 4 | # This software contains the intellectual property of EMC Corporation 5 | # or is licensed to EMC Corporation from third parties. Use of this 6 | # software and the intellectual property contained therein is expressly 7 | # limited to the terms and conditions of the License Agreement under which 8 | # it is provided by or on behalf of EMC. 9 | 10 | # packages to install before others 11 | # Must install epel-release prior to pip. 12 | list_prefix_packages='python-devel curl' 13 | # command to run after installing prefix_packages 14 | cmd_prefix_packages='curl -fsSL https://get.docker.com/ | sudo sh && sudo systemctl enable --now docker' 15 | # packages to install 16 | list_general_packages='git sqlite rsync' 17 | # command to run after installing general_packages 18 | cmd_general_packages='' 19 | # packages to install after others 20 | list_suffix_packages='' 21 | # command to run after installing suffix_packages 22 | cmd_suffix_packages='' 23 | # packages to install if a VM 24 | list_vm_packages='dkms qemu-guest-agent open-vm-tools' 25 | # command to run after installing vm_packages 26 | cmd_vm_packages='' 27 | # command to install an os package manager package 28 | cmd_in_repo_pkg='sudo apt-get -y install' 29 | # command to update all packages in the os package manager 30 | cmd_up_repo_all='sudo apt-get -y upgrade' 31 | # command to enable a repo in the os package manager 32 | cmd_enable_repo='sudo yum-config-manager --enable ' 33 | # command to rebuild the os package manager's database 34 | cmd_up_repo_db='sudo apt-get -y update' 35 | # command to set os package manager proxy 36 | cmd_set_repo_proxy="echo 'proxy=http://${http_proxy}/' | sudo tee -a /etc/yum.conf" 37 | # config script to fixup repos to properly use proxycaches 38 | cmd_set_repo_conf="sudo sed -i -e 's/#baseurl=/baseurl=/' /etc/yum.repos.d/* && sudo sed -i -e 's/mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/*" 39 | # command to set the proxy for the whole OS 40 | cmd_set_os_proxy="echo -n http_proxy=${http_proxy}\nhttps_proxy=${http_proxy}\nftp_proxy=${http_proxy} | sudo tee -a /etc/environment" 41 | # command to determine if the OS needs restarting after package updates 42 | cmd_get_os_needs_restarting='sudo stat /var/run/reboot-required' 43 | # command to reboot the system 44 | cmd_do_reboot='sudo reboot' 45 | # command to install a python PIP package 46 | cmd_in_pip_pkg='sudo pip install --upgrade' 47 | # command to download a python PIP package 48 | cmd_get_pip_pkg='pip download' 49 | -------------------------------------------------------------------------------- /bootstrap_plugins/us_en.i18n.plugin.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2015 EMC Corporation 3 | # All Rights Reserved 4 | # 5 | # This software contains the intellectual property of EMC Corporation 6 | # or is licensed to EMC Corporation from third parties. Use of this 7 | # software and the intellectual property contained therein is expressly 8 | # limited to the terms and conditions of the License Agreement under which 9 | # it is provided by or on behalf of EMC. 10 | 11 | error_building_ecs_install_image() { 12 | error "We couldn't build the ecs-install image for some reason. Please check the logs." 13 | error "If it's something simple, such as a missing base image (we use python:2.7-alpine)," 14 | error "then you may be able to get the image to build by pulling python:2.7-alpine from" 15 | error "a reliable source, such as DockerHub. If you specified a custom registry, then you" 16 | error "may need to first push the image into your registry to ensure it is available for" 17 | error "the build tool." 18 | die "If you still need more help after trying the above, you can reach us on github." 19 | } 20 | error_pulling_ecs_install_image() { 21 | error "We couldn't pull the ecs-install image for some reason. Please check the logs." 22 | error "If it's something simple, such as the ecs-install:latest image missing from" 23 | error "your custom Docker registry, or if your Internet access isn't working, then" 24 | error "you may be able to solve the problem by first solving one of the above issues." 25 | die "If you still need more help after trying the above, you can reach us on github." 26 | } 27 | error_pulling_ecs_software_image_from_registry() { 28 | error "We couldn't pull the software image for some reason. Since you're using a custom" 29 | error "registry, it may be that the image does not exist in your registry. Please ensure" 30 | error "the '${release_artifact}' image is present on your registry before trying again, or" 31 | error "perhaps you can simply pull the image directly from DockerHub." 32 | die "If you still need more help after trying the above, you can reach us on github." 33 | } 34 | error_pulling_ecs_software_image_from_dockerhub() { 35 | error "We couldn't pull the software image for some reason. It may be a temporary issue" 36 | error "or there may be an issue with your Internet access. You'll likely need to check" 37 | error "the error message from the Docker pull output (above) to see what's specifically" 38 | error "the problem." 39 | die "If you still need more help after trying the above, you can reach us on github." 40 | } 41 | -------------------------------------------------------------------------------- /contrib/scripts/fwd_settings.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2015 EMC Corporation 4 | # All Rights Reserved 5 | # 6 | # This software contains the intellectual property of EMC Corporation 7 | # or is licensed to EMC Corporation from third parties. Use of this 8 | # software and the intellectual property contained therein is expressly 9 | # limited to the terms and conditions of the License Agreement under which 10 | # it is provided by or on behalf of EMC. 11 | 12 | # Updates FirewallD settings with trusted ports in accordance with ECS requirements. 13 | 14 | tcpadd=( 2 25 111 80 389 443 636 1095 1096 1098 1198 1298 2049 2181 2889 3218 4443 5120 5123 7399 7400 7578 9010 9011 9020-9025 9028 9029 9040 9069 9091 9094-9101 9111 9201-9206 9208 9209 9212 9230 9250 9888 9898 ) 15 | udpadd=( 1095 1096 1098 1198 1298 3218 9091 9094 9100 9201 9203 9208 9209 9250 9888 ) 16 | 17 | systemctl enable firewalld 18 | systemctl start firewalld 19 | 20 | for port in "${tcpadd[@]}"; do 21 | firewall-cmd --permanent --add-port=$port/tcp 22 | done 23 | for port in "${udpadd[@]}"; do 24 | firewall-cmd --permanent --add-port=$port/udp 25 | done 26 | 27 | firewall-cmd --reload 28 | exit 0 29 | -------------------------------------------------------------------------------- /contrib/scripts/instant_docker_registry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (c) 2015 EMC Corporation 4 | # All Rights Reserved 5 | # 6 | # This software contains the intellectual property of EMC Corporation 7 | # or is licensed to EMC Corporation from third parties. Use of this 8 | # software and the intellectual property contained therein is expressly 9 | # limited to the terms and conditions of the License Agreement under which 10 | # it is provided by or on behalf of EMC. 11 | 12 | ### 13 | # Run it on your soon-to-be docker registry host for great success. 14 | ### 15 | 16 | if [ $UID != 0 ]; then 17 | echo "You need to be root for this to work." 18 | exit 1 19 | fi 20 | 21 | # Your SSL Certificate's common name (The name your host is known by, 22 | # typically an FQDN). 23 | FQDN=$(hostname -f) 24 | 25 | # Flag to restart docker 26 | FLAG_RESTART=false 27 | 28 | # This is the minimum registry version supporting _catalog in the v2 API. 29 | REGISTRY_VERSION="2" 30 | 31 | echo "Creating paths" 32 | for dir in certs store; do 33 | if ! [ -d /opt/registry/${dir} ]; then 34 | mkdir -p /opt/registry/${dir} 35 | FLAG_RESTART=true 36 | else 37 | echo "Reusing existing /opt/registry/${dir}" 38 | fi 39 | done 40 | 41 | if ! [ -f /opt/registry/certs/domain.key ] && ! [ -f /opt/registry/certs/domain.crt ]; then 42 | openssl req -newkey rsa:4096 -nodes -sha256 \ 43 | -keyout /opt/registry/certs/domain.key -x509 \ 44 | -days 9999 -out /opt/registry/certs/domain.crt \ 45 | -subj "/C=AQ/ST=local/L=local/O=local/OU=local/CN=${FQDN}" 46 | FLAG_RESTART=true 47 | else 48 | echo "Reusing existing /opt/registry/certs/domain.crt" 49 | fi 50 | 51 | if ! [ -d "/etc/docker/certs.d/${FQDN}:5000" ]; then 52 | mkdir -p "/etc/docker/certs.d/${FQDN}:5000" 53 | cp /opt/registry/certs/domain.crt \ 54 | "/etc/docker/certs.d/${FQDN}:5000/ca.crt" 55 | FLAG_RESTART=true 56 | else 57 | echo "Reusing existing /etc/docker/certs.d/${FQDN}:5000/ca.crt" 58 | fi 59 | 60 | # "The nice thing about standards is that you have so many to choose from; 61 | # furthermore, if you do not like any of them, you can just wait 62 | # for next year's model." -- -Andrew S. Tanenbaum 63 | if $FLAG_RESTART; then 64 | systemctl daemon-reload || true 65 | systemctl restart docker || restart docker || service docker restart 66 | systemctl status docker || status docker || service docker status 67 | sleep 5 68 | fi 69 | 70 | docker run \ 71 | -d \ 72 | -p 5000:5000 \ 73 | --restart=always \ 74 | --name registry \ 75 | -v /opt/registry/certs:/certs \ 76 | -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \ 77 | -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \ 78 | -v /opt/registry/store:/var/lib/registry \ 79 | registry:${REGISTRY_VERSION} 80 | 81 | docker pull hello-world:latest 82 | docker tag hello-world:latest ${FQDN}:5000/test/hello-world:test 83 | docker push ${FQDN}:5000/test/hello-world:test 84 | sleep 5 85 | curl --cacert /opt/registry/certs/domain.crt https://${FQDN}:5000/v2/_catalog 86 | -------------------------------------------------------------------------------- /contrib/sslproxycert/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the CA certificate for the internal EMC SSL proxy. If you're installing ECS Community edition on a system behind the EMC firewall, you will need to install this certificate in your list of trusted CAs. To do this on CentOS, do the following: 2 | 3 | ``` 4 | cp emc_ssl.pem /etc/pki/ca-trust/source/anchors/ 5 | update-ca-trust extract 6 | ``` 7 | 8 | Do this on each host and restart the docker process: 9 | 10 | ``` 11 | systemctl restart docker 12 | ``` 13 | 14 | -------------------------------------------------------------------------------- /contrib/sslproxycert/install-cert.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (c) 2015 EMC Corporation 4 | # All Rights Reserved 5 | # 6 | # This software contains the intellectual property of EMC Corporation 7 | # or is licensed to EMC Corporation from third parties. Use of this 8 | # software and the intellectual property contained therein is expressly 9 | # limited to the terms and conditions of the License Agreement under which 10 | # it is provided by or on behalf of EMC. 11 | 12 | 13 | if [[ $EUID -ne 0 ]]; then 14 | echo "You must run $0 as root:" 1>&2 15 | echo "\$ sudo $0 " 1>&2 16 | echo "default is 'emc_ssl.pem'" 1>&2 17 | exit 1 18 | fi 19 | 20 | cert=${1:-"emc_ssl.pem"} 21 | 22 | detect_os() { 23 | 24 | # detect Ubuntu and anything using LSB 25 | if [ -f /etc/lsb-release ]; then 26 | os=$(lsb_release -d -s) 27 | 28 | # detect Debian version 29 | elif [ -f /etc/debian_version ]; then 30 | os="debian $(cat /etc/debian_version)" 31 | 32 | # detect RHEL, CentOS, and variants 33 | elif [ -f /etc/redhat-release ]; then 34 | os=`cat /etc/redhat-release` 35 | 36 | # TODO: SuSE, Arch, etc... 37 | 38 | # fallback for unknowns, perhaps they can be added to the list. 39 | else 40 | os="$(uname -s) $(uname -r) $(cat /etc/*_ver* /etc/*-rel*)" 41 | fi 42 | 43 | # string to lowercase 44 | os="${os,,}" 45 | 46 | # output 47 | echo "$os" 48 | } 49 | 50 | ### 51 | case "$(detect_os)" in 52 | 53 | rhel*|centos*) 54 | cp "$cert" "/etc/pki/ca-trust/source/anchors/${cert}.crt" 55 | update-ca-trust extract 56 | ;; 57 | 58 | debian*|ubuntu*) 59 | cp "$cert" "/usr/local/share/ca-certificates/${cert}.crt" 60 | update-ca-certificates 61 | ;; 62 | 63 | *) 64 | echo "Auto install is not supported on this OS: $os" 65 | ;; 66 | esac 67 | 68 | echo "Reminder: restart any processes (eg. Docker) that depend on the local trust store." 69 | -------------------------------------------------------------------------------- /docs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/.gitkeep -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = ECS-CommunityEdition 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/design/bootstrapper.mmd: -------------------------------------------------------------------------------- 1 | Mind Map generated by NB MindMap plugin 2 | > __version__=`1.1`,showJumps=`true` 3 | --- 4 | 5 | # bootstrapper 6 | 7 | ## OSDetect 8 | 9 | ### lsb 10 | 11 | ### /etc/ 12 | 13 | ### uname 14 | 15 | ## UserEnvironment 16 | 17 | ### ecsdeploy symlinks 18 | 19 | ### paths 20 | 21 | ### virtualenv 22 | 23 | ## OS Plugins 24 | 25 | ### CentOS 7\.3 26 | 27 | ## SystemConfigurator 28 | 29 | ### http\_proxy 30 | 31 | ### local docker instance 32 | 33 | ### docker repo config 34 | 35 | ### SSL certificates 36 | 37 | ### base OS mirror 38 | 39 | ### Alpine mirror 40 | 41 | ### Base OS Mirror 42 | 43 | ## PackageConfigurator 44 | 45 | ### per\-OS package requirements list 46 | 47 | ### python package requirements list 48 | -------------------------------------------------------------------------------- /docs/design/deploy.yml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/design/deploy.yml.png -------------------------------------------------------------------------------- /docs/design/ecsdeploy.mmd: -------------------------------------------------------------------------------- 1 | Mind Map generated by NB MindMap plugin 2 | > __version__=`1.1`,showJumps=`true` 3 | --- 4 | 5 | # ecsdeploy 6 | 7 | ## On Host 8 | 9 | ### Deployment Configuration 10 | 11 | ### Caches 12 | 13 | ### Symlinks to utils in docker image 14 | - TOPIC 15 | 
1598759361AA
16 | 17 | ## In Docker Image 18 | > leftSide=`true` 19 | 20 | 21 | ### Ansible 22 | 23 | ### ReleaseConfiguration 24 | 25 | ### Director 26 | 27 | #### PersistentState 28 | 29 | #### Interfaces 30 | > topicLinkUID=`1598759361AA` 31 | 32 | 33 | ##### CLI 34 | 35 | ##### TUI 36 | 37 | #### AnsibleRunner 38 | -------------------------------------------------------------------------------- /docs/design/ecsmap.mmd: -------------------------------------------------------------------------------- 1 | Mind Map generated by NB MindMap plugin 2 | > __version__=`1.1`,showJumps=`true` 3 | --- 4 | 5 | # ECS 6 | - LINK 7 | 
https://www.emc.com/techpubs/api/ecs/v3-0-0-0/index.htm
8 | 9 | ## Replication Group 10 | > borderColor=`#FFFFFF`,leftSide=`true`,topicLinkUID=`154325755DEA` 11 | 12 | 13 | ## Virtual Data Center 14 | > borderColor=`#FFFFFF`,leftSide=`true`,topicLinkUID=`15432FF0944A` 15 | 16 | - TOPIC 17 | 
154325755DEA
18 | 19 | ## Storage Pool 20 | > borderColor=`#FFFFFF`,leftSide=`true` 21 | 22 | - TOPIC 23 | 
15432FF0944A
24 | 25 | ### Data Node 26 | > borderColor=`#FFFFFF`,fillColor=`#1576C0` 27 | 28 | 29 | ## Namespace 30 | - TOPIC 31 | 
154325755DEA
32 | 33 | ### Bucket 34 | - TOPIC 35 | 
1543305482BA
36 | 37 | ### Object User 38 | > topicLinkUID=`1543305482BA` 39 | 40 | - TOPIC 41 | 
15432581C8FA
42 | 43 | ## Authentication Provider 44 | > topicLinkUID=`15432581C8FA` 45 | 46 | 47 | ## System User 48 | - TOPIC 49 | 
15432581C8FA
50 | 51 | ## Configuration 52 | 53 | ### Properties 54 | 55 | #### user\-scope 56 | - TOPIC 57 | 
1543305482BA
58 | 59 | ### License 60 | 61 | ### Certificate 62 | 63 | ### Feature 64 | 65 | #### Server Side Encryption 66 | -------------------------------------------------------------------------------- /docs/design/paths.mmd: -------------------------------------------------------------------------------- 1 | Mind Map generated by NB MindMap plugin 2 | > __version__=`1.1`,showJumps=`true` 3 | --- 4 | 5 | # paths 6 | 7 | ## cache 8 | 9 | ### host 10 | 11 | #### /var/cache/emc 12 | 13 | ### image 14 | 15 | #### /opt/cache 16 | 17 | ## configs 18 | 19 | ### release config 20 | 21 | #### image 22 | 23 | ##### /usr/local/src/ui/etc 24 | 25 | ### deployment config 26 | 27 | #### host 28 | 29 | ##### /opt/emc/ecs\-install 30 | 31 | #### image 32 | 33 | ##### /opt 34 | 35 | ### state 36 | 37 | #### host 38 | 39 | ##### /opt/emc/ecs\-install 40 | 41 | #### image 42 | 43 | ##### /opt 44 | 45 | ## logs 46 | 47 | ### host 48 | 49 | #### /var/log/vipr/viprobjsvc 50 | 51 | ### image 52 | 53 | #### /var/log 54 | -------------------------------------------------------------------------------- /docs/legacy/changelog.md: -------------------------------------------------------------------------------- 1 | # See https://github.com/EMCECS/ECS-CommunityEdition/releases for changelog of current releases. 2 | 3 | These entries are included for historical record. 4 | 5 | ## Update 2017-04-30: v 3.0.0.1 (HF1) 6 | - Updated Docker image to [ECS Software v3.0 Hotfix 1]() 7 | - Go-live with Installer 2.0 and yaml-configured, Ansible-based deployment system 8 | - Legacy installer code moved to legacy/ directory. 9 | 10 | ## Update 2015-12-29: v.2.1.0.2 11 | - Updated Docker Image to [ECS Software v2.1 Hotfix 2](https://support.emc.com/docu62377_ECS_2.1_HF2_Readme.txt?language=en_US&language=en_US) 12 | - Various improvements to retry code 13 | - Changes to and fixes for VDC creation, esp. in multi-VDC builds 14 | - VDC Provisioning may now be deliberately omitted with -SkipVDCProvision 15 | - Modification of storage server (SSM) parameters to better support smaller disk configurations 16 | - Pre-existing docker images no longer removed during installation 17 | - Addition of systemd script for container start/stop 18 | - Fixes for import, formatting, and other assorted minor bugs 19 | 20 | ## Update 2015-11-30 21 | - Updated Docker Image to a [ECS Software v2.1 Hotfix 1](https://support.emc.com/docu62132_ECS_2.1_HF1_Readme.txt?language=en_US&language=en_US) 22 | - Users can now optionally specify docker image via command-line arguments in step1. 23 | - Installation script provides more inforamation to user, proceeds depending on services' availability. 24 | - Fix for authentication issues resulting from default provisioning. 25 | 26 | ## Update 2015-01-19: v2.2.0.0 27 | - Updated Docker Image to [ECS Software v2.2](https://support.emc.com/docu62941_ECS_2.2_Release_Notes.pdf?language=en_US&language=en_US) 28 | - **Note:** Due to export restrictions, ECS Community Edition does not include encryption functionality. 29 | - Updated install scripts to work with ECS 2.2. **Note:** if you want to install ECS 2.1, please download the install scripts for 2.1 from github. The changes to install 2.2 are not backward-compatible. 30 | -------------------------------------------------------------------------------- /docs/source/README.txt: -------------------------------------------------------------------------------- 1 | To update markdown (.md) documents to restructured (.rst) run the update-docs.sh script with ./update-docs.sh, 2 | you will be prompted to authenticate for sudo. 3 | 4 | This script downloads an Ubuntu-based docker image from Dockerhub with pandoc installed. 5 | When run the script uses git to find all changes to .md files and feeds them into the pandoc container. 6 | The container then spits out restructured versions of the markdown documents. 7 | 8 | NOTE: Be sure to run git status after running the script as new .rst files will be created if they do not already exist. 9 | These files will need to be added before committing the changes. 10 | -------------------------------------------------------------------------------- /docs/source/building/development-build.md: -------------------------------------------------------------------------------- 1 | # ECS Community Edition Installation 2 | 3 | ## Building from develop branch 4 | 5 | ### Prerequisites 6 | [same as standard install] 7 | 8 | ### Steps 9 | 10 | 11 | -------------------------------------------------------------------------------- /docs/source/building/development-build.rst: -------------------------------------------------------------------------------- 1 | ECS Community Edition Installation 2 | ================================== 3 | 4 | Building from develop branch 5 | ---------------------------- 6 | 7 | Prerequisites 8 | ~~~~~~~~~~~~~ 9 | 10 | [same as standard install] 11 | 12 | Steps 13 | ~~~~~ 14 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | Update 2017-04-30: v 3.0.0.1 (HF1) 4 | ---------------------------------- 5 | 6 | - Updated Docker image to `ECS Software v3.0 Hotfix 1 <>`__ 7 | - Go-live with Installer 2.0 and yaml-configured, Ansible-based 8 | deployment system 9 | - Legacy installer code moved to legacy/ directory. 10 | 11 | Update 2015-12-29: v.2.1.0.2 12 | ---------------------------- 13 | 14 | - Updated Docker Image to `ECS Software v2.1 Hotfix 15 | 2 `__ 16 | - Various improvements to retry code 17 | - Changes to and fixes for VDC creation, esp. in multi-VDC builds 18 | - VDC Provisioning may now be deliberately omitted with 19 | -SkipVDCProvision 20 | - Modification of storage server (SSM) parameters to better support 21 | smaller disk configurations 22 | - Pre-existing docker images no longer removed during installation 23 | - Addition of systemd script for container start/stop 24 | - Fixes for import, formatting, and other assorted minor bugs 25 | 26 | Update 2015-11-30 27 | ----------------- 28 | 29 | - Updated Docker Image to a `ECS Software v2.1 Hotfix 30 | 1 `__ 31 | - Users can now optionally specify docker image via command-line 32 | arguments in step1. 33 | - Installation script provides more inforamation to user, proceeds 34 | depending on services' availability. 35 | - Fix for authentication issues resulting from default provisioning. 36 | 37 | Update 2015-01-19: v2.2.0.0 38 | --------------------------- 39 | 40 | - Updated Docker Image to `ECS Software 41 | v2.2 `__ 42 | - **Note:** Due to export restrictions, ECS Community Edition does not 43 | include encryption functionality. 44 | - Updated install scripts to work with ECS 2.2. **Note:** if you want 45 | to install ECS 2.1, please download the install scripts for 2.1 from 46 | github. The changes to install 2.2 are not backward-compatible. 47 | 48 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. ECS Community Edition documentation master file, created by 2 | sphinx-quickstart on Mon Feb 6 20:03:00 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ECS Community Edition's documentation! 7 | ================================================= 8 | 9 | .. NOTE: keep this list in sync with manual/index.rst 10 | .. toctree:: 11 | :hidden: 12 | :maxdepth: 1 13 | 14 | building/building.rst 15 | installation/ECS-Installation.rst 16 | installation/Standard_Installation.rst 17 | installation/OVA_Installation.rst 18 | installation/Island_Installation.rst 19 | use/ECS-UI-Web-Interface.rst 20 | use/Migration.rst 21 | installation/deploy.yml.rst 22 | utilities/utilities.rst 23 | troubleshooting/ECS-Troubleshooting.rst 24 | troubleshooting/FAQ.rst 25 | 26 | 27 | .. include:: ../../README.rst 28 | :start-line: 0 29 | :end-before: Directory Structure 30 | -------------------------------------------------------------------------------- /docs/source/installation/ECS-Installation.md: -------------------------------------------------------------------------------- 1 | # ECS Community Edition Installation Guides 2 | 3 | For **Standard** installations (Internet connected, from source) use [this guide][standard-in]. 4 | 5 | For **Island** installations (Isolated environment, from source) use [this guide][island-in]. 6 | 7 | For **OVA** installations (connectivity agnostic, from OVA) use [this guide][ova-in]. 8 | 9 | For information on [deploy.yml][deploy_yml] file available options use [this guide][deploy_yml]. 10 | 11 | [standard-in]: Standard_Installation.md 12 | [island-in]: Island_Installation.md 13 | [ova-in]: OVA_Installation.md 14 | [deploy_yml]: deploy.yml.md 15 | -------------------------------------------------------------------------------- /docs/source/installation/ECS-Installation.rst: -------------------------------------------------------------------------------- 1 | ECS Community Edition Installation Guides 2 | ========================================= 3 | 4 | For **Standard** installations (Internet connected, from source) use 5 | `this guide `__. 6 | 7 | For **Island** installations (Isolated environment, from source) use 8 | `this guide `__. 9 | 10 | For **OVA** installations (connectivity agnostic, from OVA) use `this 11 | guide `__. 12 | 13 | For information on `deploy.yml `__ file available options 14 | use `this guide `__. 15 | -------------------------------------------------------------------------------- /docs/source/installation/public-key-authentication.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/installation/public-key-authentication.md -------------------------------------------------------------------------------- /docs/source/installation/public-key-authentication.rst: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/source/media/Create_replication_group.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/Create_replication_group.PNG -------------------------------------------------------------------------------- /docs/source/media/Dell_EMC_logo.svg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/Dell_EMC_logo.svg.png -------------------------------------------------------------------------------- /docs/source/media/create_namespace.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/create_namespace.PNG -------------------------------------------------------------------------------- /docs/source/media/create_object_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/create_object_user.png -------------------------------------------------------------------------------- /docs/source/media/create_object_user_keys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/create_object_user_keys.png -------------------------------------------------------------------------------- /docs/source/media/create_storage_vpool.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/create_storage_vpool.PNG -------------------------------------------------------------------------------- /docs/source/media/create_virtual_data_center.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/create_virtual_data_center.PNG -------------------------------------------------------------------------------- /docs/source/media/ecc-host-network-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecc-host-network-example.png -------------------------------------------------------------------------------- /docs/source/media/ecs-disk-install-step1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecs-disk-install-step1.PNG -------------------------------------------------------------------------------- /docs/source/media/ecs-disk-install-step2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecs-disk-install-step2.PNG -------------------------------------------------------------------------------- /docs/source/media/ecs-disk-install-step3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecs-disk-install-step3.PNG -------------------------------------------------------------------------------- /docs/source/media/ecs-disk-install-step4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecs-disk-install-step4.PNG -------------------------------------------------------------------------------- /docs/source/media/ecs-waiting-for-webserver.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/ecs-waiting-for-webserver.PNG -------------------------------------------------------------------------------- /docs/source/media/input_license.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/docs/source/media/input_license.PNG -------------------------------------------------------------------------------- /docs/source/troubleshooting/FAQ.md: -------------------------------------------------------------------------------- 1 | # Frequently Asked Questions 2 | 3 | ## Can I add storage to ECS-CommunityEdition after initializing an installation? 4 | 5 | No. Unfortunately because ECS Community Edition lacks the 'fabric' layer present in full featured ECS, it is not possible to add storage space after a completed installation. 6 | 7 | ## I am locked out of my ECS management console, can I reset the root password? 8 | 9 | Currently there is no procedure to reset the root password if you are locked out. 10 | 11 | ## The storage capacity statistics presented on the ECS dashboard seem wrong, what's going on here? 12 | 13 | ECS uses a data boxcarting strategy called "chunking". Chunks are pre-allocated when ECS is first initialized. When user data is written into ECS, pre-allocated chunks are filled and ECS pre-allocates however many new chunks ECS "thinks" would be best for the future based on what it knows about the past. 14 | 15 | Capacity statistics are calculated based on allocated and pre-allocated chunks at the time statistics are requested, and don't exactly reflect the actual amount of user data stored within ECS. We do this because it is a performance-enhancing heuristic that is a "good enough" representation of capacities without having to churn through the whole system to figure out the actual user data capacity numbers. In short, the numbers you are seeing are not designed to be exact, but are close estimates. 16 | 17 | ## Can I use a data store node as an NTP server for the rest of the nodes? 18 | 19 | No, this is not a supported deployment option. An external NTP server is required. 20 | 21 | ## My ECS functions but the storage pools never initialize. 22 | 23 | If you can store objects in buckets without issue, then it's likely that your storage pools and data stores are fully initialized. ECS Community Edition is a bit weird in that there are some UI/display issues with storage pools showing "Not Ready" and data stores showing "initializing" even after they have become fully initialized. If you can create VDCs, replication groups, namespaces, and buckets, then your storage pools are certainly initialized as those higher level abstractions require a working storage pool. -------------------------------------------------------------------------------- /docs/source/troubleshooting/FAQ.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | ========================== 3 | 4 | Can I add storage to ECS-CommunityEdition after initializing an installation? 5 | ----------------------------------------------------------------------------- 6 | 7 | No. Unfortunately because ECS Community Edition lacks the 'fabric' layer 8 | present in full featured ECS, it is not possible to add storage space 9 | after a completed installation. 10 | 11 | I am locked out of my ECS management console, can I reset the root password? 12 | ---------------------------------------------------------------------------- 13 | 14 | Currently there is no procedure to reset the root password if you are 15 | locked out. 16 | 17 | The storage capacity statistics presented on the ECS dashboard seem wrong, what's going on here? 18 | ------------------------------------------------------------------------------------------------ 19 | 20 | ECS uses a data boxcarting strategy called "chunking". Chunks are 21 | pre-allocated when ECS is first initialized. When user data is written 22 | into ECS, pre-allocated chunks are filled and ECS pre-allocates however 23 | many new chunks ECS "thinks" would be best for the future based on what 24 | it knows about the past. 25 | 26 | Capacity statistics are calculated based on allocated and pre-allocated 27 | chunks at the time statistics are requested, and don't exactly reflect 28 | the actual amount of user data stored within ECS. We do this because it 29 | is a performance-enhancing heuristic that is a "good enough" 30 | representation of capacities without having to churn through the whole 31 | system to figure out the actual user data capacity numbers. In short, 32 | the numbers you are seeing are not designed to be exact, but are close 33 | estimates. 34 | 35 | Can I use a data store node as an NTP server for the rest of the nodes? 36 | ----------------------------------------------------------------------- 37 | 38 | No, this is not a supported deployment option. An external NTP server is 39 | required. 40 | 41 | My ECS functions but the storage pools never initialize. 42 | -------------------------------------------------------- 43 | 44 | If you can store objects in buckets without issue, then it's likely that 45 | your storage pools and data stores are fully initialized. ECS Community 46 | Edition is a bit weird in that there are some UI/display issues with 47 | storage pools showing "Not Ready" and data stores showing "initializing" 48 | even after they have become fully initialized. If you can create VDCs, 49 | replication groups, namespaces, and buckets, then your storage pools are 50 | certainly initialized as those higher level abstractions require a 51 | working storage pool. 52 | -------------------------------------------------------------------------------- /docs/source/update-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dimage="captntuttle/ubuntu-pandoc" 4 | sts_msg="updating .rst for: " 5 | 6 | echo "building docker image $dimage from Dockerfile" 7 | 8 | for i in $(git diff --name-only --relative .); 9 | do 10 | if [[ $i == *.md ]]; then 11 | echo $sts_msg$i && sudo docker run \ 12 | -v `pwd`:/source $dimage -f markdown -t rst \ 13 | -o $(sed "s/.md/.rst/g" <<< "$i") $i; 14 | fi 15 | done 16 | 17 | 18 | -------------------------------------------------------------------------------- /docs/source/use/Migration.md: -------------------------------------------------------------------------------- 1 | # Migration 2 | 3 | ## General Cases 4 | 5 | Most migration cases can be handled by a great tool we wrote called ecs-sync, found [here](https://github.com/EMCECS/ecs-sync). 6 | 7 | ## HDFS 8 | 9 | An HDFS migration is possible with s3distcp or distcp. Please note that if using s3distcp with the s3a driver, it needs to be the latest version or you may run into issues. If using distcp, ECS's HCFS driver "viprfs" will need to be set up as a secondary FS and the distcp made from `hdfs://...` to `viprfs://...`. Instructions for installing the HCFS driver can be found [here](http://doc.isilon.com/ECS/3.0/DataAccessGuide/wwhelp/wwhimpl/js/html/wwhelp.htm#href=vipr_c_hdfs_ViPRHDFS_intro.html). 10 | 11 | -------------------------------------------------------------------------------- /docs/source/use/Migration.rst: -------------------------------------------------------------------------------- 1 | Migration 2 | ========= 3 | 4 | General Cases 5 | ------------- 6 | 7 | Most migration cases can be handled by a great tool we wrote called 8 | ecs-sync, found `here `__. 9 | 10 | HDFS 11 | ---- 12 | 13 | An HDFS migration is possible with s3distcp or distcp. Please note that 14 | if using s3distcp with the s3a driver, it needs to be the latest version 15 | or you may run into issues. If using distcp, ECS's HCFS driver "viprfs" 16 | will need to be set up as a secondary FS and the distcp made from 17 | ``hdfs://...`` to ``viprfs://...``. Instructions for installing the HCFS 18 | driver can be found 19 | `here `__. 20 | -------------------------------------------------------------------------------- /examples/local-lab-1-node/bootstrap-local.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2015 EMC Corporation 3 | # All Rights Reserved 4 | # 5 | # This software contains the intellectual property of EMC Corporation 6 | # or is licensed to EMC Corporation from third parties. Use of this 7 | # software and the intellectual property contained therein is expressly 8 | # limited to the terms and conditions of the License Agreement under which 9 | # it is provided by or on behalf of EMC. 10 | 11 | cd $HOME/ECS-CommunityEdition 12 | 13 | ./bootstrap.sh -y -b http://cache.local/alpine -g -r cache.local:5000 -d examples/local-lab-1-node/registry.crt -p cache.local:3128 -m cache.local -c examples/local-lab-1-node/deploy.yml 14 | -------------------------------------------------------------------------------- /examples/local-lab-1-node/registry.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFxTCCA62gAwIBAgIJAK6AEKqyudoxMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV 3 | BAYTAkFRMQ4wDAYDVQQIDAVsb2NhbDEOMAwGA1UEBwwFbG9jYWwxDjAMBgNVBAoM 4 | BWxvY2FsMQ4wDAYDVQQLDAVsb2NhbDEUMBIGA1UEAwwLY2FjaGUubG9jYWwxFDAS 5 | BgkqhkiG9w0BCQEWBWxvY2FsMB4XDTE2MDMwOTExMjc1MVoXDTQzMDcyNTExMjc1 6 | MVoweTELMAkGA1UEBhMCQVExDjAMBgNVBAgMBWxvY2FsMQ4wDAYDVQQHDAVsb2Nh 7 | bDEOMAwGA1UECgwFbG9jYWwxDjAMBgNVBAsMBWxvY2FsMRQwEgYDVQQDDAtjYWNo 8 | ZS5sb2NhbDEUMBIGCSqGSIb3DQEJARYFbG9jYWwwggIiMA0GCSqGSIb3DQEBAQUA 9 | A4ICDwAwggIKAoICAQDelCowQQLS4dQJWV/GymmTvbWAP/R6PARqF4mTKc+V89/W 10 | UFEart24+ZTToSVdWdtJLo4o7J6WAbMOgHC3+vrOUlRpdCZarfFdZZNNWVtwH1j/ 11 | G2QAGOgedvnjj+MiNjQ7KI/HS+6rUK1Skcqk4cr4CecDwj9zuqHLKObFnDt1QfsX 12 | AG0b/mjvJMX4RKo3UzxORSbvc9IB8mc7ZMVU3ZVlaHLiqB/4U6zffaGpiRSnwdTc 13 | Q6ascZZsMYRPYO2rTliJ3btk2L1ttGgj4SqcJJkCoeI01rBqdLsA9uPgEyygDDSv 14 | Q1+sD4dbNRanPyYlWzb8NARken3K7MAqSoKYkCDOuUkxopsq4H+rKf4phX6Rtqv5 15 | F54cB+g8qPaUB/u1bEA3WLd736uyYqE+9WA8l2C9HmlNQZWN4u2m7JvAB4gbtTXx 16 | ppR0wJcXvfEhfKObrq/7Skb42Vo7hD9MR96NWTOc5CPTJXytA5kYrhgfi2G1Qzxe 17 | /+WW0hMqWY46YNtlul3LiSWBXeBjTSkeeHOoC++oTOXBfzPy65KiJlbDvqYNCz/q 18 | wKTfrHdqJy5n179Lru5o5HK4fo+W8qUfRYnjgHFcSmE5zwAqdaF1V//CMSJhMU2L 19 | thzMpfs3LUNlmRtkyq3sDWiWv+dGGoF957GtdAGMoIFmWFR4KQzN6luFJgEtMQID 20 | AQABo1AwTjAdBgNVHQ4EFgQU5RLauKTu0TMCnIRNCJf1vq6612gwHwYDVR0jBBgw 21 | FoAU5RLauKTu0TMCnIRNCJf1vq6612gwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B 22 | AQsFAAOCAgEAZVG+Ds0LT8vZlEN35hzED/UQplYcycMpsIO5VN343dicYPuTyFIx 23 | 1RNhZLLkw6BHJ4MYiAEJiU4D27Ry4gu5zAqEURJuTUxLc8m/bwCrg+pPOmtzeG6/ 24 | wrvJxhu/Wi6YfvTOQRLzYDOrA9bMY1OnRSi4A1hJtfrXco63o5BMQwjSUqJuJF99 25 | MTSMh28wJzUJlvZ13qXzpim100p18hw1r9LyJDNJDdWvu2GlEKKF/upYQnjbMJ6w 26 | X9myU3qHCmzcHESBOKqKnRRxRPjrVPF0vau3aaECI/Ev0nt45ir8dJuFZ4PriTCK 27 | XvAk61m9arRwdFMH23PDAUbPq0nkReDsf5L6m/bRJDU1tjy7as7/UZLY4Gixn/BA 28 | PObjmbm6cBzjYFoAHhKzZSvA5JVhcEhcQkUPFv3uXNRtniLyHUUc2J23RL8zinC6 29 | uqbpwszEqVh0F/Pv6n82Ie9ExFjCxP8Ph0gK5GarGsjVD7+rLkkMd9N7ciMoCPka 30 | wkHtXB9Ik3SGav4C/l1s3TnzgnKPxoffjYx92utYyp/HxLeRJAuBqkijbBQFmAAZ 31 | 9uk37+rVhDIuT+MtKUz6dj5TVrkbK/H1oBMXZLrlCxW3pJJBz4fJ61x6m+WO8jdg 32 | AAjbuJypsOJ2rBfeiIu75HxM6IYHSxK92boiIB4KQjiU2aKdHJgBlDg= 33 | -----END CERTIFICATE----- 34 | -------------------------------------------------------------------------------- /examples/local-lab-4-node/bootstrap-local.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2015 EMC Corporation 3 | # All Rights Reserved 4 | # 5 | # This software contains the intellectual property of EMC Corporation 6 | # or is licensed to EMC Corporation from third parties. Use of this 7 | # software and the intellectual property contained therein is expressly 8 | # limited to the terms and conditions of the License Agreement under which 9 | # it is provided by or on behalf of EMC. 10 | 11 | cd $HOME/ECS-CommunityEdition 12 | 13 | ./bootstrap.sh -y -v -b http://cache.local/alpine -g -o 192.168.2.2 -k contrib/sslproxycert/emc_ssl.pem -r cache.local:5000 -d examples/local-lab-1-node/registry.crt -p cache.local:3128 -m cache.local -c examples/local-lab-4-node/deploy.yml 14 | -------------------------------------------------------------------------------- /examples/local-lab-all-in-one/bootstrap-local.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2015 EMC Corporation 3 | # All Rights Reserved 4 | # 5 | # This software contains the intellectual property of EMC Corporation 6 | # or is licensed to EMC Corporation from third parties. Use of this 7 | # software and the intellectual property contained therein is expressly 8 | # limited to the terms and conditions of the License Agreement under which 9 | # it is provided by or on behalf of EMC. 10 | 11 | cd $HOME/ECS-CommunityEdition 12 | 13 | ./bootstrap.sh -y -g -r cache.local:5000 -d examples/local-lab-1-node/registry.crt -p cache.local:3128 -m cache.local -c examples/local-lab-aio/deploy.yml 14 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Two fixes to the default 2.2.0.0 image. 2 | FROM emccorp/ecs-software-2.2:2.2.0.0 3 | 4 | ADD storageos-blobclient.jar /opt/storageos/lib/ 5 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 6 | 7 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.0/storageos-blobclient.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/patches/2.x/2.2.0.0/storageos-blobclient.jar -------------------------------------------------------------------------------- /patches/2.x/2.2.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Two fixes to the default 2.2.0.0 image. 2 | FROM emccorp/ecs-software-2.2:2.2.0.1 3 | 4 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 5 | 6 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.1/Dockerfile.dare: -------------------------------------------------------------------------------- 1 | # Two fixes to the default 2.2.0.0 image. 2 | FROM emccorp/ecs-software-dare:2.2.0.1 3 | 4 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 5 | ADD storageos-ssm.jar /opt/storageos/lib/ 6 | 7 | # For vLab, disable transformsvc 8 | ADD storageos-dataservice /etc/init.d/ 9 | 10 | # Make vnest use separate thread pools 11 | RUN grep -l 'object.UseSeparateThreadPools' /opt/storageos/conf/*.object.properties | while read f; do sed -r -i "s/object.UseSeparateThreadPools\s*=\s*false/object.UseSeparateThreadPools=true/g" "$f"; done 12 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.1/storageos-ssm.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/patches/2.x/2.2.0.1/storageos-ssm.jar -------------------------------------------------------------------------------- /patches/2.x/2.2.0.3/Dockerfile: -------------------------------------------------------------------------------- 1 | # Four fixes to the default 2.2.0.3 image. 2 | FROM emcvipr/object:2.2.0.0-75761.8c6090f-reduced 3 | 4 | # Fix disk partitioning script 5 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 6 | # Make vnest use separate thread pools to prevent deadlock 7 | # ADD vnest.object.properties /opt/storageos/conf/ 8 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed '/properties id="serviceProperties"/a true' $f 9 | # Reduce timeout for systems that are shutdown for a while 10 | ADD coordinator.properties /opt/storageos/conf/ 11 | # Increase memory for transformsvc 12 | ADD transformsvc /opt/storageos/bin/ 13 | # Patch for small disk footprint disk allocation 14 | ADD storageos-ssm.jar /opt/storageos/lib/ 15 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.3/coordinator.properties: -------------------------------------------------------------------------------- 1 | # genconfig will set flag based on control node vs. data node 2 | coordinator.useExternalConfig=true 3 | coordinator.serverClientPort=2181 4 | coordinator.dataDir=/data/zk 5 | coordinator.serverConfigFile=/data/vnest/vnest-main/configuration/coord-server 6 | coordinator.clientConfigFile=/data/vnest/vnest-main/configuration/coord-client 7 | coordinator.lastServedFile=/data/vnest/vnest-main/configuration/coord-last-served 8 | # if node down for this long, wait to see if still member before proceeding 9 | coordinator.lastServedThreshSecs=3600 10 | # if lastServedThreshSecs exceeded, wait this long for config to be updated 11 | coordinator.lastServedTimeoutSecs=5 12 | # how often to touch last served file 13 | coordinator.lastServedUpdateSecs=120 14 | -------------------------------------------------------------------------------- /patches/2.x/2.2.0.3/storageos-ssm.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/patches/2.x/2.2.0.3/storageos-ssm.jar -------------------------------------------------------------------------------- /patches/2.x/2.2.1.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Four fixes to the default 2.2.1.0 image. 2 | FROM emcvipr/object:2.2.1.0-77331.4f57cc6-reduced 3 | 4 | # Reduce timeout for systems that are shutdown for a while 5 | ADD coordinator.properties /opt/storageos/conf/ 6 | # Increase memory for transformsvc 7 | ADD transformsvc /opt/storageos/bin/ 8 | 9 | # Fix disk partitioning script 10 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 11 | 12 | # Make vnest use separate thread pools to prevent deadlock 13 | #ADD vnest.object.properties /opt/storageos/conf/ 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Remove forced exit from systool 17 | ADD systool /etc/ 18 | -------------------------------------------------------------------------------- /patches/2.x/2.2.1.0/coordinator.properties: -------------------------------------------------------------------------------- 1 | # genconfig will set flag based on control node vs. data node 2 | coordinator.useExternalConfig=true 3 | coordinator.serverClientPort=2181 4 | coordinator.dataDir=/data/zk 5 | coordinator.serverConfigFile=/data/vnest/vnest-main/configuration/coord-server 6 | coordinator.clientConfigFile=/data/vnest/vnest-main/configuration/coord-client 7 | coordinator.lastServedFile=/data/vnest/vnest-main/configuration/coord-last-served 8 | # if node down for this long, wait to see if still member before proceeding 9 | coordinator.lastServedThreshSecs=3600 10 | # if lastServedThreshSecs exceeded, wait this long for config to be updated 11 | coordinator.lastServedTimeoutSecs=5 12 | # how often to touch last served file 13 | coordinator.lastServedUpdateSecs=120 14 | -------------------------------------------------------------------------------- /patches/2.x/2.2.1.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 2.2.1.0 image. 2 | FROM emcvipr/object:2.2.1.0-77706.493e577-reduced 3 | 4 | # Reduce timeout for systems that are shutdown for a while 5 | ADD coordinator.properties /opt/storageos/conf/ 6 | # Increase memory for transformsvc 7 | ADD transformsvc /opt/storageos/bin/ 8 | 9 | # Fix disk partitioning script 10 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 11 | 12 | # Make vnest use separate thread pools to prevent deadlock 13 | #ADD vnest.object.properties /opt/storageos/conf/ 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Increase memory for transformsvc 17 | ADD transformsvc /opt/storageos/bin/ 18 | # Patch for small disk footprint disk allocation 19 | ADD storageos-ssm.jar /opt/storageos/lib/ 20 | 21 | # Remove forced exit from systool 22 | ADD systool /etc/ 23 | -------------------------------------------------------------------------------- /patches/2.x/2.2.1.1/coordinator.properties: -------------------------------------------------------------------------------- 1 | # genconfig will set flag based on control node vs. data node 2 | coordinator.useExternalConfig=true 3 | coordinator.serverClientPort=2181 4 | coordinator.dataDir=/data/zk 5 | coordinator.serverConfigFile=/data/vnest/vnest-main/configuration/coord-server 6 | coordinator.clientConfigFile=/data/vnest/vnest-main/configuration/coord-client 7 | coordinator.lastServedFile=/data/vnest/vnest-main/configuration/coord-last-served 8 | # if node down for this long, wait to see if still member before proceeding 9 | coordinator.lastServedThreshSecs=3600 10 | # if lastServedThreshSecs exceeded, wait this long for config to be updated 11 | coordinator.lastServedTimeoutSecs=5 12 | # how often to touch last served file 13 | coordinator.lastServedUpdateSecs=120 14 | -------------------------------------------------------------------------------- /patches/3.0.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.0HF image. 2 | FROM emcvipr/object:3.0.0.0-86239.1c9e5ec-reduced 3 | 4 | # Increase memory for transformsvc 5 | ADD transformsvc /opt/storageos/bin/ 6 | 7 | # Fix disk partitioning script 8 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 9 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 10 | 11 | # Make vnest use separate thread pools to prevent deadlock 12 | ADD vnest.object.properties /opt/storageos/conf/ 13 | 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | -------------------------------------------------------------------------------- /patches/3.0.0.2/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.0 HF2 reduced image. 2 | FROM emcvipr/object:3.0.0.0-86889.0a0ee19-reduced 3 | 4 | # Increase memory for transformsvc 5 | RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc 6 | 7 | # Fix disk partitioning script 8 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # COPY vnest-common-conf-template.xml /opt/storageos/conf/vnest-common-conf-template.xml 11 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | 13 | # Make vnest use separate thread pools to prevent deadlock 14 | RUN printf "\n# Use separate thread pools to prevent deadlock in vnest init\nobject.UseSeparateThreadPools=true\n" >> /opt/storageos/conf/vnest.object.properties 15 | 16 | -------------------------------------------------------------------------------- /patches/3.1.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.1.0.0 reduced image. 2 | 3 | # Build on RC4 object image (GA release) 4 | FROM emcvipr/object:3.1.0.0-95266.ab2753a-reduced 5 | 6 | # Increase memory for transformsvc 7 | RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc 8 | 9 | # Fix disk partitioning script 10 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 11 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 12 | 13 | # Set VNets useSeperateThreadPools to True 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 17 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f 18 | 19 | # Configure CM Object properties: Disable minimum storage device count 20 | RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 21 | 22 | # Allow allocation of different blocks of a chunk to be stored on the same partition 23 | RUN sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml 24 | -------------------------------------------------------------------------------- /patches/3.1.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.1.0.1 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.1.0.1-95464.df479c1-reduced 5 | 6 | # Increase memory for transformsvc 7 | RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc 8 | 9 | # Fix disk partitioning script 10 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 11 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 12 | 13 | # Set VNets useSeperateThreadPools to True 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 17 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f 18 | 19 | # Configure CM Object properties: Disable minimum storage device count 20 | RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 21 | 22 | # Allow allocation of different blocks of a chunk to be stored on the same partition 23 | RUN sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml 24 | -------------------------------------------------------------------------------- /patches/3.1.0.2/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.1.0.2 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.1.0.2-95467.5ec917e-reduced 5 | 6 | # Increase memory for transformsvc 7 | RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc 8 | 9 | # Fix disk partitioning script 10 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 11 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 12 | 13 | # Set VNets useSeperateThreadPools to True 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 17 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f 18 | 19 | # Configure CM Object properties: Disable minimum storage device count 20 | RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 21 | 22 | # Allow allocation of different blocks of a chunk to be stored on the same partition 23 | RUN sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml 24 | -------------------------------------------------------------------------------- /patches/3.1.0.3/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.1.0.3 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.1.0.3-95716.cf3f257-reduced 5 | 6 | # Increase memory for transformsvc 7 | RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc 8 | 9 | # Fix disk partitioning script 10 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 11 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 12 | 13 | # Set VNets useSeperateThreadPools to True 14 | RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 15 | 16 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 17 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f 18 | 19 | # Configure CM Object properties: Disable minimum storage device count 20 | RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 21 | 22 | # Allow allocation of different blocks of a chunk to be stored on the same partition 23 | RUN sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml 24 | -------------------------------------------------------------------------------- /patches/3.1.0.3/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | IMAGE_REPO="emccorp/ecs-software-3.1.0" 3 | IMAGE_VERSION="3.1.0.3" 4 | 5 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 6 | -------------------------------------------------------------------------------- /patches/3.2.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.2.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.2.0.0-101423.d3b297f-reduced 5 | 6 | # Increase memory for transformsvc 7 | RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 8 | 9 | # Set memory for objcontrolsvc 10 | RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 11 | 12 | # Fix disk partitioning script 13 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh 14 | 15 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 16 | 17 | # Set VNest useSeperateThreadPools to True 18 | RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 19 | 20 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 21 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 22 | 23 | RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 24 | 25 | # Configure CM Object properties: Disable minimum storage device count 26 | RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 27 | 28 | # Allow allocation of different blocks of a chunk to be stored on the same partition 29 | RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 30 | 31 | -------------------------------------------------------------------------------- /patches/3.2.0.0/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | IMAGE_REPO="emccorp/ecs-software-3.2.0" 3 | IMAGE_VERSION="3.2.0.0" 4 | 5 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 6 | -------------------------------------------------------------------------------- /patches/3.2.0.0/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.2.0.0-101423.d3b297f-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.2.0" 3 | IMAGE_VERSION="3.2.0.0" 4 | -------------------------------------------------------------------------------- /patches/3.2.0.1/.dockerignore: -------------------------------------------------------------------------------- 1 | nile-object-reduced.txz 2 | build.sh 3 | image.conf 4 | -------------------------------------------------------------------------------- /patches/3.2.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.2.0.1 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.2.0.1-101633.6533e6c-reduced 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | 13 | # Allow allocation of different blocks of a chunk to be stored on the same partition 14 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 15 | 16 | ## Increase memory for transformsvc 17 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 18 | 19 | ## Set memory for objcontrolsvc 20 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 21 | 22 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 23 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 25 | 26 | # Configure CM Object properties: Disable minimum storage device count 27 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 28 | -------------------------------------------------------------------------------- /patches/3.2.0.1/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.2.0.1/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.2.0.1-101633.6533e6c-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.2.0" 3 | IMAGE_VERSION="3.2.0.1" 4 | -------------------------------------------------------------------------------- /patches/3.3.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.3.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.3.0.0-108986.c0575c7-reduced 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | 13 | # Allow allocation of different blocks of a chunk to be stored on the same partition 14 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 15 | 16 | ## Increase memory for transformsvc 17 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 18 | 19 | ## Set memory for objcontrolsvc 20 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 21 | 22 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 23 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 25 | 26 | # Configure CM Object properties: Disable minimum storage device count 27 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 28 | -------------------------------------------------------------------------------- /patches/3.3.0.0/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.3.0.0/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.3.0.0-109089.4bf8a5a-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.3.0" 3 | IMAGE_VERSION="3.3.0.0" 4 | -------------------------------------------------------------------------------- /patches/3.3.0.2/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.3.0.2 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emccorp/ecs-software-3.3.0:3.3.0.2 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | 13 | # Allow allocation of different blocks of a chunk to be stored on the same partition 14 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 15 | 16 | ## Increase memory for transformsvc 17 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 18 | 19 | ## Set memory for objcontrolsvc 20 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 21 | 22 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 23 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 25 | 26 | # Configure CM Object properties: Disable minimum storage device count 27 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 28 | -------------------------------------------------------------------------------- /patches/3.3.0.2/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.3.0.2/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.3.0.0-109089.4bf8a5a-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.3.0" 3 | IMAGE_VERSION="3.3.0.2" 4 | -------------------------------------------------------------------------------- /patches/3.4.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.3.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.4.0.0-113607.75b42b5-reduced 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | 13 | # Allow allocation of different blocks of a chunk to be stored on the same partition 14 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 15 | 16 | ## Increase memory for transformsvc 17 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 18 | 19 | ## Set memory for objcontrolsvc 20 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 21 | 22 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 23 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 25 | 26 | # Configure CM Object properties: Disable minimum storage device count 27 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 28 | -------------------------------------------------------------------------------- /patches/3.4.0.0/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.4.0.0/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.4.0.0-113607.75b42b5-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.4.0 " 3 | IMAGE_VERSION="3.4.0.0" 4 | -------------------------------------------------------------------------------- /patches/3.5.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.5.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.5.1.2-121905.b6ac907bc40-reduced 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | RUN f=/opt/storageos/conf/vnest-common-conf.xml; sed -i '547 i \ \ \ \ \ \ \ \ true' $f 13 | 14 | # Allow allocation of different blocks of a chunk to be stored on the same partition 15 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 16 | 17 | ## Increase memory for transformsvc 18 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 19 | 20 | ## Set memory for objcontrolsvc 21 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 22 | 23 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 25 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 26 | 27 | # Configure CM Object properties: Disable minimum storage device count 28 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 29 | -------------------------------------------------------------------------------- /patches/3.5.0.0/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.5.0.0/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.5.1.2-121905.b6ac907bc40-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.5.0" 3 | IMAGE_VERSION="3.5.1.2" 4 | -------------------------------------------------------------------------------- /patches/3.5.1.2/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.5.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.5.1.2-121905.b6ac907bc40-reduced 5 | 6 | # Fix disk partitioning script 7 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 8 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Set VNest useSeperateThreadPools to True 11 | #RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f 12 | RUN f=/opt/storageos/conf/vnest-common-conf.xml; sed -i '547 i \ \ \ \ \ \ \ \ true' $f 13 | 14 | # Allow allocation of different blocks of a chunk to be stored on the same partition 15 | #RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '' $f || sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml $f 16 | 17 | ## Increase memory for transformsvc 18 | #RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc 19 | 20 | ## Set memory for objcontrolsvc 21 | #RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc 22 | 23 | # Set georeceiver's initialBufferNumOnHeap to something smaller for CE 24 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f 25 | #RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '10' $f || sed -i 's#80#10#g' $f 26 | 27 | # Configure CM Object properties: Disable minimum storage device count 28 | #RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f 29 | -------------------------------------------------------------------------------- /patches/3.5.1.2/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.5.1.2/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.5.1.2-121905.b6ac907bc40-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.5.0" 3 | IMAGE_VERSION="3.5.1.2" 4 | -------------------------------------------------------------------------------- /patches/3.8.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.8.0.1 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.8.0.1-138598.3d5db7c96f2-reduced 5 | 6 | #Modified SS partition script for CE. 7 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 8 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | 10 | # Modify the SS partition script for CE. 11 | RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \ 12 | && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 13 | 14 | # Allow allocation of different blocks of a chunk to be stored on the same partition for CE. 15 | RUN f=/opt/storageos/conf/ssm-cf-conf.xml ; sed -i '187s/value="false"/value="true"/' $f 16 | 17 | #Modified dashboard for ECS CE. 18 | 19 | ADD app.d4e76e5fa6bc1dc79cf5.js /opt/storageos/ecsportal/public/scripts/ 20 | -------------------------------------------------------------------------------- /patches/3.8.0.1/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.8.0.1/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.8.0.1-138598.3d5db7c96f2-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.8.0" 3 | IMAGE_VERSION="3.8.0.1" 4 | -------------------------------------------------------------------------------- /patches/3.8.0.1/tune_sr: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | sudo docker exec -it ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.chunk.gc.btree.occupancy.min_major_interval --value 1 --reason "ecs-ce-tune" 3 | sudo docker exec -it ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.prtable.gc.btree.min_major_interval --value 1 --reason "ecs-ce-tune" 4 | sudo docker exec -it ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.prtable.gc.record_expiration --value "0 days" --reason "ecs-ce-tune" 5 | sudo docker exec -it ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.chunk.gc.journal.protection_period --value "0 days" --reason "ecs-ce-tune" 6 | -------------------------------------------------------------------------------- /patches/3.9.0.0/Dockerfile: -------------------------------------------------------------------------------- 1 | # Fixes to the default 3.9.0.0 reduced image. 2 | 3 | # Build on object-reduced image (GA release) 4 | FROM emcvipr/object:3.9.0.0-141228.b9e3c09a0d1-reduced 5 | 6 | #Modified SS partition script for CE. 7 | ADD storageserver-partition-config.sh /opt/storageos/bin/ 8 | RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh 9 | ADD rm /opt/storageos/bin/ 10 | RUN /usr/bin/chmod +x /opt/storageos/bin/rm 11 | ADD eventsvc /opt/storageos/bin/ 12 | RUN /usr/bin/chmod +x /opt/storageos/bin/eventsvc 13 | ADD storageos-dataservice /etc/init.d/storageos-dataservice 14 | RUN /usr/bin/chmod +x /etc/init.d/storageos-dataservice 15 | ADD shared-threadpool-conf.xml /opt/storageos/conf/shared-threadpool-conf.xml 16 | # Allow allocation of different blocks of a chunk to be stored on the same partition for CE. 17 | RUN f=/opt/storageos/conf/ssm-cf-conf.xml ; sed -i '200s/value="false"/value="true"/' $f 18 | #UI removal of monitoring 19 | ADD app.6fb9f5b2bd922a3cb087.js /opt/storageos/ecsportal/public/scripts/ 20 | -------------------------------------------------------------------------------- /patches/3.9.0.0/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source image.conf 3 | 4 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 5 | -------------------------------------------------------------------------------- /patches/3.9.0.0/image.conf: -------------------------------------------------------------------------------- 1 | BASE_IMAGE="emcvipr/object:3.9.0.0-141228.b9e3c09a0d1-reduced" 2 | IMAGE_REPO="emccorp/ecs-software-3.9.0" 3 | IMAGE_VERSION="3.9.0.0" 4 | -------------------------------------------------------------------------------- /patches/build-ecs-software.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -z "$*" ]; then 4 | echo "Usage: $0 " 5 | echo "Example: $0 3.2.0.0" 6 | exit 1 7 | fi 8 | 9 | IMAGE_DIR=$1 10 | IMAGE_CONF="${IMAGE_DIR}/image.conf" 11 | if ! [ -f "${IMAGE_CONF}" ]; then 12 | echo "Unable to open ${IMAGE_CONF}" 13 | exit 1 14 | fi 15 | 16 | source "${IMAGE_CONF}" 17 | 18 | bash libexec/gen_dockerfile.sh "${BASE_IMAGE}" "${IMAGE_DIR}/Dockerfile" 19 | cp libexec/patch_script.sh "${IMAGE_DIR}" 20 | cd "${IMAGE_DIR}" 21 | docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" . 22 | cd - 23 | -------------------------------------------------------------------------------- /patches/libexec/gen_dockerfile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # this script should be used to generate an overlay Dockerfile, which may be applied on top of an object container image, reducing various parameters. 4 | 5 | if [ "$#" -lt 2 ]; then 6 | echo "Usage: $0 " 7 | echo "Example: $0 emcvipr/object:3.2.0.0-101423.d3b297f-reduced /path/to/Dockerfile" 8 | exit 1 9 | fi 10 | 11 | obj_image=$1 12 | df=$2 13 | 14 | echo "$0 creating $df to build reduced image from $obj_image" 15 | 16 | cat>$df<" 18 | echo "Here is a list of hosts you can query:" 19 | echo "Data Node(s):" 20 | ansible --list-hosts data_node 21 | echo "Install Node:" 22 | ansible --list-hosts install_node 23 | } 24 | 25 | if ! [ -z "${host}" ]; then 26 | if [ -f "${facts}/${host}" ]; then 27 | jq -CS . "${facts}/${host}" | less -R 28 | else 29 | echo "host '${host}' does not (yet) have facts registered in the fact cache" 30 | exit 1 31 | fi 32 | else 33 | usage 34 | exit 1 35 | fi 36 | 37 | exit 0 38 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_access.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node:install_node 3 | gather_facts: false 4 | vars: 5 | ansible_ssh_pipelining: false 6 | vars_files: 7 | - vars/{{auth_type}}_auth.yml 8 | tasks: 9 | - ping: "{{inventory_hostname}}" 10 | 11 | - name: Installer | Gather facts and slice into OS groups 12 | hosts: data_node:install_node 13 | gather_facts: true 14 | vars: 15 | ansible_ssh_pipelining: false 16 | vars_files: 17 | - vars/{{auth_type}}_auth.yml 18 | tasks: 19 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 20 | 21 | - name: CentOS 7 | Configure access 22 | hosts: os_CentOS_7 23 | roles: 24 | - CentOS_7_configure_ssh 25 | - firewalld_configure_access 26 | 27 | - name: Common | Configure hostnames 28 | hosts: data_node 29 | roles: 30 | - common_set_hostname 31 | 32 | - name: Common | Configure /etc/hosts 33 | hosts: data_node:install_node 34 | roles: 35 | - common_etc_hosts 36 | 37 | - name: Common | Test inter-node access 38 | hosts: data_node:install_node 39 | roles: 40 | - common_access_test 41 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_access_host.yml: -------------------------------------------------------------------------------- 1 | - name: Installer | Setup access between ecs-install and install node 2 | hosts: install_node 3 | vars: 4 | ansible_ssh_pipelining: false 5 | vars_files: 6 | - vars/{{auth_type}}_auth.yml 7 | tasks: 8 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 9 | 10 | - name: Installer | Create SSH keys 11 | hosts: ecs_install 12 | roles: 13 | - installer_generate_ssh_keys 14 | 15 | - name: CentOS 7 | Setup SSH on install node 16 | hosts: os_CentOS_7 17 | vars: 18 | ansible_ssh_pipelining: false 19 | vars_files: 20 | - vars/{{auth_type}}_auth.yml 21 | roles: 22 | - CentOS_7_configure_ssh 23 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_bootstrap.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups for package install 8 | hosts: data_node 9 | gather_facts: True 10 | tasks: 11 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 12 | 13 | - name: CentOS 7 | Synchronize cache 14 | hosts: os_CentOS_7 15 | vars: 16 | os_tag: os_CentOS_7 17 | num_data_nodes: "{{ groups['data_node'] | length }}" 18 | top_data_node: "{{ groups['data_node'][0] }}" 19 | install_node: "{{ groups['install_node'][0] }}" 20 | roles: 21 | - CentOS_7_sync_caches_prep 22 | - common_sync_caches_by_torrent 23 | 24 | - name: CentOS 7 | Install cached packages 25 | hosts: os_CentOS_7 26 | vars: 27 | os_tag: os_CentOS_7 28 | num_data_nodes: "{{ groups['data_node'] | length }}" 29 | top_data_node: "{{ groups['data_node'][0] }}" 30 | install_node: "{{ groups['install_node'][0] }}" 31 | roles: 32 | - CentOS_7_baseline_install 33 | 34 | - name: Installer | Disable torrent ffx 35 | hosts: ecs_install 36 | vars: 37 | num_data_nodes: "{{ groups['data_node'] | length }}" 38 | top_data_node: "{{ groups['data_node'][0] }}" 39 | install_node: "{{ groups['install_node'][0] }}" 40 | tasks: 41 | - file: 42 | path: "{{ffx_sem}}" 43 | state: absent 44 | when: not ( num_data_nodes|int == 1 and top_data_node == install_node ) 45 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_cache.yml: -------------------------------------------------------------------------------- 1 | - name: Installer | Build the package cache 2 | vars: 3 | num_data_nodes: "{{ groups['data_node'] | length }}" 4 | top_data_node: "{{ groups['data_node'][0] }}" 5 | install_node: "{{ groups['install_node'][0] }}" 6 | hosts: ecs_install 7 | roles: 8 | - installer_build_cache 9 | 10 | - name: Installer | Enable torrent ffx 11 | vars: 12 | num_data_nodes: "{{ groups['data_node'] | length }}" 13 | top_data_node: "{{ groups['data_node'][0] }}" 14 | install_node: "{{ groups['install_node'][0] }}" 15 | hosts: ecs_install 16 | tasks: 17 | - file: 18 | path: "{{ffx_sem}}" 19 | state: touch 20 | when: not ( num_data_nodes|int == 1 and top_data_node == install_node ) 21 | 22 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_deploy.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Common | Install ECS 8 | gather_facts: True 9 | hosts: data_node 10 | vars: 11 | num_data_nodes: "{{ groups['data_node'] | length }}" 12 | top_data_node: "{{ groups['data_node'][0] }}" 13 | install_node: "{{ groups['install_node'][0] }}" 14 | roles: 15 | - common_baseline_install 16 | - common_deploy 17 | 18 | - name: Installer | Slice nodes into OS groups for init scripts 19 | hosts: data_node 20 | gather_facts: True 21 | tasks: 22 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 23 | 24 | - name: CentOS 7 | Install init scripts 25 | gather_facts: True 26 | hosts: os_CentOS_7 27 | roles: 28 | - CentOS_7_service_policies 29 | 30 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_disable_cache.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups 8 | hosts: data_node 9 | tasks: 10 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 11 | 12 | - name: Common | Disable cache handling 13 | hosts: data_node 14 | roles: 15 | - common_disable_cache 16 | 17 | - name: Installer | Disable torrent ffx 18 | hosts: ecs_install 19 | tasks: 20 | - file: 21 | path: "{{ffx_sem}}" 22 | state: absent 23 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_enable_cache.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups 8 | hosts: data_node 9 | tasks: 10 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 11 | 12 | - name: Common | Enable cache handling 13 | hosts: data_node 14 | roles: 15 | - common_enable_cache 16 | 17 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_preflight.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups 8 | hosts: data_node 9 | tasks: 10 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 11 | 12 | - name: Installer | Perform preflight check 13 | hosts: data_node 14 | roles: 15 | - common_collect_facts 16 | - common_baseline_check 17 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_purge.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups 8 | hosts: data_node 9 | tasks: 10 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 11 | 12 | - name: Common | Remove containers 13 | hosts: data_node 14 | roles: 15 | - common_purge_containers 16 | 17 | - name: Common | Cleanup Common items 18 | hosts: data_node 19 | roles: 20 | - common_purge_cleanup 21 | 22 | #- name: CentOS 7 | Cleanup OS-specific items 23 | # hosts: data_node:&os_CentOS_7 24 | # roles: 25 | # - CentOS_7_purge 26 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_reboot.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Installer | Slice nodes into OS groups 8 | hosts: data_node 9 | tasks: 10 | - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }} 11 | 12 | - name: CentOS 7 | Conditionally reboot nodes 13 | hosts: data_node:&os_CentOS_7 14 | roles: 15 | - CentOS_7_reboot 16 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_start.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Common | Start and enable ECS services 8 | hosts: data_node 9 | roles: 10 | - common_start_enable_service 11 | # - common_run_cf_client 12 | -------------------------------------------------------------------------------- /ui/ansible/clicmd_stop.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Ping data nodes before doing anything else 2 | hosts: data_node 3 | gather_facts: false 4 | tasks: 5 | - ping: "{{inventory_hostname}}" 6 | 7 | - name: Common | Stop and disable ECS services 8 | hosts: data_node 9 | roles: 10 | - common_stop_disable_service 11 | -------------------------------------------------------------------------------- /ui/ansible/group_vars/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/group_vars/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/group_vars/install_node.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ### 3 | flag_install_node: true 4 | -------------------------------------------------------------------------------- /ui/ansible/group_vars/os_CentOS_7.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ansible_ssh_common_args: -o IdentitiesOnly=yes -o PreferredAuthentications=publickey -o PubkeyAuthentication=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s 3 | disable_services: 4 | - mta 5 | enable_services: 6 | - docker 7 | - ntp 8 | - firewall 9 | service_names: 10 | ntp: ntpd 11 | docker: docker 12 | ssh: sshd 13 | nm: NetworkManager 14 | mta: postfix 15 | firewall: firewalld 16 | host_package_dir: /var/cache/yum 17 | -------------------------------------------------------------------------------- /ui/ansible/group_vars/os_CentOS_7_configure_access.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s 3 | -------------------------------------------------------------------------------- /ui/ansible/host_vars/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/host_vars/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/library/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/library/__init__.py -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_baseline_install/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: CentOS 7 | Update CA trust chain 3 | command: "update-ca-trust extract" 4 | 5 | - name: CentOS 7 | Restart Docker 6 | service: 7 | name: "{{ service_names.docker }}" 8 | state: restarted 9 | enabled: yes 10 | 11 | - name: CentOS 7 | Stop ntp 12 | service: 13 | name: "{{ service_names.ntp }}" 14 | state: stopped 15 | enabled: yes 16 | 17 | - name: CentOS 7 | Set the clock 18 | command: ntpd -gq 19 | 20 | - name: CentOS 7 | Start ntp 21 | service: 22 | name: "{{ service_names.ntp }}" 23 | state: started 24 | enabled: yes 25 | 26 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_baseline_install/templates/emcecs_ssl_proxy.pem.j2: -------------------------------------------------------------------------------- 1 | {{ mitm_proxy_cert }} 2 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_baseline_install/templates/firewalld.trusted.zone.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | Trusted 4 | All network connections are accepted. 5 | {# Round up all the IPv4 addresses Ansible knows about and whitelist them #} 6 | {% for host in groups['all'] | map('extract', hostvars, 'ansible_all_ipv4_addresses') | list %} 7 | 8 | {% endfor %} 9 | {# Decorate arbitrary IPv4s with /32 masks as a convenience feature #} 10 | {% for host in trusted_hosts %} 11 | 12 | {% endfor %} 13 | {# Add arbitrary prefixes #} 14 | {% for cidr in trusted_networks %} 15 | 16 | {% endfor %} 17 | 18 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_baseline_install/templates/ntp.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | # permit the source to query or modify the service on this system. 3 | restrict default nomodify notrap nopeer noquery 4 | 5 | # Permit all access over the loopback interface. This could 6 | # be tightened as well, but to do so would effect some of 7 | # the administrative functions. 8 | restrict 127.0.0.1 9 | restrict ::1 10 | 11 | # Hosts on local network are less restricted. 12 | #restrict 192.168.2.0 mask 255.255.255.0 nomodify notrap 13 | 14 | # Use NTP servers configured via ECS installer 15 | {% for server in ntp_servers -%} 16 | server {{ server }} iburst 17 | {% endfor %} 18 | 19 | #broadcast 192.168.2.255 autokey # broadcast server 20 | #broadcastclient # broadcast client 21 | #broadcast 224.0.1.1 autokey # multicast server 22 | #multicastclient 224.0.1.1 # multicast client 23 | #manycastserver 239.255.254.254 # manycast server 24 | #manycastclient 239.255.254.254 autokey # manycast client 25 | 26 | # Enable public key cryptography. 27 | #crypto 28 | 29 | includefile /etc/ntp/crypto/pw 30 | 31 | # Key file containing the keys and key identifiers used when operating 32 | # with symmetric key cryptography. 33 | keys /etc/ntp/keys 34 | 35 | # Specify the key identifiers which are trusted. 36 | #trustedkey 4 8 42 37 | 38 | # Specify the key identifier to use with the ntpdc utility. 39 | #requestkey 8 40 | 41 | # Specify the key identifier to use with the ntpq utility. 42 | #controlkey 8 43 | 44 | # Enable writing of statistics records. 45 | #statistics clockstats cryptostats loopstats peerstats 46 | 47 | # Disable the monitoring facility to prevent amplification attacks using ntpdc 48 | # monlist command when default restrict does not include the noquery flag. See 49 | # CVE-2013-5211 for more details. 50 | # Note: Monitoring will not be disabled with the limited restriction flag. 51 | disable monitor 52 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_baseline_install/templates/resolv.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | {% for server in dns_servers -%} 3 | nameserver {{ server }} 4 | {% endfor %} 5 | {% if dns_domain is defined -%} 6 | domain {{ dns_domain }} 7 | {% endif %} 8 | {% if dns_search is defined -%} 9 | search {{ dns_domain }} {% for path in dns_search %}{{ path }} {% endfor %} 10 | {% endif %} 11 | 12 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_configure_ssh/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: CentOS 7 | Restart sshd 3 | service: name="{{ service_names.ssh }}" state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_configure_ssh/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: CentOS 7 | Distribute ed25519 ssh key 2 | authorized_key: 3 | state: present 4 | user: "{{ ansible_user }}" 5 | key: "{{ lookup('file', '{{ssh_dir}}/ed25519.pub') }}" 6 | when: keys_ed25519.enabled == true 7 | 8 | - name: CentOS 7 | Distribute rsa ssh key 9 | authorized_key: 10 | state: present 11 | user: "{{ ansible_user }}" 12 | key: "{{ lookup('file', '{{ssh_dir}}/id_rsa.pub') }}" 13 | when: keys_rsa.enabled == true 14 | 15 | - name: CentOS 7 | Disable SSH UseDNS 16 | lineinfile: 17 | dest: /etc/ssh/sshd_config 18 | regexp: '^UseDNS no$' 19 | state: present 20 | line: 'UseDNS no' 21 | validate: '/usr/sbin/sshd -t -f %s' 22 | notify: CentOS 7 | Restart sshd 23 | tags: ssh 24 | 25 | - name: CentOS 7 | Disable requiretty 26 | lineinfile: 27 | dest: /etc/sudoers 28 | regexp: '^Defaults.*requiretty' 29 | state: absent 30 | validate: '/usr/sbin/visudo -cf %s' 31 | vars: 32 | ansible_ssh_pipelining: false 33 | tags: sudoers 34 | 35 | - name: CentOS 7 | Disable sudo password reverification for admin group 36 | lineinfile: 37 | dest: /etc/sudoers 38 | state: present 39 | regexp: '^%admin ALL\=' 40 | line: '%admin ALL=(ALL) NOPASSWD:ALL' 41 | validate: '/usr/sbin/visudo -cf %s' 42 | tags: sudoers 43 | 44 | - name: CentOS 7 | Disable sudo password reverification for wheel group 45 | lineinfile: 46 | dest: /etc/sudoers 47 | state: present 48 | regexp: '^%wheel.*ALL\=.*' 49 | line: '%wheel ALL=(ALL) NOPASSWD:ALL' 50 | validate: '/usr/sbin/visudo -cf %s' 51 | tags: sudoers 52 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_purge/tasks/main.yml: -------------------------------------------------------------------------------- 1 | k#- name: CentOS 7 | Destroy GPT on ECS block devices 2 | # shell: echo -e 'x\nz\nY\nY\n' | gdisk {{ item }} 3 | # with_items: 4 | # - "{{ ecs_block_devices }}" 5 | # ignore_errors: true 6 | # failed_when: false 7 | 8 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_reboot/handlers/main.yml: -------------------------------------------------------------------------------- 1 | - name: CentOS 7 | Reboot required 2 | debug: 3 | msg: "!!! SYSTEM {{ inventory_hostname }} MUST BE REBOOTED TO CONTINUE INSTALLATION !!!" 4 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_reboot/tasks/main.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: CentOS 7 | Check needs-restarting 3 | shell: /bin/needs-restarting -r 4 | register: needs_restarting 5 | ignore_errors: True 6 | failed_when: False 7 | changed_when: False 8 | when: ( ansible_local is defined and ansible_local.ova is not defined ) 9 | 10 | ## 776 : /sbin/dhclient 11 | #- name: "Debug the needs_restarting output" 12 | # debug: 13 | # msg: "{{ needs_restarting }}\n {{ needs_restarting.stdout.find('/') }}\n {{ needs_restarting.stdout | search('[0-9]* : /.*') }}" 14 | 15 | - name: CentOS 7 | Set reboot action on nodes that need restarting 16 | debug: 17 | msg: "Node flagged for reboot by package manager" 18 | when: 19 | - ( needs_restarting|failed ) and flag_install_node is not defined 20 | - not ( ansible_local is defined and ansible_local.ova is defined ) 21 | 22 | - name: CentOS 7 | Check if install node also needs restarting 23 | debug: 24 | msg: "The install node also needs restarting, but we can't reboot right now." 25 | notify: 26 | - CentOS 7 | Reboot required 27 | when: 28 | - ( needs_restarting|failed ) and flag_install_node is defined 29 | - not ( ansible_local is defined and ansible_local.ova is defined ) 30 | 31 | - name: CentOS 7 | Reboot node(s) 32 | become: True 33 | shell: sleep 5 && /sbin/shutdown -r now "Node flagged for reboot by package manager" 34 | async: 1 35 | poll: 0 36 | ignore_errors: True 37 | when: 38 | - ( needs_restarting|failed ) and flag_install_node is not defined 39 | - not ( ansible_local is defined and ansible_local.ova is defined ) 40 | 41 | - name: CentOS 7 | Wait for node(s) to reboot 42 | become: False 43 | local_action: wait_for host="{{ ansible_host | default(inventory_hostname) }}" port="{{ ansible_port }}" state=started delay=15 timeout=300 44 | when: 45 | - ( needs_restarting|failed ) and flag_install_node is not defined 46 | - not ( ansible_local is defined and ansible_local.ova is defined ) 47 | 48 | # host={{ ansible_default_ipv4.address }} port=22 state=started delay=60 timeout=120 49 | 50 | #- name: CentOS 7 | Reboot node 51 | # shell: sleep 5 && shutdown -r now "Node flagged for reboot by package manager" 52 | # become: yes 53 | # async: 1 54 | # poll: 0 55 | # ignore_errors: True 56 | # failed_when: False 57 | # changed_when: ( needs_restarting|failed ) and flag_install_node is not defined 58 | # 59 | #- name: CentOS 7 | Wait for node to reboot 60 | # wait_for: 61 | # port: 22 62 | # host: "{{ ansible_host | default(inventory_hostname) }}" 63 | # delay: 10 64 | # connect_timeout: 10 65 | # timeout: 300 66 | # state: started 67 | # delegate_to: "{{ groups['install_node'][0] }}" 68 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_service_policies/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: CentOS 7 | Insert ecs.service into systemd 2 | template: 3 | src: ecs.service.j2 4 | dest: /etc/systemd/system/ecs.service 5 | mode: 644 6 | owner: root 7 | group: root 8 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_service_policies/templates/ecs.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description={{ product_vendor }} {{ product_name }} {{ product_flavor }} {{ product_version }} 3 | Requires=docker.service 4 | After=docker.service 5 | 6 | [Service] 7 | Restart=always 8 | ExecStart=/usr/bin/docker start -a ecs-storageos 9 | ExecStop=/usr/bin/docker stop -t 60 ecs-storageos 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /ui/ansible/roles/CentOS_7_sync_caches_prep/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: CentOS 7 | Fetch torrent package names 2 | shell: find /var/cache/yum/ -name 'aria2*.x86_64.rpm' -o -name 'c-ares*.rpm' 3 | register: aria_packages 4 | delegate_to: "{{ groups['install_node'][0] }}" 5 | when: not ( num_data_nodes|int == 1 and top_data_node == install_node ) 6 | 7 | - name: CentOS 7 | Make yum cache paths for torrent packages 8 | shell: mkdir -p $(dirname {{item}}) 9 | with_items: "{{ aria_packages.stdout_lines }}" 10 | when: flag_install_node is not defined 11 | loop_control: 12 | label: "{{item}}" 13 | #creates: "{{host_cache_dir}}/disable_package_cache.sem" 14 | 15 | - name: CentOS 7 | Copy torrent packages to nodes 16 | copy: 17 | src: "{{item}}" 18 | dest: "{{item}}" 19 | with_items: "{{ aria_packages.stdout_lines }}" 20 | when: flag_install_node is not defined 21 | loop_control: 22 | label: "{{item}}" 23 | #creates: "{{host_cache_dir}}/disable_package_cache.sem" 24 | 25 | - name: CentOS 7 | Install torrent packages on nodes 26 | yum: 27 | disable_gpg_check: yes 28 | name: "{{item}}" 29 | state: installed 30 | update_cache: no 31 | with_items: "{{ aria_packages.stdout_lines }}" 32 | when: flag_install_node is not defined 33 | loop_control: 34 | label: "{{item}}" 35 | #creates: "{{host_cache_dir}}/disable_package_cache.sem" 36 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_access_test/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # Check network requirements 2 | # [X]Make sure DNS is setup forward and backward 3 | # We'll call hosts files good enough 4 | 5 | - name: Common | Check node connectivity by IP 6 | ping: "{{item}}" 7 | with_items: "{{ groups['data_node']|map('extract', hostvars, 'ansible_all_ipv4_addresses')|list }}" 8 | 9 | - name: Common | Check node connectivity by short name 10 | ping: "{{item}}" 11 | with_items: "{{ groups['data_node']|map('extract', hostvars, 'ansible_hostname')|list }}" 12 | 13 | - name: Common | Check node connectivity by fqdn 14 | ping: "{{item}}" 15 | with_items: "{{ groups['data_node']|map('extract', hostvars, 'ansible_hostname')|list }}" 16 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/files/additional_prep.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | MOUNT=/bin/mount 4 | UMOUNT=/bin/umount 5 | GREP=/usr/bin/grep 6 | AWK=/usr/bin/awk 7 | DF=/bin/df 8 | FALLOCATE=/usr/bin/fallocate 9 | XFS_DB=/usr/sbin/xfs_db 10 | #XFS_CHECK=/usr/sbin/xfs_check 11 | XFS_REPAIR=/sbin/xfs_repair 12 | DD=/bin/dd 13 | 14 | # size of chunk file 15 | FILE_SIZE_GB=10 16 | 17 | function usage { 18 | echo "usage: $0 " 19 | exit 1 20 | } 21 | 22 | device=$1 23 | mount_point=`$MOUNT | $GREP $device | $AWK '{print $3}'` 24 | 25 | if [ -z "$device" -o -z "$mount_point" ]; then 26 | usage 27 | fi 28 | 29 | # mount file system to create files on it 30 | $MOUNT | $GREP $device -q 31 | if [ $? -eq 1 ]; then 32 | echo "mount $device" 33 | $MOUNT $device $mount_point 34 | fi 35 | 36 | # TODO: Update the docs to reflect this so users aren't surprised by capacity differences between block storage devices and ECS 37 | # NOTE: Sizes are in GiB = 2^30 or 1,073,741,824, rather than GB = 10^9 or 1,000,000,000 38 | # So keep this in mind when doing the maths 39 | num_files=$($DF -BG $mount_point | $GREP $mount_point | $AWK -v FGB=$FILE_SIZE_GB '{gsub("G", "", $4); print int($4 / FGB)}') 40 | 41 | # Remove one block so we can account for XFS reserved space. 42 | null_output=$((num_files--)) 43 | 44 | declare -a inodes 45 | for ((i=0;i<$num_files;i++)) { 46 | file=`printf "%04d\n" $i` 47 | echo "create file $file" 48 | $FALLOCATE -l ${FILE_SIZE_GB}G $mount_point/$file 49 | inodes[$i]=`ls -i $mount_point/$file | $AWK '{print $1}'` 50 | } 51 | 52 | # umount file system and force allocation of data blocks 53 | $UMOUNT $device 54 | 55 | for inode in ${inodes[@]}; do 56 | num_extents=`$XFS_DB -c "inode $inode" -c "print u.bmx" $device | $AWK '{if ($0 ~ "not found") {print 0} else {print NF - 3}}'` 57 | # inode locates on offset inode # * inode size (256 by default) on disk 58 | # and the first extent locates at offset 100 in the inode 59 | extent_offset=$((inode*256+100)) 60 | for ((i=0;i<$num_extents;i++)) { 61 | echo "set extent $i of inode $inode at $extent_offset" 62 | # the extent status flag is the first two MSB 63 | # as the file size is just a few GB, the rest bits in the first byte are all zeros 64 | $DD if=/dev/zero of=$device bs=1 count=1 seek=$extent_offset > /dev/null 2>&1 65 | 66 | # each extent is 16 bytes 67 | extent_offset=$((extent_offset+16)) 68 | } 69 | 70 | # show the extents after force allocation 71 | $XFS_DB -c "inode $inode" -c "print u.bmx" $device 72 | done 73 | 74 | $XFS_REPAIR -n $device 75 | if [ $? -ne 0 ]; then 76 | echo "fs check failed" 77 | exit 1 78 | fi 79 | 80 | # mount again, the file system is now ready 81 | $MOUNT $device 82 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/agent.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "endpoint": "https://{{ansible_fqdn}}:9240" 3 | } 4 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/id-old.json.j2: -------------------------------------------------------------------------------- 1 | {%- set spanner = joiner("-") -%} 2 | {%- set this_host = ansible_hostname -%} 3 | {%- set this_sp = hostvars[inventory_hostname]['sp'] -%} 4 | {%- set this_vdc = hostvars[inventory_hostname]['vdc'] -%} 5 | { 6 | "agent_id": "{{ this_host }}-{{ spanner() }}{{ hostvars[inventory_hostname]['group_names'] }}-{{ this_vdc }}" 7 | } 8 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/id.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "agent_id": "{{ ansible_local.data_node.node_uuid }}" 3 | } 4 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/object-main_network.json.j2: -------------------------------------------------------------------------------- 1 | {%- set comma = joiner(",") -%} 2 | {%- set vdc = hostvars[inventory_hostname]['vdc'] -%} 3 | { 4 | "cluster_info": [ 5 | {%- for node in groups['data_node'] -%} 6 | {%- if ( (hostvars[node]['vdc'] is defined) and 7 | (hostvars[node]['vdc'] == vdc) ) -%}{{ comma() }} 8 | { 9 | "network": { 10 | "mgmt_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 11 | "hostname": "{{hostvars[node].ansible_fqdn}}", 12 | "data_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}", 13 | "replication_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 14 | "data2_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}", 15 | "private_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}", 16 | "data2_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 17 | "private_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 18 | "replication_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}", 19 | "public_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 20 | "mgmt_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}", 21 | "data_ip": "{{hostvars[node].ansible_default_ipv4.address}}", 22 | "public_interface_name": "{{hostvars[node].ansible_default_ipv4.alias}}" 23 | }, 24 | "agent_endpoint": "https://{{hostvars[node].ansible_fqdn}}:9240", 25 | "agent_id": "{{hostvars[node].ansible_local.data_node.node_uuid}}" 26 | } 27 | {% endif %}{% endfor %} 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/rev0-network.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "private_interface_name": "{{ ansible_default_ipv4.alias }}", 3 | "public_interface_name": "{{ ansible_default_ipv4.alias }}", 4 | "hostname": "{{ ansible_hostname }}", 5 | "data_ip": "{{ ansible_default_ipv4.address }}", 6 | "mgmt_ip": "{{ ansible_default_ipv4.address }}", 7 | "replication_ip": "{{ ansible_default_ipv4.address }}" 8 | } 9 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/rev1-network.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "data_interface_name": "{{ansible_default_ipv4.alias}}", 3 | "mgmt_interface_name": "{{ansible_default_ipv4.alias}}", 4 | "hostname": "{{ansible_hostname}}", 5 | "replication_ip": "{{ansible_default_ipv4.address}}", 6 | "data2_interface_name": "{{ansible_default_ipv4.alias}}", 7 | "private_ip": "{{ansible_default_ipv4.address}}", 8 | "data_ip": "{{ansible_default_ipv4.address}}", 9 | "data2_ip": "{{ansible_default_ipv4.address}}", 10 | "public_ip": "{{ansible_default_ipv4.address}}", 11 | "replication_interface_name": "{{ansible_default_ipv4.alias}}", 12 | "public_interface_name": "{{ansible_default_ipv4.alias}}", 13 | "mgmt_ip": "{{ansible_default_ipv4.address}}", 14 | "private_interface_name": "{{ansible_default_ipv4.alias}}" 15 | } 16 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/templates/seeds.j2: -------------------------------------------------------------------------------- 1 | {%- set comma = joiner(",") -%} 2 | {%- set vdc = hostvars[inventory_hostname]['vdc'] -%} 3 | {%- for host in groups.data_node -%} 4 | {%- if ( (hostvars[host]['vdc'] is defined) and 5 | (hostvars[host]['vdc'] == vdc) ) -%} 6 | {{ comma() }}{{ hostvars[host]['ansible_default_ipv4']['address'] }} 7 | {%- endif -%} 8 | {%- endfor %} 9 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_baseline_install/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ecs_docker_dirs: 3 | - path: /ecs 4 | mode: 755 5 | owner: 444 6 | group: 444 7 | state: directory 8 | - path: /host 9 | mode: 755 10 | owner: 444 11 | group: 444 12 | state: directory 13 | - path: /host/data 14 | mode: 755 15 | owner: 444 16 | group: 444 17 | state: directory 18 | - path: /host/files 19 | mode: 755 20 | owner: 444 21 | group: 444 22 | state: directory 23 | - path: /data 24 | mode: 755 25 | owner: 444 26 | group: 444 27 | state: directory 28 | - path: /var/log/vipr/emcvipr-object 29 | mode: 755 30 | owner: 444 31 | group: 444 32 | state: directory 33 | - path: /data/is_community_edition 34 | mode: 755 35 | owner: 444 36 | group: 444 37 | state: touch 38 | ecs_docker_dirs_post: 39 | - path: /ecs 40 | mode: 755 41 | owner: 444 42 | group: 444 43 | state: directory 44 | recurse: yes 45 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_collect_facts/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: "Common | Create custom facts directory" 2 | file: 3 | path: "/etc/ansible/facts.d" 4 | state: "directory" 5 | 6 | - name: "Common | Insert data_node.fact file" 7 | template: 8 | src: data_node.fact.j2 9 | dest: /etc/ansible/facts.d/data_node.fact 10 | mode: 0755 11 | 12 | - name: "Common | Reload facts to pick up new items" 13 | setup: ~ 14 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_collect_facts/templates/data_node.fact.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ECS_BLOCK_DEVICES="{%- set comma = joiner(" ") -%} 4 | {%- for blockdev in ecs_block_devices -%} 5 | {{ comma() }}{{ blockdev }} 6 | {%- endfor %}" 7 | 8 | total_bytes = 0 9 | for blockdev in ${ECS_BLOCK_DEVICES}; do 10 | total_bytes="$(( ${total_bytes} + $(blockdev --getsize64 ${blockdev}) ))" 11 | done 12 | 13 | echo "{ 14 | \"ecs_block_devices\": \"${ECS_BLOCK_DEVICES}\", 15 | \"ecs_block_size\": \"${total_bytes}\", 16 | \"node_uuid\": \"$(uuidgen -t)\" 17 | }" 18 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_deploy/files/cron_patch_template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will attempt to install python requests module 4 | # and self-destruct if successful. 5 | # Once inserted into cron.hourly, it will be run at system firstboot 6 | # and then every hour. Odds are it won't run twice. 7 | 8 | provides="/tmp/postinstall_patch.sem" 9 | 10 | _execute_payload() { 11 | 12 | } 13 | 14 | if ! [ -f "$provides" ]; then 15 | 16 | my_name="/etc/cron.hourly/postinstall_patch" 17 | 18 | # Some env cron doesn't set 19 | PROFILEREAD=true 20 | ENV=/etc/bash.bashrc 21 | FROM_HEADER= 22 | PATH=/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin 23 | CPU=x86_64 24 | LANG=POSIX 25 | PYTHONSTARTUP=/etc/pythonstart 26 | OSTYPE=linux 27 | MACHTYPE=x86_64-suse-linux 28 | G_BROKEN_FILENAMES=1 29 | 30 | _execute_payload >> /host/postinstall.log 31 | 32 | # signal success 33 | touch "$provides" 34 | 35 | sleep 2 36 | 37 | fi 38 | 39 | [ -f "$provides" ] && rm "$my_name" 40 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_deploy/files/postinstall_patch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will attempt to install python requests module 4 | # and self-destruct if successful. 5 | # Once inserted into cron.hourly, it will be run at system firstboot 6 | # and then every hour. Odds are it won't run twice. 7 | 8 | provides="/tmp/postinstall_patch.sem" 9 | 10 | _execute_payload() { 11 | # Ensure /opt/storageos/bin/storageserver-partition-config.sh is executable 12 | # addresses upstream bug in D@RE image 13 | chmod -v +x /opt/storageos/bin/storageserver-partition-config.sh 14 | } 15 | 16 | if ! [ -f "$provides" ]; then 17 | 18 | my_name="/etc/cron.hourly/postinstall_patch" 19 | 20 | # Some env cron doesn't set 21 | PROFILEREAD=true 22 | ENV=/etc/bash.bashrc 23 | FROM_HEADER= 24 | PATH=/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin 25 | CPU=x86_64 26 | LANG=POSIX 27 | PYTHONSTARTUP=/etc/pythonstart 28 | OSTYPE=linux 29 | MACHTYPE=x86_64-suse-linux 30 | G_BROKEN_FILENAMES=1 31 | 32 | _execute_payload >> /host/postinstall.log 33 | 34 | # signal success 35 | touch "$provides" 36 | 37 | sleep 2 38 | 39 | fi 40 | 41 | [ -f "$provides" ] && rm "$my_name" 42 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_deploy/files/setup_python_requests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will attempt to install python requests module 4 | # and self-destruct if successful. 5 | # Once inserted into cron.hourly, it will be run at system firstboot 6 | # and then every hour. Odds are it won't run twice. 7 | 8 | provides="/usr/local/lib64/python2.6/site-packages/requests-2.9.1-py2.6.egg" 9 | 10 | _execute_payload() { 11 | ecs_python_dir="/host/ecs_python" 12 | requests="v2.9.1.tar.gz" 13 | requests_dir="requests-2.9.1" 14 | ezsetup="ez_setup.py" 15 | setuptools="setuptools-20.7.0.zip" 16 | 17 | cd "$ecs_python_dir" 18 | tar -xzf "$requests" 19 | python "$ezsetup" 2>&1 >/dev/null 20 | cd "$requests_dir" 21 | python setup.py -q install 22 | } 23 | 24 | if ! [ -d "$provides" ]; then 25 | 26 | my_name="/etc/cron.hourly/setup_python_requests" 27 | 28 | # Some env cron doesn't set 29 | PROFILEREAD=true 30 | ENV=/etc/bash.bashrc 31 | FROM_HEADER= 32 | PATH=/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin 33 | CPU=x86_64 34 | LANG=POSIX 35 | PYTHONSTARTUP=/etc/pythonstart 36 | OSTYPE=linux 37 | MACHTYPE=x86_64-suse-linux 38 | G_BROKEN_FILENAMES=1 39 | 40 | _execute_payload >> /host/postinstall.log 41 | 42 | sleep 2 43 | 44 | fi 45 | 46 | [ -d "$provides" ] && rm "$my_name" 47 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_deploy/templates/common.object.properties.j2: -------------------------------------------------------------------------------- 1 | # 2 | # common configs for dataservice across processes 3 | # 4 | 5 | # 6 | # ownership module 7 | # 8 | 9 | # num of directories per cos 10 | object.NumDirectoriesPerCoSForSystemDT={{ '32' if ansible_memtotal_mb < 65536 else '128' }} 11 | object.NumDirectoriesPerCoSForUserDT={{ '32' if ansible_memtotal_mb < 65536 else '128' }} 12 | # total # of hashes used 13 | object.TotalNumOfHashesForSystemDT=8192 14 | object.TotalNumOfHashesForUserDT=8192 15 | # max # of hashes used for spreading listing entries. 16 | # changing this number will require to redistribute the existing list keys. 17 | # Please DON'T change it. 18 | object.MaxHashesForListingPerParentForSystemDT=32 19 | object.MaxHashesForListingPerParentForUserDT=32 20 | 21 | 22 | object.LbEnabled=true 23 | object.FailoverEnabled=true 24 | 25 | #the period between successive executions of LivenessMonitor 26 | object.LivenessMonitorFrequencyMillis=7500 27 | 28 | # frequency of LB runs 29 | object.LoadBalanceFrequencyMillis=300000 30 | 31 | #frequency of ownership refresher runs 32 | object.OwnershipRefresherFrequencyMillis=1800000 33 | 34 | # max interval time between LB runs 35 | object.LoadBalanceMaxIntervalMillis=600000 36 | 37 | # initial delay of LB run after service starts up 38 | object.LoadBalanceInitDelayMillis=600000 39 | 40 | # num of LB runs per round 41 | object.MaxLbRunsPerRound=3 42 | 43 | # time out of ownership ping 44 | object.OwnershipPingTimeOutMillis=5000 45 | 46 | #default stale flag for bucket 47 | object.IsStaleAllowed=false 48 | 49 | 50 | object.Level2DTReInitWaitMinMillis = 10000 51 | object.Level2DTReInitWaitMaxMillis = 20000 52 | 53 | object.ChunkBroadcastQueryLimit = 500 54 | 55 | object.ChunkGcScanStatusBatchSize = 500 56 | object.ChunkGcScanStatusBatchPauseMillis = 500 57 | object.ChunkGcScanStatusGeoBatchSize = 1000 58 | object.ChunkGcScanStatusGeoBatchPauseMillis = 200 59 | object.ChunkGcScanStatusBatchChunkNumberMax = 20000 60 | object.ChunkGcScanStatusBatchChunkNumberMin = 1 61 | 62 | object.DtLockTimeOutMillis = 300000 63 | object.JournalLockTimeOutMillis = 1800000 64 | object.DtDumpThresholdForLB = 180 65 | object.MaxDumpTryBeforeForceLB = 2 66 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_deploy/vars/main.yml: -------------------------------------------------------------------------------- 1 | ecs_environnment: 2 | SS_GENCONFIG: 1 3 | data_container: ecs-config 4 | config_files: 5 | - /opt/storageos/conf/ssm.object.properties 6 | - /opt/storageos/conf/ssm-cf-conf.xml 7 | - /opt/storageos/conf/common.object.properties 8 | - /opt/storageos/conf/cm.object.properties 9 | - /opt/storageos/conf/cm-conf.xml 10 | - /opt/storageos/bin/dtquery 11 | - /opt/storageos/bin/transformsvc 12 | - /opt/storageos/bin/objcontrolsvc 13 | - /opt/storageos/bin/sr 14 | - /opt/storageos/bin/blobsvc 15 | - /opt/storageos/conf/georeceiver-conf.xml 16 | - /opt/storageos/conf/vnest-common-conf.xml 17 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_disable_cache/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Common | Disable datanode package cache handling 4 | file: 5 | path: "{{host_cache_dir}}/disable_package_cache.sem" 6 | state: touch 7 | 8 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_enable_cache/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Common | Enable datanode package cache handling 4 | file: 5 | path: "{{host_cache_dir}}/disable_package_cache.sem" 6 | state: absent 7 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_etc_hosts/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Add install node to /etc/hosts 2 | lineinfile: 3 | dest: /etc/hosts 4 | regexp: '.*{{ hostvars[item].ansible_hostname }}$' 5 | line: "{{hostvars[item].ansible_default_ipv4.address}} {{hostvars[item].ansible_fqdn}} {{hostvars[item].ansible_hostname}}" 6 | state: present 7 | with_items: "{{ groups.install_node }}" 8 | 9 | - name: Common | Add data nodes to /etc/hosts 10 | lineinfile: 11 | dest: /etc/hosts 12 | regexp: '.*{{ hostvars[item].ansible_hostname }}$' 13 | line: "{{hostvars[item].ansible_default_ipv4.address}} {{hostvars[item].ansible_fqdn}} {{hostvars[item].ansible_hostname}}" 14 | state: present 15 | with_items: "{{ groups.data_node }}" 16 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_export_ssl_pem/task/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Export ECS API SSL certificate 2 | shell: echo | openssl s_client -connect {{item}}:4443 2>/dev/null | openssl x509 -text >/opt/emc/ecs-install/ssl/{{item}}.crt 3 | 4 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_purge_containers/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Stop and disable ECS service if running 2 | service: 3 | name: ecs 4 | state: stopped 5 | enabled: no 6 | ignore_errors: true 7 | failed_when: false 8 | 9 | - name: Common | Delete ECS Docker container ecs-storageos 10 | shell: docker rm -fv ecs-storageos 11 | ignore_errors: true 12 | failed_when: false 13 | 14 | - name: Common | Delete ECS Docker container ecs-config 15 | shell: docker rm -fv ecs-config 16 | ignore_errors: true 17 | failed_when: false 18 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_run_cf_client/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Run cf_client in ecs-storageos container 2 | command: /usr/bin/docker exec ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.ssm.allocation.allowAllocationOnIgnoredPartitions --value true --reason allow 3 | 4 | #The reason is L1 DT table can’t allocate chunk from L2 tables and keep read only, then it block the system finish initialize. 5 | #The reason can’t allocate chunk is because we only has one partition (it may cause by MustHaveEnoughResources=true during installing, or there will 5 data partitions in a vm node). We need enable another flag in ssm to allow allocate all blocks in one partition. 6 | #luna:/opt/storageos/tools # ./cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.ssm.allocation.allowAllocationOnIgnoredPartitions --value true --reason allow 7 | #After change the value, I add VDC and RG manually from GUI. ECS works now. 8 | 9 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_set_hostname/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: autonames.yml 2 | 3 | - name: Common | Find node hostname 4 | set_fact: 5 | index_number: "{{ item.0 }}" 6 | index_hostname: "{{ autonames[item.0] }}.{{dns_domain}}" 7 | with_indexed_items: "{{groups.data_node}}" 8 | when: item.1 == inventory_hostname 9 | 10 | - name: Common | Set node hostname 11 | hostname: name="{{index_hostname}}" 12 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_start_enable_service/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Enable ECS service and start ECS 2 | service: 3 | enabled: yes 4 | name: ecs 5 | state: started 6 | # No sleep in systemd 7 | # sleep: 60 8 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_stop_disable_service/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Common | Disable ECS service and stop ECS 2 | service: 3 | enabled: no 4 | name: ecs 5 | sleep: 60 6 | state: stopped 7 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_sync_caches_by_copy/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Selinux | Set selinux permissive 4 | selinux: policy=targeted state=permissive 5 | 6 | - name: Common | Build cache paths 7 | vars: 8 | become: true 9 | file: 10 | state: directory 11 | path: "{{host_cache_dir}}/{{ item }}" 12 | with_items: "{{ caches }}" 13 | when: flag_install_node is not defined 14 | tags: files 15 | 16 | # This could be faster with synchronize (rsync) but 17 | - name: Common | Sync caches to nodes 18 | vars: 19 | become: true 20 | copy: 21 | src: "{{cache_dir}}/{{item.key}}/" 22 | dest: "{{host_cache_dir}}/{{item.key}}/" 23 | when: ( item.value.applies_to in group_names ) and ( flag_install_node is not defined )" 24 | with_dict: "{{ caches }}" 25 | -------------------------------------------------------------------------------- /ui/ansible/roles/common_sync_caches_by_torrent/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Selinux | Set selinux permissive 4 | selinux: policy=targeted state=permissive 5 | 6 | - name: Common | Create ecs-install host directory on nodes 7 | vars: 8 | become: true 9 | file: 10 | state: directory 11 | path: "{{host_root_dir}}" 12 | when: flag_install_node is not defined 13 | tags: files 14 | 15 | - name: Common | Create ecs-install cache directory on nodes 16 | vars: 17 | become: true 18 | file: 19 | state: directory 20 | path: "{{host_cache_dir}}" 21 | when: flag_install_node is not defined 22 | tags: files 23 | 24 | - name: Common | Copy cache torrent digest to nodes 25 | vars: 26 | become: true 27 | copy: 28 | src: "{{cache_dir}}/cache.torrent" 29 | dest: "{{host_cache_dir}}/cache.torrent" 30 | force: yes 31 | when: 32 | - flag_install_node is not defined 33 | - ( ansible_local is defined and ansible_local.ova is not defined ) 34 | #creates: "{{host_cache_dir}}/disable_package_cache.sem" 35 | 36 | - name: Common | Torrent sync caches with nodes 37 | vars: 38 | become: true 39 | shell: aria2c --file-allocation=trunc -T {{host_cache_dir}}/cache.torrent --bt-tracker-connect-timeout=1 --bt-tracker-interval=1 --bt-tracker-timeout=2 --listen-port=6883-6999 --enable-dht=true --dht-entry-point={{ groups['install_node'][0] }}:6882 --dht-message-timeout=2 --dht-listen-port=6882 --seed-time=0 --allow-overwrite=true --check-integrity --bt-enable-lpd=false && touch {{host_cache_dir}}/cache_control.sem 40 | args: 41 | chdir: "{{host_cache_dir}}/.." 42 | # creates: "{{host_cache_dir}}/cache_control.sem" 43 | creates: "{{host_cache_dir}}/disable_package_cache.sem" 44 | when: 45 | - flag_install_node is not defined 46 | - ( ansible_local is defined and ansible_local.ova is not defined ) 47 | -------------------------------------------------------------------------------- /ui/ansible/roles/firewalld_configure_access/handlers/main.yml: -------------------------------------------------------------------------------- 1 | - name: Firewalld | Reload Firewall 2 | service: 3 | name: firewalld 4 | state: restarted 5 | enabled: yes 6 | -------------------------------------------------------------------------------- /ui/ansible/roles/firewalld_configure_access/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Firewalld | Ensure service is started 2 | service: 3 | name: firewalld 4 | state: started 5 | 6 | - name: Firewalld | Add install node to firewalld trusted zone 7 | firewalld: 8 | source: "{{groups['install_node'][0]}}/32" 9 | state: enabled 10 | zone: trusted 11 | immediate: true 12 | permanent: true 13 | 14 | - name: Firewalld | Add all data nodes to firewalld trusted zone 15 | firewalld: 16 | source: "{{item}}/32" 17 | state: enabled 18 | zone: trusted 19 | immediate: true 20 | permanent: true 21 | with_items: "{{ groups['data_node'] | map('extract', hostvars, 'ansible_all_ipv4_addresses') | list }}" 22 | 23 | - name: Firewalld | Whitelist management prefixes 24 | firewalld: 25 | source: "{{item}}" 26 | state: enabled 27 | zone: trusted 28 | immediate: true 29 | permanent: true 30 | with_items: "{{management_clients}}" 31 | 32 | - name: Firewalld | Add all public service ports to firewalld public zone 33 | firewalld: 34 | state: enabled 35 | zone: public 36 | immediate: true 37 | permanent: true 38 | port: "{{item}}" 39 | with_items: "{{firewall_ports_service_clients}}" 40 | 41 | - name: Firewalld | Ensure service is started 42 | service: 43 | name: firewalld 44 | state: restarted 45 | enabled: yes 46 | -------------------------------------------------------------------------------- /ui/ansible/roles/installer_build_cache/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Installer | Create cache directories 4 | file: state=directory path={{ cache_dir }}/{{ item }} 5 | with_items: "{{ caches }}" 6 | when: 7 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 8 | - not ( ansible_local is defined and ansible_local.ova is defined ) 9 | loop_control: 10 | label: "{{ cache_dir }}/{{ item }}" 11 | 12 | - name: Installer | Create compressed cache files 13 | shell: "{{ item.value.pack_cmd }}" 14 | args: 15 | chdir: "{{ host_cache_dir }}/{{ item.key }}" 16 | creates: "{{ host_cache_dir }}/{{ item.key }}/{{ item.value.dest }}" 17 | with_dict: "{{ caches }}" 18 | delegate_to: "{{ groups['install_node'][0] }}" 19 | register: cacheresults 20 | when: 21 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 22 | - not ( ansible_local is defined and ansible_local.ova is defined ) 23 | loop_control: 24 | label: "{{ host_cache_dir }}/{{ item.key }}/{{ item.value.dest }}" 25 | 26 | - name: Installer | Create cache distribution torrent file 27 | shell: /usr/bin/mktorrent-borg -a udp://{{ groups['install_node'][0] }}:6881 -a http://{{ groups['install_node'][0] }}:6881/announce -o {{ cache_dir }}/cache.torrent {{ cache_dir }} 28 | args: 29 | creates: "{{ cache_dir }}/cache.torrent" 30 | when: 31 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 32 | - not ( ansible_local is defined and ansible_local.ova is defined ) 33 | -------------------------------------------------------------------------------- /ui/ansible/roles/installer_generate_ssh_keys/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Installer | Check RSA Keypair 2 | stat: path="{{keys_rsa.priv_key}}" 3 | register: rsa_keypair 4 | when: keys_rsa.enabled == True 5 | 6 | - name: Installer | Generate RSA Keypair 7 | command: "{{keys_rsa.keygen_cmd}} {{keys_rsa.priv_key}}" 8 | when: keys_rsa.enabled == True and ( not rsa_keypair.stat.exists ) 9 | 10 | - name: Installer | Check ed25519 Keypair 11 | stat: path="{{keys_ed25519.priv_key}}" 12 | register: ed25519_keypair 13 | when: keys_ed25519.enabled 14 | 15 | - name: Installer | Generate ed25519 Keypair 16 | command: "{{keys_ed25519.keygen_cmd}} {{keys_ed25519.priv_key}}" 17 | when: keys_ed25519.enabled and ( not ed25519_keypair.stat.exists ) 18 | 19 | - name: Installer | Fail when no crypto selected 20 | fail: 21 | msg: "No ssh pubkey auth crypto type selected. Check ssh_defaults/ssh_crypto in deploy.yml." 22 | when: ( not keys_rsa.enabled ) and ( not keys_ed25519.enabled ) 23 | 24 | - name: Installer | Ensure directory permissions on ssh keystore 25 | file: 26 | path: "{{item.key}}" 27 | state: "{{item.value.state}}" 28 | mode: "{{item.value.mode}}" 29 | owner: "{{item.value.owner}}" 30 | group: "{{item.value.group}}" 31 | with_dict: "{{ssh_dir_paths}}" 32 | loop_control: 33 | label: "{{item.key}}" 34 | -------------------------------------------------------------------------------- /ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml: -------------------------------------------------------------------------------- 1 | # This is all now in ui/ansible/templates/all.j2 2 | 3 | #keys_rsa: 4 | # keygen_cmd: ssh-keygen -N '' -t rsa -b 4096 -o -a 100 -f 5 | # priv_key: /opt/ssh/id_rsa 6 | # pub_key: /opt/ssh/id_rsa.pub 7 | #keys_ed25519: 8 | # keygen_cmd: ssh-keygen -N '' -t ed25519 -o -a 100 -f 9 | # priv_key: /opt/ssh/id_ed25519 10 | # pub_key: /opt/ssh/id_ed25519.pub 11 | #ssh_dir_paths: 12 | # /opt/ssh: 13 | # mode: 700 14 | # owner: root 15 | # group: root 16 | # state: directory 17 | # /opt/ssh/id_ed25519: 18 | # mode: 600 19 | # owner: root 20 | # group: root 21 | # state: file 22 | # /opt/ssh/id_rsa: 23 | # mode: 600 24 | # owner: root 25 | # group: root 26 | # state: file 27 | -------------------------------------------------------------------------------- /ui/ansible/roles/testing/files/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/roles/testing/files/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/roles/testing/tasks/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/roles/testing/tasks/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/roles/testing/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include_vars: caches.yml 2 | 3 | - name: Debug 4 | debug: 5 | msg: "data nodes: {{num_data_nodes|int}} top data node: {{top_data_node}} install node: {{groups['install_node'][0]}} is ova?: {{ansible_local is defined and ansible_local.ova is defined}}" 6 | 7 | - name: Debug 2 8 | debug: 9 | msg: "{{not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] )}}" 10 | 11 | - name: Debug 3 12 | debug: 13 | msg: "{{not ( ansible_local is defined and ansible_local.ova is defined )}}" 14 | 15 | - name: Debug 4 16 | debug: 17 | msg: "ansible_local: {{ansible_local is defined}} ansible_local.ova: {{ansible_local is defined and ansible_local.ova is defined}}" 18 | 19 | - name: Installer | Create cache directories 20 | file: state=directory path={{ cache_dir }}/{{ item }} 21 | with_items: "{{ caches }}" 22 | when: 23 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 24 | - not ( ansible_local is defined and ansible_local.ova is defined ) 25 | loop_control: 26 | label: "{{ cache_dir }}/{{ item }}" 27 | 28 | - name: Installer | Create compressed cache files 29 | shell: "{{ item.value.pack_cmd }}" 30 | args: 31 | chdir: "{{ host_cache_dir }}/{{ item.key }}" 32 | creates: "{{ host_cache_dir }}/{{ item.key }}/{{ item.value.dest }}" 33 | with_dict: "{{ caches }}" 34 | delegate_to: "{{ groups['install_node'][0] }}" 35 | register: cacheresults 36 | when: 37 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 38 | - not ( ansible_local is defined and ansible_local.ova is defined ) 39 | loop_control: 40 | label: "{{ host_cache_dir }}/{{ item.key }}/{{ item.value.dest }}" 41 | 42 | - name: Installer | Create cache distribution torrent file 43 | shell: /usr/bin/mktorrent-borg -ig 'facts*' -a udp://{{ groups['install_node'][0] }}:6881 -a http://{{ groups['install_node'][0] }}:6881/announce -o {{ cache_dir }}/cache.torrent -pub {{ cache_dir }} 44 | args: 45 | creates: "{{ cache_dir }}/cache.torrent" 46 | when: 47 | - not ( num_data_nodes|int == 1 and top_data_node == groups['install_node'][0] ) 48 | - not ( ansible_local is defined and ansible_local.ova is defined ) 49 | -------------------------------------------------------------------------------- /ui/ansible/roles/testing/templates/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/roles/testing/templates/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/roles/testing/vars/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/roles/testing/vars/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/templates/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/templates/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/templates/autonames.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | # Selected autonames: {{ facts['node_defaults']['autonaming'] }} 3 | autonames: 4 | {%- set autonaming = facts['node_defaults']['autonaming'] | default('moons') -%} 5 | {%- for name in autonames[autonaming] %} 6 | - {{ name -}} 7 | {% endfor %} 8 | # 9 | -------------------------------------------------------------------------------- /ui/ansible/templates/caches.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | caches: 3 | {%- for dir, cache in caches.items() %} 4 | {{ dir }}: 5 | {%- for key, value in cache.items() %} 6 | {{key}}: {{ value }} 7 | {%- endfor %} 8 | {%- endfor %} 9 | 10 | # 11 | -------------------------------------------------------------------------------- /ui/ansible/templates/dumpall.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | # Module Variables ("vars"): 3 | # -------------------------------- 4 | {{ vars | to_nice_yaml }} 5 | 6 | --- 7 | # Environment Variables ("environment"): 8 | # -------------------------------- 9 | {{ environment | to_nice_yaml }} 10 | 11 | --- 12 | # GROUP NAMES Variables ("group_names"): 13 | # -------------------------------- 14 | {{ group_names | to_nice_yaml }} 15 | 16 | --- 17 | # GROUPS Variables ("groups"): 18 | # -------------------------------- 19 | {{ groups | to_nice_yaml }} 20 | 21 | --- 22 | # HOST Variables ("hostvars"): 23 | # -------------------------------- 24 | {{ hostvars | to_nice_yaml }} 25 | -------------------------------------------------------------------------------- /ui/ansible/templates/dumpvars.yml.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_hostname }}: 2 | # -------------------------------- 3 | {% for item in hostvars[ansible_host] | list %} 4 | {{ item | to_nice_yaml }} 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /ui/ansible/templates/hosts.ini.j2: -------------------------------------------------------------------------------- 1 | ;---------------------------------------------------------------------------------- 2 | ; This file was generated by ecs-install and it may be best to leave it alone. 3 | ;---------------------------------------------------------------------------------- 4 | ; 5 | [data_node] 6 | ; A list of nodes on which to deploy ECS. 7 | ; 8 | ; Examples: 9 | ; 169.154.254.23 change_to=somehost.domain.tld 10 | ; myhostname change_to=mynewname 11 | ; neighbor.local ansible_python_interpreter=/usr/local/bin/python 12 | ; knownhost.domain.tld 13 | 14 | {% for node in facts.data_nodes -%} 15 | {{ node }} 16 | {% endfor %} 17 | 18 | ; If you want the installer node to be a data node in addition to it's 19 | ; installer role, it may be necessary to allocate additional storage resources 20 | ; to the installer node's '/var' or '/' partitions, depending on the node's 21 | ; filesystem layout. For example, if you have one filesystem on a single 22 | ; partition spanning the entire disk, then your storage partition is 23 | ; probably '/'. If you have multiple partitions for different kinds of storage 24 | ; (eg. '/var' for logs, '/usr' for programs, '/home' for userfiles) then you'll 25 | ; want to examine your filesystems to see where you can add additional space. 26 | ; 27 | ; For space estimates across different Linux distributions, please see the github page. 28 | 29 | [install_node] 30 | ; The node from which the installer and deployment system operates. This node also 31 | ; serves all packages upon which nodes in the deployment depend when working in 32 | ; isolated "island" environments. This behavior is configurable via 33 | ; island_environment: True 34 | ; 35 | {{ facts.install_node }} 36 | 37 | ;---------------------------------------------------------------------------------- 38 | ; This is the installer Docker image. Please don't fiddle with it unless you 39 | ; fully understand the consequences. 40 | ;---------------------------------------------------------------------------------- 41 | [ecs_install] 42 | localhost ansible_become=false ansible_connection=local ansible_python_interpreter=/usr/local/bin/python 43 | 44 | 45 | ;end 46 | 47 | -------------------------------------------------------------------------------- /ui/ansible/testing.yml: -------------------------------------------------------------------------------- 1 | - name: Installer | Build the package cache 2 | vars: 3 | num_data_nodes: "{{ groups['data_node'] | length }}" 4 | top_data_node: "{{ groups['data_node'][0] }}" 5 | install_node: "{{ groups['install_node'][0] }}" 6 | hosts: ecs_install 7 | roles: 8 | - installer_build_cache 9 | 10 | - name: Installer | Enable torrent ffx 11 | vars: 12 | num_data_nodes: "{{ groups['data_node'] | length }}" 13 | top_data_node: "{{ groups['data_node'][0] }}" 14 | install_node: "{{ groups['install_node'][0] }}" 15 | hosts: ecs_install 16 | tasks: 17 | - file: 18 | path: "{{ffx_sem}}" 19 | state: touch 20 | when: not ( num_data_nodes|int == 1 and top_data_node == install_node ) 21 | 22 | -------------------------------------------------------------------------------- /ui/ansible/vars/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/ansible/vars/.gitkeep -------------------------------------------------------------------------------- /ui/ansible/vars/password_auth.yml: -------------------------------------------------------------------------------- 1 | ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s 2 | -------------------------------------------------------------------------------- /ui/ansible/vars/ports.yml: -------------------------------------------------------------------------------- 1 | ecs_published_ports: 2 | # 25: 3 | # service: smtp 4 | 80: 5 | service: portal 6 | 111: 7 | service: nfs 8 | 443: 9 | service: portal 10 | 2049: 11 | service: nfs 12 | 3218: 13 | service: cas 14 | 4443: 15 | service: management_api 16 | 9020: 17 | service: s3 18 | 9021: 19 | service: s3 20 | 9022: 21 | service: atmos 22 | 9023: 23 | service: atmos 24 | 9024: 25 | service: swift 26 | 9025: 27 | service: swift 28 | 9040: 29 | service: hdfs 30 | 9094: 31 | service: replication 32 | 9095: 33 | service: replication 34 | 9096: 35 | service: replication 36 | 9097: 37 | service: replication 38 | 9098: 39 | service: replication 40 | 10000: 41 | service: nfs 42 | 10110: 43 | service: metering 44 | 64443: 45 | service: management_api 46 | 47 | services: 48 | # smtp: 49 | # name: SMTP test port - unused in ECS CE 50 | # description: testing only 51 | # issue: N/A 52 | nfs: 53 | name: ECS NFS 54 | description: ECS must be the sole NFS provider on this system 55 | issue: https://github.com/EMCECS/ECS-CommunityEdition/issues/179 56 | metering: 57 | name: ECS Metering 58 | descripion: Conflict between ECS Metering and IBM Tivoli Monitoring (k10agent) 59 | issue: https://github.com/EMCECS/ECS-CommunityEdition/issues/178 60 | portal: 61 | name: ECS Portal 62 | description: None 63 | issue: None 64 | management_api: 65 | name: ECS Management API 66 | description: None 67 | issue: None 68 | s3: 69 | name: ECS S3 Object API 70 | description: None 71 | issue: None 72 | atmos: 73 | name: ECS Atmos Object API 74 | description: None 75 | issue: None 76 | swift: 77 | name: ECS Swift Object API 78 | description: None 79 | issue: None 80 | hdfs: 81 | name: ECS HDFS Access Port 82 | description: None 83 | issue: None 84 | replication: 85 | name: ECS Geo and XOR replication ports 86 | description: None 87 | issue: None 88 | esrs: 89 | name: Dell EMC Secure Remote Service 90 | description: None 91 | issue: None 92 | cas: 93 | name: ECS CAS API endpoint 94 | description: None 95 | issue: None 96 | 97 | ecs_internal_ports: 98 | # - 25 99 | - 10111 100 | - 10114 101 | - 10115 102 | - 1095 103 | - 1096 104 | - 1098 105 | - 1198 106 | - 1298 107 | - 2181 108 | - 3218 109 | - 7399 110 | - 7400 111 | - 7443 112 | - 9010 113 | - 9011 114 | - 9028 115 | - 9029 116 | - 9040 117 | - 9069 118 | - 9091 119 | - 9099 120 | - 9100 121 | - 9101 122 | - 9201 123 | - 9202 124 | - 9203 125 | - 9204 126 | - 9205 127 | - 9209 128 | - 9211 129 | - 9212 130 | - 9888 131 | - 9898 132 | -------------------------------------------------------------------------------- /ui/ansible/vars/pubkey_auth.yml: -------------------------------------------------------------------------------- 1 | ansible_ssh_common_args: -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s 2 | -------------------------------------------------------------------------------- /ui/ecsremove.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | """ 4 | basic text-based init 5 | """ 6 | 7 | import logging 8 | import os 9 | import subprocess 10 | import shutil 11 | import sys 12 | import click 13 | import tui 14 | import yaml 15 | from sarge import Capture, run, shell_format, capture_both, get_both 16 | 17 | from tui.constants import * 18 | 19 | # TAG: DevTest 20 | # sys.dont_write_bytecode = True 21 | 22 | """ 23 | Globals 24 | """ 25 | 26 | 27 | """ 28 | # Logging 29 | """ 30 | 31 | 32 | logging.basicConfig(filename=ui_log, level=logging.DEBUG) 33 | logging.debug('-' * 40 + os.path.abspath(__file__) + '-' * 40) 34 | 35 | 36 | """ 37 | # Helpers 38 | """ 39 | 40 | 41 | def play(playbook, verbosity=0): 42 | 43 | if verbosity > 0: 44 | verb = "-{0}".format('v' * verbosity) 45 | cmd = shell_format('ansible-playbook {0} {1}.yml', verb, playbook) 46 | else: 47 | cmd = shell_format('ansible-playbook {0}.yml', playbook) 48 | 49 | if run(cmd, cwd=ansible_root).returncode > 0: 50 | click.echo('Operation failed.') 51 | return False 52 | else: 53 | return True 54 | 55 | 56 | def purge_install_node(): 57 | """ 58 | Purges the install node cache 59 | """ 60 | click.echo("Purging install node caches") 61 | sh = run('rm -rf {0}'.format(cache_root)) 62 | sh = run('mkdir -p {0} {1}'.format(cache_root, ansible_facts)) 63 | click.echo("OK") 64 | 65 | 66 | def purge_data_nodes(conf): 67 | """ 68 | Uninstalls ECS and purges the data node cache 69 | """ 70 | playbook = 'clicmd_purge' 71 | if not play(playbook, conf.config.verbosity): 72 | sys.exit(1) 73 | 74 | 75 | """ 76 | # Config 77 | """ 78 | 79 | 80 | class Conf(tui.Director): 81 | """ 82 | Director subclass for 83 | """ 84 | def __init__(self): 85 | tui.Director.__init__(self) 86 | 87 | 88 | pass_conf = click.make_pass_decorator(Conf, ensure=True) 89 | 90 | 91 | """ 92 | # Commands 93 | """ 94 | 95 | 96 | @click.group(chain=True) 97 | @click.option('-v', '--verbose', count=True, help="Use multiple times for more verbosity") 98 | @pass_conf 99 | def ecsremove(conf, verbose): 100 | """ 101 | Command line interface to remove ECS bits 102 | """ 103 | conf.config.verbosity = verbose 104 | 105 | 106 | @ecsremove.command('purge-nodes', short_help='Uninstall ECS and purge artifacts from data nodes') 107 | @pass_conf 108 | def purge_nodes(conf): 109 | purge_data_nodes(conf) 110 | 111 | 112 | @ecsremove.command('purge-installer', short_help='Purge caches from install node') 113 | @pass_conf 114 | def purge_installer(conf): 115 | purge_install_node() 116 | 117 | 118 | @ecsremove.command('purge-all', short_help='Uninstall ECS and purge artifacts from all nodes') 119 | @pass_conf 120 | def purge_all(conf): 121 | purge_install_node() 122 | purge_data_nodes(conf) 123 | 124 | 125 | if __name__ == '__main__': 126 | ecsremove() 127 | -------------------------------------------------------------------------------- /ui/etc/caches.yml: -------------------------------------------------------------------------------- 1 | caches: 2 | # OS package caches are a special situation where we rely on ansible's unarchive module 3 | # to do the unpacking, so always always always create tarballs here! 4 | os_CentOS_7: 5 | applies_to: os_CentOS_7 6 | pack_cmd: "tar -c {{ caches.os_CentOS_7.src }} | pigz -c5 > {{ caches.os_CentOS_7.dest }}" 7 | src: /var/cache/yum 8 | dest: package_cache.tgz 9 | unpack_cmd: "tar -xzf {{ caches.os_CentOS_7.dest }}" 10 | unpack_sem: cache_control.sem 11 | 12 | docker: 13 | applies_to: data_node 14 | pack_cmd: "docker save {{ caches.docker.src }} | pigz -c5 > {{ caches.docker.dest }}" 15 | # How to make this src: dynamic? Rockerfile ENV? 16 | src: "{{ product.common_name }}" 17 | dest: ecs-software.tgz 18 | unpack_cmd: "pigz -dc {{ caches.docker.dest }} | docker load" 19 | unpack_sem: cache_control.sem 20 | 21 | # This cache doesn't need an unpack command - that is handled by setuptools inside the 22 | # ECS storageos container. 23 | # ecs_python: 24 | # applies_to: data_node 25 | # pack_cmd: "wget -q caches.ecs_python.src " 26 | # src: https://github.com/kennethreitz/requests/archive/v2.9.1.tar.gz https://bootstrap.pypa.io/ez_setup.py https://pypi.python.org/packages/source/s/setuptools/setuptools-20.7.0.tar.gz#md5=5d12b39bf3e75e80fdce54e44b255615 27 | # dest: ez_setup.py 28 | # unpack_cmd: none 29 | # unpack_sem: cache_control.sem 30 | -------------------------------------------------------------------------------- /ui/etc/config.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015 EMC Corporation 2 | # All Rights Reserved 3 | # 4 | # This software contains the intellectual property of EMC Corporation 5 | # or is licensed to EMC Corporation from third parties. Use of this 6 | # software and the intellectual property contained therein is expressly 7 | # limited to the terms and conditions of the License Agreement under which 8 | # it is provided by or on behalf of EMC. 9 | 10 | # Jinja2 templated YAML 11 | # One level of recursion is supported. If you need more, use aliases or refactor. 12 | # http://pyyaml.org/wiki/PyYAMLDocumentation#Aliases 13 | --- 14 | ui: 15 | name: ECS Community Edition Install Node 16 | version: 2.9.1r 17 | host_root_dir: /opt/emc/ecs-install 18 | state_file: /opt/state.yml 19 | deploy_file: /opt/deploy.yml 20 | host_deploy_file: /opt/emc/ecs-install/deploy.yml 21 | host_state_file: /opt/emc/ecs-install/state.yml 22 | log: /var/log/ecs-install.ui.log 23 | host_log: /var/log/emc/ecs-install.ui.log 24 | cache_dir: /var/cache/emc/ecs-install 25 | host_cache_dir: /var/cache/emc/ecs-install 26 | ssl_dir: /opt/ssl 27 | host_ssl_dir: /opt/emc/ecs-install/ssl 28 | ssh_dir: /opt/ssh 29 | host_ssh_dir: /opt/emc/ecs-install/ssh 30 | ansible_templates: /usr/local/src/ui/ansible/templates 31 | ansible_resources: /usr/local/src/ui/resources/ 32 | ffx_sem: /opt/ffx.sem 33 | product: 34 | name: ECS 35 | version: 3.9.0.0 36 | vendor: Dell EMC 37 | flavor: Community Edition 38 | slogan: Free and Frictionless 39 | license_file: /usr/local/src/license.txt 40 | common_name: "emccorp/ecs-software:latest" 41 | -------------------------------------------------------------------------------- /ui/etc/generic_help.j2.yml: -------------------------------------------------------------------------------- 1 | help_refs: 2 | - name: GitHub ECS Community and Help 3 | url: "http://github.com/EMCECS/ECS-CommunityEdition" 4 | - name: "@EMCECS on Twitter" 5 | url: "http://twitter.com/EMCECS" 6 | - name: ECS Resources at EMC 7 | url: "http://community.emc.com/community/products/vipr" 8 | generic_help: &generic_help |- 9 | Use the arrow and page keys to navigate and [ENTER] or [SPACE] to select. 10 | If your terminal has mouse support, you can point and click around most pages. 11 | If you need more help, there are several resources to assist you. 12 | 13 | {% for ref in help_refs %}{{ ref.name }}: {{ref.url}} 14 | {% endfor %} 15 | -------------------------------------------------------------------------------- /ui/etc/release.conf: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015 EMC Corporation 2 | # All Rights Reserved 3 | # 4 | # This software contains the intellectual property of EMC Corporation 5 | # or is licensed to EMC Corporation from third parties. Use of this 6 | # software and the intellectual property contained therein is expressly 7 | # limited to the terms and conditions of the License Agreement under which 8 | # it is provided by or on behalf of EMC. 9 | 10 | release_name="ECS Community Edition" 11 | release_version="3.9.0.0" 12 | release_product="ECS Software" 13 | release_artifact="emccorp/ecs-software-3.9.0" 14 | release_tag="3.9.0.0" 15 | 16 | 17 | release_common_name="emccorp/ecs-software:latest" 18 | 19 | docker_host_root="/opt/emc/ecs-install" 20 | docker_host_logs="/var/log/emc" 21 | 22 | default_mount_opts=() 23 | default_mount_opts+=('--volumes-from ecs-install-data') 24 | default_mount_opts+=("-v ${docker_host_root}:/opt") 25 | default_mount_opts+=("-v ${docker_host_root}/ssh:/root/.ssh") 26 | default_mount_opts+=("-v ${docker_host_root}/ssl:/usr/local/share/ca-certificates") 27 | default_mount_opts+=("-v ${docker_host_logs}:/var/log") 28 | default_mount_opts+=("-v /var/cache:/var/cache") 29 | 30 | data_container_name="ecs-install-data" 31 | from_image='alpine:3.8' 32 | repo_name='emccorp' 33 | image_name='ecs-install' 34 | tag='latest' 35 | ver_maj='2' 36 | ver_min='9' 37 | ver_rev='0' 38 | ver_tag='r' 39 | serial=0 40 | 41 | # TODO: Fill in ecsdeploy_ vars with production values! 42 | # rocker_artifact_url="https://github.com/grammarly/rocker/releases/download/1.3.0/rocker_linux_amd64.tar.gz" 43 | rocker_artifact_url="https://github.com/grammarly/rocker/releases/download/1.3.1/rocker-1.3.1-linux_amd64.tar.gz" 44 | ecsdeploy_git_url="https://github.com/EMCECS/ECS-CommunityEdition.git" 45 | ecsdeploy_git_target="release" 46 | ansible_git_url="https://github.com/ansible/ansible.git" 47 | # New 2.1.1.0 release 48 | #ansible_git_target="780c363" 49 | # Old 2.0.0.1 dev 50 | #ansible_git_target="8d60b29" 51 | alpine_version="v3.8" 52 | image_release="${repo_name}/${image_name}:${tag}" 53 | -------------------------------------------------------------------------------- /ui/libexec/includes.sh: -------------------------------------------------------------------------------- 1 | #@IgnoreInspection BashAddShebang 2 | 3 | # DEPRECATED: shell-style environment flags are deprecated and will be replaced 4 | # DEPRECATED: by new context system, but some scripts may still depend on these. 5 | DRONE=${DRONE:-false} 6 | BUILD=${BUILD:-false} 7 | MONITOR=${DRONE:-false} 8 | 9 | etc="${root}/ui/etc" 10 | export etc 11 | source "${etc}/release.conf" 12 | 13 | if [ -f "${root}/release.conf" ]; then 14 | source "${root}/release.conf" 15 | fi 16 | 17 | if [ -f "${root}/bootstrap.conf" ]; then 18 | source "${root}/bootstrap.conf" 19 | fi 20 | 21 | source "${lib}/console.sh" 22 | log_file="${root}/install.log" 23 | 24 | source "${lib}/osutils.sh" 25 | source "${lib}/dockerutils.sh" 26 | source "${lib}/shipit.lib.sh" 27 | 28 | yay "${etc}/config.yml" 29 | import_vars config 30 | -------------------------------------------------------------------------------- /ui/libexec/versioning.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | context='local' 4 | 5 | # TODO: Better develop test than detecting ci directory 6 | if [ -d "ci" ]; then 7 | 8 | if [ -f "ci/ci_serial" ]; then 9 | serial="$(cat ci/ci_serial)" 10 | echo "$((++serial))" > "ci/ci_serial" 11 | else 12 | serial=0 13 | echo "$((++serial))" > "ci/ci_serial" 14 | fi 15 | 16 | ver_tag="dev.$(hostname -s).${serial}" 17 | context='develop' 18 | repo_name='padthaitofuhot' 19 | tag='develop' 20 | 21 | elif [ -f "build/serial" ]; then 22 | 23 | serial="$(cat build/serial)" 24 | echo "$((++serial))" > build/serial 25 | ver_tag="${ver_tag}-${serial}" 26 | context='release' 27 | 28 | elif [ -f local_serial ]; then 29 | 30 | serial="$(cat local_serial)" 31 | echo "$((++serial))" > local_serial 32 | ver_tag="local.$(hostname -s).${serial}" 33 | 34 | else 35 | 36 | serial=0 37 | echo "$((++serial))" > local_serial 38 | ver_tag="local.$(hostname -s).${serial}" 39 | 40 | fi 41 | 42 | version="${ver_maj}.${ver_min}.${ver_rev}-${ver_tag}" 43 | 44 | registry_flag=${registry_flag:-false} 45 | 46 | if $registry_flag; then 47 | repo_path="${registry_val}/${repo_name}" 48 | else 49 | repo_path="${repo_name}" 50 | fi 51 | 52 | full_image_path="${repo_path}/${image_name}:${version}" 53 | untagged_image_path="${repo_path}/${image_name}" 54 | latest_image_path="${repo_path}/${image_name}:${tag}" 55 | dockerhub_image_path="${repo_name}/${image_name}:${version}" 56 | -------------------------------------------------------------------------------- /ui/pull_image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | shopt -s extglob 3 | shopt -s xpg_echo 4 | set -o pipefail 5 | 6 | # Copyright (c) 2015 EMC Corporation 7 | # All Rights Reserved 8 | # 9 | # This software contains the intellectual property of EMC Corporation 10 | # or is licensed to EMC Corporation from third parties. Use of this 11 | # software and the intellectual property contained therein is expressly 12 | # limited to the terms and conditions of the License Agreement under which 13 | # it is provided by or on behalf of EMC. 14 | 15 | ##### Boilerplate ############################################################ 16 | # The build environment is always determined by the last bootstrap.sh run 17 | source "$HOME/.ecsinstallrc" 18 | source ${INSTALL_ROOT}/bootstrap.conf 19 | root=${INSTALL_ROOT} 20 | lib=${root}/ui/libexec 21 | cd ${root} 22 | # 23 | # Imports and import configs 24 | source ${lib}/includes.sh 25 | # 26 | ############################################################################## 27 | 28 | if ! [ -z "$1" ] && [ "$1" == "--clean" ]; then 29 | docker_clean 30 | exit 0 31 | fi 32 | 33 | registry_flag=${registry_flag:-false} 34 | 35 | if $registry_flag; then 36 | repo_path="${registry_val}/${image_release}" 37 | else 38 | repo_path="${image_release}" 39 | fi 40 | 41 | o "Cleaning up..." 42 | docker_clean 43 | 44 | #o "Login into Docker for image pull (rate limits)..." 45 | #docker_login 46 | 47 | o "Pulling image ${repo_path}" 48 | sudo docker pull ${repo_path} || img_pull_fail 49 | 50 | o "Tagging ${repo_path} -> ${image_release}" 51 | sudo docker tag "${repo_path}" "${image_release}" || img_pull_fail 52 | 53 | exit 0 54 | -------------------------------------------------------------------------------- /ui/requirements.txt: -------------------------------------------------------------------------------- 1 | --index-url https://pypi.python.org/simple/ 2 | 3 | -e . 4 | -------------------------------------------------------------------------------- /ui/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/resources/__init__.py -------------------------------------------------------------------------------- /ui/resources/docker/ecs-install-requirements.txt: -------------------------------------------------------------------------------- 1 | click>=6.7 2 | dotmap>=1.2.20 3 | futures>=3.1.1 4 | httplib2>=0.10.3 5 | ipaddress>=1.0.18 6 | itsdangerous>=0.24 7 | Jinja2>=2.9.6 8 | pager>=3.3 9 | paramiko>=2.3.0 10 | ptpython>=0.41 11 | cryptography>=2.1.4 12 | pycrypto>=2.6.1 13 | pyOpenSSL>=17.3.0 14 | python-ecsclient>=1.1.6 15 | PyYAML>=3.12 16 | requests>=2.18.4 17 | sarge>=0.1.4 18 | simplejson>=3.11.1 19 | six>=1.11.0 20 | urwid>=1.3.1 21 | urwid_timed_progress>=1.1.1 22 | pykwalify>=1.6.0 23 | -------------------------------------------------------------------------------- /ui/resources/docker/torrent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ash 2 | # Copyright (c) 2015 EMC Corporation 3 | # All Rights Reserved 4 | # 5 | # This software contains the intellectual property of EMC Corporation 6 | # or is licensed to EMC Corporation from third parties. Use of this 7 | # software and the intellectual property contained therein is expressly 8 | # limited to the terms and conditions of the License Agreement under which 9 | # it is provided by or on behalf of EMC. 10 | 11 | # Start torrent tracker and seeder 12 | torrent() { 13 | if [ -f /opt/ffx.sem ]; then 14 | opentracker -i 0.0.0.0 -p 6881 -d /var/run/opentracker -u nobody & 15 | cd /var/cache/emc 16 | aria2c -q -l /var/log/torrent.log -T /var/cache/emc/ecs-install/cache.torrent --seed-ratio=0.0 \ 17 | --dht-listen-port=6882 --allow-overwrite=true --check-integrity \ 18 | --listen-port=6883-6999 --enable-mmap=true 1>/dev/null 2>/dev/null & 19 | cd / 20 | fi 21 | } 22 | 23 | torrent 24 | -------------------------------------------------------------------------------- /ui/resources/futures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/resources/futures/__init__.py -------------------------------------------------------------------------------- /ui/resources/futures/caches.yml: -------------------------------------------------------------------------------- 1 | python: 2 | command: pip --no-cache-dir --disable-pip-version-check download {{ caches.python.src }} && touch success.sem 3 | src: docker-py 4 | dest: success.sem 5 | applies_to: data_node 6 | -------------------------------------------------------------------------------- /ui/resources/futures/mt_urwid/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.DS_Store 3 | -------------------------------------------------------------------------------- /ui/resources/futures/mt_urwid/Readme: -------------------------------------------------------------------------------- 1 | Overview 2 | -------- 3 | 4 | If you've not discovered it yet I strongly having a look at urwid: 5 | http://excess.org/urwid/ 6 | 7 | Bascially its a ncurses style console library but nicely wrapped in python and 8 | with much better support for making complex widgets. 9 | 10 | This repo basically includes a few widgets I built to experiment with the 11 | library. At the moment we have a scrollview, splitview and a tabview. 12 | 13 | 14 | License 15 | ------- 16 | 17 | Copyright (c) 2013 Mountainstorm 18 | 19 | Permission is hereby granted, free of charge, to any person obtaining a copy 20 | of this software and associated documentation files (the "Software"), to deal 21 | in the Software without restriction, including without limitation the rights 22 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | copies of the Software, and to permit persons to whom the Software is 24 | furnished to do so, subject to the following conditions: 25 | 26 | The above copyright notice and this permission notice shall be included in all 27 | copies or substantial portions of the Software. 28 | 29 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | SOFTWARE. -------------------------------------------------------------------------------- /ui/resources/futures/mt_urwid/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # 3 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/README.md: -------------------------------------------------------------------------------- 1 | pyservmonitor 2 | ============= 3 | 4 | Pyservmonitor is a simple python urwid interface to your one-liner scripts. 5 | 6 | the different entries in the interface are easily editable (in [config.ini](config.ini)) 7 | 8 | Available one-liners: 9 | 10 | * docker (using local library) 11 | * apache/nginx 12 | * fail2ban 13 | * portspoof 14 | * ... 15 | 16 | examples/pyservmonitor.png 17 | 18 | ![](examples/pyservmonitor.png "pyservmonitor") 19 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # 3 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/cfg_reader.py: -------------------------------------------------------------------------------- 1 | # author: deadc0de6 2 | # contact: https://github.com/deadc0de6 3 | # 4 | # the config file reader 5 | # 6 | # Copyright (C) 2014 deadc0de6 7 | # This program is free software; you can redistribute it and/or 8 | # modify it under the terms of the GNU General Public License 9 | # as published by the Free Software Foundation; either version 2 10 | # of the License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | # 21 | # format: 22 | # :: 23 | 24 | import re 25 | import csv 26 | from plugin import plugin 27 | 28 | class cfg_reader(): 29 | DEBUG = False 30 | 31 | def __init__(self, path): 32 | self._path = path 33 | self._plugins = [] 34 | self._parse() 35 | 36 | def _parse(self): 37 | with open(self._path, 'r') as f: 38 | reader = csv.reader(f, delimiter=':', quoting=csv.QUOTE_MINIMAL, quotechar='\'', escapechar='\\') 39 | for row in reader: 40 | self._parse_line(row) 41 | 42 | def _parse_line(self, fields): 43 | if self.DEBUG: 44 | print 'parsing %s ...' % (fields) 45 | if len(fields) != 3: 46 | if self.DEBUG: 47 | print 'line %s too few elemtns' % (fields) 48 | return 49 | if fields[0][0] == '#': 50 | if self.DEBUG: 51 | print 'line %s is a comment' % (fields) 52 | return 53 | 54 | name = fields[0] 55 | entry = fields[1] 56 | cmd = fields[2] 57 | 58 | if self.DEBUG: 59 | print 'adding: %s' % (fields) 60 | self._plugins.append(plugin(name, entry, cmd)) 61 | 62 | def get_plugins(self): 63 | return self._plugins 64 | 65 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/examples/pyservmonitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/resources/futures/pyservmonitor/examples/pyservmonitor.png -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/plugin.py: -------------------------------------------------------------------------------- 1 | # author: deadc0de6 2 | # contact: https://github.com/deadc0de6 3 | # 4 | # class representing a plugin 5 | # - name 6 | # - menu entry 7 | # - cmd to execute 8 | # 9 | # Copyright (C) 2014 deadc0de6 10 | # This program is free software; you can redistribute it and/or 11 | # modify it under the terms of the GNU General Public License 12 | # as published by the Free Software Foundation; either version 2 13 | # of the License, or (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program; if not, write to the Free Software 22 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 23 | # 24 | 25 | class plugin(): 26 | ''' 27 | @name: the plugin name 28 | @menu_txt: what will be displayed in the menu 29 | @cmd: command (shell one-liner) exected when menu selected 30 | ''' 31 | def __init__(self, name, menu_txt, cmd): 32 | self.name = name 33 | self.menu_txt = menu_txt 34 | self.cmd = cmd 35 | 36 | def __repr__(self): 37 | return 'name: \"%s\", menu: \"%s\", cmd: \"%s\"' % (self.name, self.menu_txt, self.cmd) 38 | -------------------------------------------------------------------------------- /ui/resources/futures/pyservmonitor/pyservmonitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # author: deadc0de6 3 | # contact: https://github.com/deadc0de6 4 | # 5 | # server monitorer frontend 6 | # 7 | # Copyright (C) 2014 deadc0de6 8 | # This program is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU General Public License 10 | # as published by the Free Software Foundation; either version 2 11 | # of the License, or (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | # 22 | # deps: 23 | # urwid (python-urwid in debian) 24 | 25 | import plugin 26 | from cfg_reader import cfg_reader 27 | import argparse 28 | from ui import * 29 | import os 30 | import sys 31 | 32 | def err(string): 33 | sys.stderr.write(string+'\n') 34 | 35 | def out(string): 36 | sys.stdout.write(string+'\n') 37 | 38 | parser = argparse.ArgumentParser(description='server manager frontend') 39 | parser.add_argument('--config', required=False, default='config.ini', help='the config file') 40 | args = parser.parse_args() 41 | cfg = args.config 42 | 43 | # check user is root 44 | #if os.geteuid() != 0: 45 | # err('get r00t !') 46 | # sys.exit(0) 47 | 48 | if not os.path.isfile(cfg) or not os.path.exists(cfg): 49 | err('config file \"%s\" does not exist !' % (cfg)) 50 | sys.exit(1) 51 | 52 | cfgr = cfg_reader(cfg) 53 | plugins = cfgr.get_plugins() 54 | err('%i entries parsed from config \"%s\"' % (len(plugins), cfg)) 55 | 56 | # Let's construct the lists for the ui 57 | # - up_entries: the menu entry 58 | # - script_match: the commands 59 | up_entries = [] 60 | script_match = {} 61 | for i,p in enumerate(plugins): 62 | up_entries.append(Entry('%i) %s' % (i+1, p.menu_txt))) 63 | script_match[i] = p.cmd 64 | 65 | ui(up_entries, script_match) 66 | 67 | -------------------------------------------------------------------------------- /ui/resources/futures/sqllog/log.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE log_script_bootstrap 2 | ( 3 | id INT PRIMARY KEY, 4 | ts DATETIME DEFAULT CURRENT_TIMESTAMP, 5 | log TEXT 6 | ); 7 | 8 | CREATE TABLE log_script_install 9 | ( 10 | id INT PRIMARY KEY, 11 | ts DATETIME DEFAULT CURRENT_TIMESTAMP, 12 | log TEXT 13 | ); 14 | 15 | CREATE TABLE log_configurator 16 | ( 17 | id INT PRIMARY KEY, 18 | ts DATETIME DEFAULT CURRENT_TIMESTAMP, 19 | log TEXT 20 | ); 21 | 22 | CREATE TABLE log_ansible 23 | ( 24 | id INT PRIMARY KEY, 25 | ts DATETIME DEFAULT CURRENT_TIMESTAMP, 26 | log TEXT 27 | ); 28 | 29 | CREATE TABLE log_general 30 | ( 31 | id INT PRIMARY KEY, 32 | ts DATETIME DEFAULT CURRENT_TIMESTAMP, 33 | log TEXT 34 | ); 35 | -------------------------------------------------------------------------------- /ui/resources/futures/sqllog/log.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/f0db9348c0d75273cc3b701072397292291ddc4a/ui/resources/futures/sqllog/log.sqlite -------------------------------------------------------------------------------- /ui/resources/futures/weave/weave.yml: -------------------------------------------------------------------------------- 1 | # - name: Networks 2 | # type: GatherForm 3 | # title: Deployment Networks 4 | # brief: Weave Networks Subnets 5 | # help: *generic_help 6 | # keys: 7 | # - help 8 | # - save 9 | # - exit 10 | # - color 11 | # args: 12 | # form: 13 | # - statement: > 14 | # The installer will connect deployed nodes with Docker Weave. 15 | # It needs two different private subnets to connect everything. 16 | # It is almost always safe to use the default RFC 3927 subnets because 17 | # nodes on a Docker Weave network are effectively link-local. 18 | # - question: Weave VPN Subnet 19 | # type: edit_line 20 | # var: weave_net_subnet 21 | # default: 169.254.253.0/24 22 | # - question: Weave DNS Subnet 23 | # type: edit_line 24 | # var: weave_dns_subnet 25 | # default: 169.254.254.0/24 26 | -------------------------------------------------------------------------------- /ui/setup.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from setuptools import setup, find_packages 3 | 4 | setup( 5 | name='ecsdeploy', 6 | version='2.11.0r', 7 | packages=find_packages(), 8 | scripts=['ui.py', 9 | 'ecsdeploy.py', 10 | 'ecsremove.py', 11 | 'ecsconfig.py', 12 | 'inventory.py'], 13 | include_package_data=True, 14 | package_data={'': ['*.yml']}, 15 | author="Travis Wichert @EMC", 16 | author_email="padthaitofuhot@users.noreply.github.com", 17 | license="Proprietary, https://raw.githubusercontent.com/EMCECS/ECS-CommunityEdition/master/license.txt", 18 | url='https://github.com/EMCECS/ECS-CommunityEdition', 19 | install_requires=[ 20 | 'Click', 21 | 'requests', 22 | 'simplejson', 23 | 'dotmap', 24 | 'PyYAML', 25 | 'Jinja2', 26 | 'httplib2', 27 | 'ipaddress', 28 | 'futures', 29 | 'urwid', 30 | 'urwid-timed-progress', 31 | 'pyopenssl', 32 | 'sarge', 33 | 'pykwalify', 34 | 'python-ecsclient' 35 | ], 36 | entry_points=''' 37 | [console_scripts] 38 | menu=ui:main 39 | ecsdeploy=ecsdeploy:ecsdeploy 40 | ecsremove=ecsremove:ecsremove 41 | ecsconfig=ecsconfig:ecsconfig 42 | inventory=inventory:inventory 43 | ''', 44 | ) 45 | -------------------------------------------------------------------------------- /ui/tui/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import constants 3 | from constants import * 4 | from tui import * 5 | from tools import * 6 | from tabview import * 7 | from widgets import * 8 | from ecsconf import * 9 | from director import * 10 | from ecsdiag import * 11 | # from ansibler import * 12 | -------------------------------------------------------------------------------- /ui/tui/ecsapi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ECS Python SDK Interface Class 3 | """ 4 | from ui.ecsmgmt import * 5 | -------------------------------------------------------------------------------- /ui/tui/ecsdiag.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for querying ECS diagnostic service 3 | """ 4 | 5 | import requests 6 | from xml.etree import ElementTree 7 | from constants import * 8 | 9 | 10 | class ECSDiag(object): 11 | """ 12 | A class to query the diagnostic service 13 | """ 14 | 15 | def __init__(self, endpoint): 16 | self.address = endpoint 17 | self.endpoint = '{0}://{1}:{2}'.format(DIAGNOSTIC_PROTOCOL, endpoint, DIAGNOSTIC_PORT) 18 | 19 | def get_dt_status(self): 20 | """ 21 | Query diag service for Data Table statuses 22 | """ 23 | status_dict = {} 24 | try: 25 | dt_xml = requests.get('{0}/stats/dt/DTInitStat/'.format(self.endpoint)) 26 | etree = ElementTree.fromstring(dt_xml.text) 27 | for entry in etree.findall("./entry//"): 28 | status_dict.update({entry.tag: int(entry.text)}) 29 | 30 | except Exception as e: 31 | status_dict.update({'unready_dt_num': 0, 'unknown_dt_num': -1}) 32 | 33 | status_dict.update({'endpoint': self.address}) 34 | 35 | return status_dict 36 | -------------------------------------------------------------------------------- /ui/tui/schema_functions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import socket 3 | import re 4 | 5 | Ipv4AddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" 6 | Ipv4CidrRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$" 7 | HostnameRegex = "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$" 8 | UnixPathRegex = "^(\/[^\/ ]*)+\/?$" 9 | AlphanumericRegex = "^[a-zA-Z0-9]+([_-]?[a-zA-Z0-9])*$" 10 | DockerImageRegex = "^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?